diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index aafd968ee0f..4f1a18e925e 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -6,12 +6,13 @@ ## Handwritten 1-3 sentence description - diff --git a/.github/workflows/osv-scan.yml b/.github/workflows/osv-scan.yml index 1c7e8b2841a..5016957bd6f 100644 --- a/.github/workflows/osv-scan.yml +++ b/.github/workflows/osv-scan.yml @@ -5,6 +5,9 @@ on: branches: - main - "litellm_**" + - "rc/**" + - "stable/**" + - "lts/**" schedule: - cron: "23 6 * * *" workflow_dispatch: diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2c05ee827a3..f1456b1fde8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -5,6 +5,9 @@ on: branches: - main - "litellm_**" + - "rc/**" + - "stable/**" + - "lts/**" push: branches: - main @@ -280,6 +283,7 @@ jobs: tests/unit/llms/e2b tests/unit/llms/edenai tests/unit/llms/elevenlabs + tests/unit/llms/empiriolabs tests/unit/llms/exa_ai tests/unit/llms/fal_ai tests/unit/llms/fastcrw @@ -415,6 +419,7 @@ jobs: - shard: router test-path: >- + tests/unit/router_rust tests/unit/test_router python-version: "3.12" workers: 4 @@ -441,6 +446,7 @@ jobs: tests/unit/ocr tests/unit/passthrough tests/unit/rag + tests/unit/cookbook tests/unit/realtime_api tests/unit/rerank_api tests/unit/search @@ -972,10 +978,10 @@ jobs: dist: loadscope steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }} + timeout-minutes: 3 with: persist-credentials: false - fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} + fetch-depth: 1 - name: Detect relevant changes id: changes @@ -1008,13 +1014,11 @@ jobs: with: artifact: ${{ needs.rust-bridge.outputs.artifact }} - - name: Install dependencies and check schema + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 8 + timeout-minutes: 5 env: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} - UNIT_SHARD: ${{ matrix.shard }} - SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }} run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then @@ -1026,9 +1030,21 @@ jobs: uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" fi uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' - if [ "$UNIT_SHARD" = "db-and-spend" ]; then - uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF" - fi + + - name: Check schema.prisma against the merge base + if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'db-and-spend' && github.event_name == 'pull_request' + timeout-minutes: 3 + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + test -n "$merge_base" + git fetch --no-tags --depth=1 origin "$merge_base" + uv run --no-sync python scripts/check_schema_compatibility.py --base-commit "$merge_base" - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' diff --git a/README.md b/README.md index 3b5710fa0c5..31a93bf8c67 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | | | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | | +| [EmpirioLabs AI (`empiriolabs`)](https://docs.litellm.ai/docs/providers/empiriolabs) | ✅ | | ✅ | | | | | | | | | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | | [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | | | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | @@ -444,6 +445,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | | | [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | | | [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | | +| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | | | [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | | | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | | | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py new file mode 100644 index 00000000000..827c65cdbfe --- /dev/null +++ b/cookbook/gpt_realtime_translate.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import base64 +import json +import os +import sys +import wave +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urlunsplit + +import websockets +from websockets.asyncio.client import ClientConnection + +SAMPLE_RATE = 24_000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 +CHUNK_DURATION_SECONDS = 0.1 +CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) +RESPONSE_TIMEOUT_SECONDS = 30.0 +AUDIO_EVENT_TYPES = frozenset( + { + "session.output_audio.delta", + "response.audio.delta", + "response.output_audio.delta", + } +) +TRANSCRIPT_EVENT_TYPES = frozenset( + { + "session.output_transcript.delta", + "response.text.delta", + "response.output_audio_transcript.delta", + } +) + + +@dataclass(frozen=True, slots=True) +class Settings: + input_wav: Path + output_wav: Path + base_url: str + model: str + target_language: str + trailing_silence_seconds: float + api_key: str + + +def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: + sys.stdout.write(f"{message}{end}") + if flush: + sys.stdout.flush() + + +def write_stderr(message: str) -> None: + sys.stderr.write(f"{message}\n") + + +def parse_args(argv: Sequence[str] | None = None) -> Settings | str: + parser = argparse.ArgumentParser( + description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", + ) + parser.add_argument("input_wav", type=Path) + parser.add_argument("--output", type=Path, default=Path("translated.wav")) + parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) + parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) + parser.add_argument("--target-language", default="fr") + parser.add_argument("--trailing-silence", type=float, default=1.5) + parsed = parser.parse_args(argv) + api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") + if not api_key: + return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" + if parsed.trailing_silence < 0: + return "--trailing-silence must be zero or greater" + return Settings( + input_wav=parsed.input_wav, + output_wav=parsed.output, + base_url=parsed.base_url, + model=parsed.model, + target_language=parsed.target_language, + trailing_silence_seconds=parsed.trailing_silence, + api_key=api_key, + ) + + +def translation_url(base_url: str, model: str) -> str | None: + parsed = urlsplit(base_url.rstrip("/")) + scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) + if not scheme or not parsed.netloc: + return None + base_path = parsed.path.rstrip("/") + realtime_path = ( + f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" + ) + return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) + + +def read_pcm16_wav(path: Path) -> bytes | str: + try: + with wave.open(str(path), "rb") as source: + actual_format = ( + source.getnchannels(), + source.getsampwidth(), + source.getframerate(), + source.getcomptype(), + ) + expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") + if actual_format != expected_format: + return ( + f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " + f"channels={actual_format[0]}, sample_width={actual_format[1]}, " + f"sample_rate={actual_format[2]}, compression={actual_format[3]}" + ) + return source.readframes(source.getnframes()) + except (OSError, EOFError, wave.Error) as exc: + return f"Unable to read {path}: {exc}" + + +def audio_chunks(audio: bytes) -> Iterator[bytes]: + return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) + + +def audio_message(audio: bytes) -> str: + return json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(audio).decode("ascii"), + } + ) + + +async def configure_session(connection: ClientConnection, target_language: str) -> str | None: + await connection.send( + json.dumps( + { + "type": "session.update", + "session": {"audio": {"output": {"language": target_language}}}, + } + ) + ) + while True: + raw_event = await asyncio.wait_for(connection.recv(), timeout=20) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type == "session.created": + write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") + if event_type == "session.updated": + return None + if event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + + +async def send_audio( + connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event +) -> None: + silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + try: + for chunk in audio_chunks(pcm + silence): + await connection.send(audio_message(chunk)) + await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + await connection.send(json.dumps({"type": "session.close"})) + finally: + finished.set() + + +async def receive_translation( + connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event +) -> str | None: + audio_received = asyncio.Event() + try: + with wave.open(str(output_path), "wb") as output: + output.setnchannels(CHANNELS) + output.setsampwidth(SAMPLE_WIDTH) + output.setframerate(SAMPLE_RATE) + write_stdout("Translation: ", end="", flush=True) + while True: + try: + raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS) + except TimeoutError: + if not sender_finished.is_set(): + continue + return ( + f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session" + ) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type in AUDIO_EVENT_TYPES: + output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) + audio_received.set() + elif event_type in TRANSCRIPT_EVENT_TYPES: + write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) + elif event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + elif event_type == "session.closed": + write_stdout() + return None if audio_received.is_set() else "The translation stream ended without translated audio" + except (OSError, wave.Error) as exc: + return f"Unable to write {output_path}: {exc}" + + +async def translate(settings: Settings, pcm: bytes) -> str | None: + url = translation_url(settings.base_url, settings.model) + if not url: + return f"Invalid --base-url: {settings.base_url}" + sender_finished = asyncio.Event() + try: + async with websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {settings.api_key}"}, + proxy=None, + open_timeout=20, + close_timeout=5, + ) as connection: + configuration_error = await configure_session(connection, settings.target_language) + if configuration_error: + return configuration_error + async with asyncio.TaskGroup() as tasks: + receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) + tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) + return receiver.result() + except Exception as exc: + return f"Translation failed: {type(exc).__name__}: {exc}" + + +def main(argv: Sequence[str] | None = None) -> int: + settings = parse_args(argv) + if isinstance(settings, str): + write_stderr(settings) + return 2 + pcm = read_pcm16_wav(settings.input_wav) + if isinstance(pcm, str): + write_stderr(pcm) + return 2 + error = asyncio.run(translate(settings, pcm)) + if error: + write_stderr(error) + return 1 + write_stdout(f"Translated audio: {settings.output_wav}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index cfe4ca98145..69e36a0c076 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -200,7 +200,6 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi unpriced: Final = _without_read_price(billed.response) cost: Final = logging_obj.response_cost_calculator(result=unpriced) end_time: Final = datetime.now() - logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time) GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( async_coroutine=logging_obj.async_success_handler( result=unpriced, start_time=logging_obj.start_time, end_time=end_time diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index a3cb388ffc6..059889cb667 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.3 +version: 1.1.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index 694cca05b52..bc6e29632f1 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -56,8 +56,8 @@ If `db.useStackgresOperator` is used (not yet implemented): | `billingMetrics.enabled` | Enable enterprise billable-request metering. Requires an enterprise license. | `false` | | `billingMetrics.endpoint` | Collector that the billable-request counter is pushed to. | `https://telemetry.litellm.ai` | -| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`. | `litellm-billing-metrics-mtls` | -| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI. | `""` | +| `billingMetrics.tokenUrl` | Cognito OAuth2 client-credentials token endpoint. | `https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token` | +| `billingMetrics.secretName` | Name of an existing Secret with `client-id` and `client-secret` keys. | `litellm-billing-metrics` | | `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset. | `""` | #### Example `proxy_config` ConfigMap from values (default): @@ -109,10 +109,10 @@ type: Opaque #### Enterprise billable-request metering -Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on: +Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector with Cognito bearer authentication. Create a Secret with `client-id` and `client-secret` keys, then enable the chart values: ``` -kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key +kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=... ``` ``` @@ -120,7 +120,8 @@ billingMetrics: enabled: true ``` -Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time. +The endpoint and token URL must use HTTPS. The chart requires both URLs and +the Secret name when billing metrics are enabled ### Database Settings diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 929fc415163..230d2acaeef 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -50,53 +50,27 @@ app.kubernetes.io/name: {{ include "litellm.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} -{{/* -Enterprise billable-request metering. The client certificate identifies the -deployment to LiteLLM's collector, so it is mounted read-only from an existing -Secret rather than passed through the environment. -*/}} -{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}} -{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}} - {{- define "litellm.billingMetricsEnv" -}} - name: LITELLM_BILLING_METRICS_ENDPOINT value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }} -{{- if .Values.billingMetrics.caSecretName }} -- name: LITELLM_BILLING_METRICS_CA_CERT - value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }} -{{- end }} +- name: LITELLM_BILLING_METRICS_TOKEN_URL + value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }} +- name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-id +- name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-secret {{- with .Values.billingMetrics.exportIntervalMs }} - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: {{ . | quote }} {{- end }} {{- end -}} -{{- define "litellm.billingMetricsVolumes" -}} -- name: billing-metrics-mtls - secret: - secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }} -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - secret: - secretName: {{ .Values.billingMetrics.caSecretName }} -{{- end }} -{{- end -}} - -{{- define "litellm.billingMetricsVolumeMounts" -}} -- name: billing-metrics-mtls - mountPath: {{ include "litellm.billingMetrics.certDir" . }} - readOnly: true -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - mountPath: {{ include "litellm.billingMetrics.caDir" . }} - readOnly: true -{{- end }} -{{- end -}} - {{/* Create the name of the service account to use */}} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index 37e7eaec9e9..a37193f0efa 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -148,9 +148,6 @@ spec: - name: npm mountPath: /.npm {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} {{- if include "litellm.collector.socketDir" . }} - name: collector-socket mountPath: {{ include "litellm.collector.socketDir" . }} @@ -234,9 +231,6 @@ spec: items: - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }} path: "config.yaml" - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} - {{- end }} {{- if include "litellm.collector.socketDir" . }} - name: collector-socket emptyDir: diff --git a/helm/litellm-helm/tests/billing_metrics_tests.yaml b/helm/litellm-helm/tests/billing_metrics_tests.yaml index 71803df378c..52185c8dd9d 100644 --- a/helm/litellm-helm/tests/billing_metrics_tests.yaml +++ b/helm/litellm-helm/tests/billing_metrics_tests.yaml @@ -1,31 +1,21 @@ -suite: test billingMetrics wiring on the proxy deployment +suite: test bearer-only billingMetrics wiring templates: - deployment.yaml - configmap-litellm.yaml - - migrations-job.yaml tests: - - it: is off by default, adding no env, volume, or mount + - it: is disabled by default template: deployment.yaml asserts: - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - notContains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - notContains: path: spec.template.spec.containers[0].env content: name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai + - notContains: + path: spec.template.spec.volumes + content: + name: billing-metrics - - it: renders the endpoint and the mounted cert paths when enabled + - it: renders required URLs and Cognito secret references when enabled template: deployment.yaml set: billingMetrics: @@ -39,119 +29,34 @@ tests: - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: /etc/litellm/billing-mtls/tls.crt + name: LITELLM_BILLING_METRICS_TOKEN_URL + value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: /etc/litellm/billing-mtls/tls.key - - # The conventional Secret name is the default, so enabling the block is enough. - - it: mounts the default cert secret read-only alongside the config volume - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - - it: honours a secretName override - template: deployment.yaml - set: - billingMetrics: - enabled: true - secretName: my-billing-mtls - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: my-billing-mtls - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - - it: honours an endpoint override - template: deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: https://collector.internal:4318 - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-id - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://collector.internal:4318 - - # The production collector presents a public web-PKI certificate, so the CA - # override must stay absent unless a private collector is configured. - - it: omits the CA env, volume, and mount when no caSecretName is set - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-secret - notContains: path: spec.template.spec.volumes content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca + name: billing-metrics - notContains: path: spec.template.spec.containers[0].volumeMounts content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt + name: billing-metrics - - it: mounts the CA secret when caSecretName is set - template: deployment.yaml - set: - billingMetrics: - enabled: true - caSecretName: billing-ca - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true - - - it: passes the export interval through only when set + - it: renders the export interval only when set template: deployment.yaml set: billingMetrics: @@ -164,129 +69,7 @@ tests: name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: "5000" - - it: omits the export interval when unset - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS - value: "60000" - - # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing - # entries must render after .Values.envVars or a user could silently redirect - # the metering export. The three billing entries are the last ones emitted here - # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case). - - it: renders the billing endpoint after envVars so it cannot be shadowed - template: deployment.yaml - set: - migrationJob: - enabled: false - billingMetrics: - enabled: true - envVars: - LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://shadowed.example - - equal: - path: spec.template.spec.containers[0].env[-3] - value: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - equal: - path: spec.template.spec.containers[0].env[-2].name - value: LITELLM_BILLING_METRICS_CLIENT_CERT - - equal: - path: spec.template.spec.containers[0].env[-1].name - value: LITELLM_BILLING_METRICS_CLIENT_KEY - - - it: keeps user-supplied volumes and mounts alongside the billing secret - template: deployment.yaml - set: - billingMetrics: - enabled: true - volumes: - - name: custom-callbacks - configMap: - name: my-callbacks - volumeMounts: - - name: custom-callbacks - mountPath: /app/callbacks - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: custom-callbacks - configMap: - name: my-callbacks - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: custom-callbacks - mountPath: /app/callbacks - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - - it: still mounts the proxy config when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - # Only the proxy serves billable traffic. The migrations Job must never mount - # the client certificate, and it renders its own env and volumes, so nothing - # stops a future edit from wiring the billing include into it by mistake. - - it: does not touch the migrations job when enabled - template: migrations-job.yaml - set: - billingMetrics: - enabled: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - notExists: - path: spec.template.spec.containers[0].volumeMounts - - notExists: - path: spec.template.spec.volumes - - - it: fails loudly when enabled with an emptied secretName - template: deployment.yaml - set: - billingMetrics: - enabled: true - secretName: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key) - - - it: fails loudly when enabled without an endpoint + - it: requires the endpoint when enabled template: deployment.yaml set: billingMetrics: @@ -295,3 +78,23 @@ tests: asserts: - failedTemplate: errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true + + - it: requires the token URL when enabled + template: deployment.yaml + set: + billingMetrics: + enabled: true + tokenUrl: "" + asserts: + - failedTemplate: + errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true + + - it: requires the secret name when enabled + template: deployment.yaml + set: + billingMetrics: + enabled: true + secretName: "" + asserts: + - failedTemplate: + errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true diff --git a/helm/litellm-helm/tests/collector_tests.yaml b/helm/litellm-helm/tests/collector_tests.yaml index 0340b1161b7..1365290e16c 100644 --- a/helm/litellm-helm/tests/collector_tests.yaml +++ b/helm/litellm-helm/tests/collector_tests.yaml @@ -191,7 +191,7 @@ tests: metricsServer.port: 9090 billingMetrics.enabled: true billingMetrics.endpoint: https://metering.example.com - billingMetrics.secretName: billing-mtls + billingMetrics.secretName: billing-metrics asserts: - contains: path: spec.template.spec.containers[0].env @@ -214,9 +214,9 @@ tests: name: LITELLM_BILLING_METRICS_ENDPOINT any: true - notContains: - path: spec.template.spec.containers[1].volumeMounts + path: spec.template.spec.containers[1].env content: - name: billing-metrics-mtls + name: LITELLM_BILLING_METRICS_CLIENT_SECRET any: true - it: should give the sidecar the same scratch mounts as the proxy on a read-only root diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index f51f8bdd173..9afce050261 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -154,19 +154,13 @@ masterkeySecretName: "" # if set, use this secret key for the master key; otherwise, use the default key masterkeySecretKey: "" -# Optional: enterprise billable-request metering. When enabled, the proxy counts -# successful requests to inference, MCP, and A2A endpoints and pushes them to -# LiteLLM's collector over mutual TLS. Requires an enterprise license. -# The client certificate identifies the deployment, so it is mounted read-only -# from an existing Secret and never passed through the environment. +# Optional: enterprise billable-request metering. Requires an enterprise license. billingMetrics: enabled: false - endpoint: https://telemetry.litellm.ai # collector to push the counter to - secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key - # Only for private or test collectors whose server certificate is not on the - # public web PKI. The production collector needs no CA override. - caSecretName: "" # existing Secret holding ca.crt - exportIntervalMs: "" # push cadence; the proxy defaults to 60000 + endpoint: https://telemetry.litellm.ai + tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token + secretName: litellm-billing-metrics + exportIntervalMs: "" proxyConfigMap: # when true, creates a new configmap diff --git a/helm/litellm/Chart.yaml b/helm/litellm/Chart.yaml index e67f5790c7e..341172da925 100644 --- a/helm/litellm/Chart.yaml +++ b/helm/litellm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: litellm description: LiteLLM componentized — gateway, UI backend, and UI as separate services type: application -version: 0.1.0 +version: 0.1.1 appVersion: "0.1.0" annotations: org.opencontainers.image.source: "https://github.com/BerriAI/litellm" diff --git a/helm/litellm/README.md b/helm/litellm/README.md index 97b94754402..d66bb3284bf 100644 --- a/helm/litellm/README.md +++ b/helm/litellm/README.md @@ -22,6 +22,23 @@ backend: gatewayUrl: https://gateway.example.com ``` +## Enterprise billing metrics + +Billing metrics use Cognito bearer authentication. Create the Secret with +`client-id` and `client-secret` keys, then enable billing metrics: + +```bash +kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=... +``` + +```yaml +billingMetrics: + enabled: true +``` + +The chart requires an HTTPS endpoint, an HTTPS token URL, and the Secret name +when billing metrics are enabled + ## Routes to another Service For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 18e938564e9..b704782f5d1 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -58,56 +58,27 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- end -}} -{{/* -Enterprise billable-request metering. Wired into gateway and backend, not the -migrations job. The gateway serves nearly all billable traffic, but the backend -keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a -SpendLogs row, so metering only the gateway would silently drop that traffic. -The client certificate identifies the deployment to LiteLLM's collector, so it is -mounted read-only from an existing Secret rather than passed through the -environment. -*/}} -{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}} -{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}} - {{- define "litellm.billingMetricsEnv" -}} - name: LITELLM_BILLING_METRICS_ENDPOINT value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }} -{{- if .Values.billingMetrics.caSecretName }} -- name: LITELLM_BILLING_METRICS_CA_CERT - value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }} -{{- end }} +- name: LITELLM_BILLING_METRICS_TOKEN_URL + value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }} +- name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-id +- name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-secret {{- with .Values.billingMetrics.exportIntervalMs }} - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: {{ . | quote }} {{- end }} {{- end -}} -{{- define "litellm.billingMetricsVolumes" -}} -- name: billing-metrics-mtls - secret: - secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }} -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - secret: - secretName: {{ .Values.billingMetrics.caSecretName }} -{{- end }} -{{- end -}} - -{{- define "litellm.billingMetricsVolumeMounts" -}} -- name: billing-metrics-mtls - mountPath: {{ include "litellm.billingMetrics.certDir" . }} - readOnly: true -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - mountPath: {{ include "litellm.billingMetrics.caDir" . }} - readOnly: true -{{- end }} -{{- end -}} {{/* Per-component selector labels — used in both Service selectors and Deployment matchLabels. diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 7360261dab1..7db14ad87a5 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -79,16 +79,13 @@ spec: {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} {{- include "litellm.envFrom" .Values.backend | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config mountPath: /app/config/config.yaml subPath: config.yaml {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} {{- with .Values.backend.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -114,16 +111,13 @@ spec: {{- with .Values.backend.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.backend.volumes }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config configMap: name: {{ include "litellm.gateway.fullname" . }}-config {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} - {{- end }} {{- with .Values.backend.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index b0e7f0dcebb..987109e361e 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -86,7 +86,7 @@ spec: {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} {{- end }} {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -101,9 +101,6 @@ spec: - name: collector-socket mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} {{- with .Values.gateway.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -206,7 +203,7 @@ spec: {{- with .Values.gateway.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config @@ -222,9 +219,6 @@ spec: emptyDir: sizeLimit: 1Mi {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} - {{- end }} {{- with .Values.gateway.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/litellm/tests/billing_metrics_tests.yaml b/helm/litellm/tests/billing_metrics_tests.yaml index ceba0bd1430..d17b8192fce 100644 --- a/helm/litellm/tests/billing_metrics_tests.yaml +++ b/helm/litellm/tests/billing_metrics_tests.yaml @@ -1,39 +1,30 @@ -suite: test billingMetrics wiring on gateway and backend +suite: test bearer-only billingMetrics wiring templates: - gateway/deployment.yaml - gateway/configmap.yaml - backend/deployment.yaml + - backend/configmap.yaml - migrations-job.yaml values: - ./values/required.yaml tests: - - it: is off by default, adding no env, volume, or mount + - it: is disabled by default template: gateway/deployment.yaml asserts: - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - equal: - path: spec.template.spec.containers[0].volumeMounts - value: - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - notContains: path: spec.template.spec.containers[0].env content: name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai + - notContains: + path: spec.template.spec.volumes + content: + name: billing-metrics - - it: renders the endpoint and the mounted cert paths when enabled + - it: wires gateway URLs and bearer credentials from Secret keys template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls asserts: - contains: path: spec.template.spec.containers[0].env @@ -43,87 +34,68 @@ tests: - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: /etc/litellm/billing-mtls/tls.crt + name: LITELLM_BILLING_METRICS_TOKEN_URL + value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: /etc/litellm/billing-mtls/tls.key - - - it: mounts the cert secret read-only alongside the config volume - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-id - contains: - path: spec.template.spec.volumes + path: spec.template.spec.containers[0].env content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - # The production collector presents a public web-PKI certificate, so the CA - # override must stay absent unless a private collector is configured. - - it: omits the CA env, volume, and mount when no caSecretName is set - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-secret - notContains: path: spec.template.spec.volumes content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca + name: billing-metrics - notContains: - path: spec.template.spec.containers[0].env + path: spec.template.spec.containers[0].volumeMounts content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt + name: billing-metrics - - it: mounts the CA secret when caSecretName is set - template: gateway/deployment.yaml + - it: wires billing metrics on the backend without volumes + template: backend/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls - caSecretName: billing-ca asserts: - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt - - contains: + name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-secret + - notContains: path: spec.template.spec.volumes content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true + name: billing-metrics - - it: passes the export interval through only when set + - it: does not add billing metrics to migrations + template: migrations-job.yaml + set: + billingMetrics: + enabled: true + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + + - it: renders an optional export interval template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls exportIntervalMs: 5000 asserts: - contains: @@ -132,102 +104,27 @@ tests: name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: "5000" - - it: keeps user-supplied gateway volumes alongside the billing secret + - it: requires the endpoint when enabled template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls - gateway.volumes: - - name: custom-callbacks - configMap: - name: my-callbacks - gateway.volumeMounts: - - name: custom-callbacks - mountPath: /app/callbacks + endpoint: "" asserts: - - contains: - path: spec.template.spec.volumes - content: - name: custom-callbacks - configMap: - name: my-callbacks - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls + - failedTemplate: + errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true - # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp), - # which writes a SpendLogs row, so it must meter too or that traffic is lost. - - it: meters the backend as well, since it serves the MCP transport - template: backend/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - - it: leaves the backend alone when metering is off - template: backend/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - # The migrations job runs prisma and serves no traffic; it must never receive - # the client key. - - it: never mounts the billing cert on the migrations job - template: migrations-job.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - isNull: - path: spec.template.spec.volumes - - # The conventional Secret name is the default, so enabling metering needs no - # secretName at all; the guard below only fires on an explicitly blanked one. - - it: uses the conventional secret name by default + - it: requires the token URL when enabled template: gateway/deployment.yaml set: billingMetrics: enabled: true + tokenUrl: "" asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls + - failedTemplate: + errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true - - it: fails loudly when the secretName is explicitly blanked + - it: requires the secret name when enabled template: gateway/deployment.yaml set: billingMetrics: @@ -235,15 +132,4 @@ tests: secretName: "" asserts: - failedTemplate: - errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key) - - - it: fails loudly when enabled without an endpoint - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: "" - secretName: billing-mtls - asserts: - - failedTemplate: - errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true + errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index 2ebb1b44926..b9e52ba7123 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -91,7 +91,7 @@ tests: app.kubernetes.io/name: litellm app.kubernetes.io/instance: RELEASE-NAME app.kubernetes.io/managed-by: Helm - helm.sh/chart: litellm-0.1.0 + helm.sh/chart: litellm-0.1.1 app.kubernetes.io/component: migrations - it: renders pod-level and container-level securityContext in their own scopes diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 92a48153c5a..cb37656cb77 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -180,23 +180,14 @@ masterKey: secretKey: master-key # Optional: enterprise billable-request metering. When enabled, the gateway and -# backend count successful requests to inference, MCP, and A2A endpoints and push -# them to LiteLLM's collector over mutual TLS. Both components serve billable -# routes: the backend keeps the named-server MCP transport. Requires an -# enterprise license. The client certificate identifies the deployment, so it is -# mounted read-only from an existing Secret and never passed through the env. +# backend count successful requests to inference, MCP, and A2A endpoints. Requires +# an enterprise license. billingMetrics: enabled: false - endpoint: https://telemetry.litellm.ai # collector to push the counter to - # An existing Secret holding the client certificate under tls.crt and its key - # under tls.key, usually created from the onboarding artifact. The default is - # the conventional name, so the common path is to create that Secret and set - # enabled: true. Override only if yours is named differently. - secretName: litellm-billing-metrics-mtls - # Only for private or test collectors whose server certificate is not on the - # public web PKI. The production collector needs no CA override. - caSecretName: "" # existing Secret holding ca.crt - exportIntervalMs: "" # push cadence; the proxy defaults to 60000 + endpoint: https://telemetry.litellm.ai + tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token + secretName: litellm-billing-metrics + exportIntervalMs: "" # External Postgres connection. database: diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql new file mode 100644 index 00000000000..b00cf892b68 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql @@ -0,0 +1,19 @@ +SET LOCAL lock_timeout = '10s'; +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" ( + "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + "request_id" TEXT NOT NULL, + "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending', + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "applied_at" TIMESTAMP(3), + "last_error" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "accounting_payload" JSONB NOT NULL, + "is_batch" BOOLEAN NOT NULL +); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_next_attempt_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "next_attempt_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_applied_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "applied_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_request_id_idx" + ON "LiteLLM_SpendLogDelivery" ("request_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104000000_skill_zip_upload/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104000000_skill_zip_upload/migration.sql new file mode 100644 index 00000000000..acbe87515d9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104000000_skill_zip_upload/migration.sql @@ -0,0 +1,9 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginUpload" ( + "plugin_name" TEXT NOT NULL, + "content" BYTEA NOT NULL, + CONSTRAINT "LiteLLM_ClaudeCodePluginUpload_pkey" PRIMARY KEY ("plugin_name"), + CONSTRAINT "LiteLLM_ClaudeCodePluginUpload_plugin_name_fkey" FOREIGN KEY ("plugin_name") + REFERENCES "LiteLLM_ClaudeCodePluginTable"("name") ON DELETE CASCADE ON UPDATE CASCADE +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql new file mode 100644 index 00000000000..cba6a18368f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql @@ -0,0 +1,5 @@ +SET LOCAL lock_timeout = '10s'; +DROP TABLE IF EXISTS "LiteLLM_AccountingDelivery"; +DROP TABLE IF EXISTS "LiteLLM_AccountingEvent"; +DROP TYPE IF EXISTS "AccountingDeliveryTarget"; +DROP TYPE IF EXISTS "AccountingEventKind"; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 179b741fe7c..78f16d9f1fd 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1615,9 +1615,18 @@ model LiteLLM_ClaudeCodePluginTable { updated_at DateTime? @default(now()) @updatedAt created_by String? + uploaded_archive LiteLLM_ClaudeCodePluginUpload? + @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginUpload { + plugin_name String @id + content Bytes + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_name], references: [name], onDelete: Cascade, onUpdate: Cascade) +} + + // User/team-scoped memory store with a GLOBAL unique key. // `value` is a string (typically markdown/text meant for LLM context); // `metadata` is an optional JSON envelope for structured tags without schema changes. @@ -2062,74 +2071,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f91deaaa401..23ebfc7d3cf 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3389,15 +3389,19 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-credential-types", + "aws-runtime", "aws-sdk-sts", "aws-sigv4", "aws-smithy-runtime-api", + "aws-smithy-types", "aws-types", "bytes", "futures-util", "litellm-auth-types", "litellm-error", "litellm-http", + "litellm-secret-value", + "litellm-secrets", "moka", "reqwest", "rstest", @@ -3406,6 +3410,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "url", "veil", ] @@ -3418,6 +3423,7 @@ dependencies = [ "litellm-auth-types", "litellm-error", "litellm-secret-value", + "litellm-secrets", "moka", "rstest", "serde_json", @@ -3431,6 +3437,7 @@ dependencies = [ name = "litellm-auth-gcp" version = "0.1.0" dependencies = [ + "axum", "base64 0.22.1", "gcp_auth", "google-cloud-auth", @@ -3440,10 +3447,12 @@ dependencies = [ "litellm-secret-value", "moka", "rstest", + "rustls 0.23.42", "serde", "serde_json", "sha2 0.10.9", "tempfile", + "time", "tokio", "veil", ] @@ -3569,37 +3578,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "litellm-cache-redis" -version = "0.1.0" -dependencies = [ - "litellm-cache", - "litellm-cache-testing", - "litellm-error", - "r2d2", - "redis", - "redis-test", - "rstest", - "serde_json", - "thiserror 2.0.19", - "tokio", -] - -[[package]] -name = "litellm-cache-redis-semantic" -version = "0.1.0" -dependencies = [ - "litellm-cache", - "litellm-cache-redis", - "litellm-cache-testing", - "redis", - "redis-test", - "rstest", - "serde_json", - "sha2 0.10.9", - "tokio", -] - [[package]] name = "litellm-cache-response" version = "0.1.0" @@ -3607,11 +3585,8 @@ dependencies = [ "litellm-cache", "litellm-cache-gcs", "litellm-cache-memory", - "litellm-cache-redis", "litellm-http", "py_literal", - "redis", - "redis-test", "rstest", "serde", "serde_json", @@ -3650,24 +3625,6 @@ dependencies = [ "litellm-cache", ] -[[package]] -name = "litellm-cache-valkey-semantic" -version = "0.1.0" -dependencies = [ - "litellm-cache", - "litellm-cache-redis", - "litellm-cache-testing", - "litellm-error", - "redis", - "redis-test", - "rstest", - "serde_json", - "sha2 0.10.9", - "thiserror 2.0.19", - "tokio", - "uuid", -] - [[package]] name = "litellm-callbacks-legacy-python" version = "0.1.0" @@ -4331,6 +4288,7 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "async-trait", "aws-sdk-secretsmanager", "bytes", "criterion", @@ -4339,8 +4297,6 @@ dependencies = [ "litellm-auth", "litellm-auth-aws", "litellm-cache", - "litellm-cache-memory", - "litellm-cache-redis", "litellm-cache-response", "litellm-callbacks-legacy-python", "litellm-core-utils", @@ -4357,6 +4313,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-model-catalog", + "litellm-router-core", "litellm-router-types", "litellm-secret-value", "litellm-secrets", @@ -4369,7 +4326,6 @@ dependencies = [ "prost", "pyo3", "pyo3-async-runtimes", - "redis", "reqwest", "rstest", "serde", @@ -4439,6 +4395,24 @@ dependencies = [ [[package]] name = "litellm-router-core" version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "jiff", + "litellm-cache", + "litellm-router-types", + "parking_lot", + "rand 0.8.7", + "rstest", + "serde", + "serde_json", + "serde_with", + "sha2 0.10.9", + "strum", + "thiserror 2.0.19", + "tokio", + "tracing", +] [[package]] name = "litellm-router-types" @@ -5782,17 +5756,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - [[package]] name = "rand" version = "0.8.7" @@ -5966,18 +5929,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "redis-test" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804d36862e4323b69f96440cbb13c9894fc90176abdeaf91264e21d5d77f6aca" -dependencies = [ - "rand 0.9.5", - "redis", - "socket2 0.6.5", - "tempfile", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -6454,15 +6405,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "schemars" version = "0.9.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 75cac2a5d63..71914a0bffe 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" } litellm-error = { path = "crates/error" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } +litellm-router-core = { path = "crates/router-core" } litellm-tracing = { path = "crates/tracing" } litellm-spend-clickhouse = { path = "crates/spend-clickhouse" } litellm-storage-clickhouse = { path = "crates/storage-clickhouse" } @@ -57,11 +58,9 @@ litellm-db-testing = { path = "crates/db-testing" } litellm-cache = { path = "crates/cache" } litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } -litellm-cache-redis = { path = "crates/cache-redis" } litellm-cache-s3 = { path = "crates/cache-s3" } litellm-cache-gcs = { path = "crates/cache-gcs" } litellm-cache-disk = { path = "crates/cache-disk" } -litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" } litellm-cache-response = { path = "crates/cache-response" } litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" } litellm-cache-testing = { path = "crates/cache-testing" } @@ -78,6 +77,8 @@ litellm-cost = { path = "crates/cost" } litellm-model-catalog = { path = "crates/model-catalog" } litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" } +arc-swap = "1.9.2" +async-trait = "0.1.91" askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] } tracing = "0.1" axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] } @@ -97,7 +98,6 @@ macro_rules_attribute = "0.2.3" schemars = "1" reqwest = { version = "0.13", default-features = false, features = ["json", "multipart", "rustls", "http2", "stream", "query"] } redis = "1.8.0" -redis-test = "1.0.4" testcontainers-modules = "0.15.0" qdrant-client = { version = "1.19.0", default-features = false } uuid = { version = "1", features = ["v4"] } diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index e4578777a08..c8ed7bb0f14 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -11,14 +11,16 @@ litellm-error.workspace = true litellm-http.workspace = true reqwest.workspace = true -moka = { workspace = true, features = ["sync"] } +moka.workspace = true +litellm-secret-value.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } sha2.workspace = true thiserror.workspace = true +url.workspace = true veil.workspace = true -aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] } +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] } aws-sigv4 = "1.5.1" @@ -26,6 +28,9 @@ aws-types = "1.4.0" aws-smithy-runtime-api = "1.13.0" [dev-dependencies] +litellm-secrets.workspace = true +aws-smithy-types = "1.6.4" +aws-runtime = { version = "1.9.4", default-features = false } rstest.workspace = true litellm-http = { workspace = true, features = ["test-support"] } bytes.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index 81aa4b068bc..e6e32754b4f 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -1,13 +1,16 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; -use moka::sync::Cache; -use sha2::{Digest, Sha256}; +use aws_credential_types::provider::{SharedCredentialsProvider, error::CredentialsError}; +use litellm_secret_value::SecretValue; +use moka::{Expiry, future::Cache}; +use std::{future::Future, pin::Pin, sync::Arc}; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ - SignableBody, SignableRequest, SigningParams, SigningSettings, sign, + SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams, + SigningSettings, sign, }; use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; @@ -23,72 +26,106 @@ use super::constants::{ }; use crate::error::{CredentialFlow, Error, SigningStage}; -const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); -const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600); +const CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600); +const EXPIRY_BUFFER: Duration = Duration::from_secs(60); #[derive(Clone)] pub struct AwsAuthService { - static_credentials: Cache, - ambient_credentials: Cache, + providers: Cache, + credentials: Cache, + loader: Arc, } impl Default for AwsAuthService { fn default() -> Self { + Self::new(Arc::new(NativeProviderLoader)) + } +} + +impl AwsAuthService { + fn new(loader: Arc) -> Self { Self { - static_credentials: Cache::builder() + providers: Cache::builder().max_capacity(200).build(), + credentials: Cache::builder() .max_capacity(200) - .time_to_live(STATIC_CREDENTIALS_TTL) - .build(), - ambient_credentials: Cache::builder() - .max_capacity(200) - .time_to_live(AMBIENT_CREDENTIALS_TTL) + .expire_after(CredentialExpiry) .build(), + loader, } } } -fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option { - match flow { - AwsAuthFlow::StaticKeys { .. } => Some(STATIC_CREDENTIALS_TTL), - AwsAuthFlow::DefaultChain => Some(AMBIENT_CREDENTIALS_TTL), - AwsAuthFlow::WebIdentity { .. } - | AwsAuthFlow::AssumeRole { .. } - | AwsAuthFlow::Profile { .. } - | AwsAuthFlow::SessionToken { .. } => None, +struct CredentialExpiry; + +impl Expiry for CredentialExpiry { + fn expire_after_create( + &self, + _: &AwsAuthConfig, + credentials: &Credentials, + _: std::time::Instant, + ) -> Option { + Some(credentials.expiry().map_or(CREDENTIAL_CACHE_TTL, |expiry| { + expiry + .duration_since(SystemTime::now()) + .unwrap_or_default() + .saturating_sub(EXPIRY_BUFFER) + .min(CREDENTIAL_CACHE_TTL) + })) } } -#[derive(Clone, Debug, Default, PartialEq, Eq)] +type ProviderFuture<'a> = + Pin> + Send + 'a>>; + +trait ProviderLoader: Send + Sync { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>; +} + +struct NativeProviderLoader; + +impl ProviderLoader for NativeProviderLoader { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { + Box::pin(build_provider(config, flow)) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] pub struct AwsAuthConfig { - pub access_key_id: Option, - pub secret_access_key: Option, - pub session_token: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, pub region_name: Option, pub session_name: Option, pub profile_name: Option, pub role_name: Option, - pub web_identity_token: Option, + pub web_identity_token: Option, pub sts_endpoint: Option, - pub external_id: Option, + pub external_id: Option, } impl AwsAuthConfig { fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Self { Self { - access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)), + access_key_id: self + .access_key_id + .or_else(|| env_lookup(AWS_ACCESS_KEY_ID).map(SecretValue::new)), secret_access_key: self .secret_access_key - .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)), - session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)), + .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY).map(SecretValue::new)), + session_token: self + .session_token + .or_else(|| env_lookup(AWS_SESSION_TOKEN).map(SecretValue::new)), region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)), session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)), profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)), role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)), web_identity_token: self .web_identity_token - .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)), + .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN).map(SecretValue::new)), sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)), - external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)), + external_id: self + .external_id + .or_else(|| env_lookup(AWS_EXTERNAL_ID).map(SecretValue::new)), } } } @@ -96,52 +133,31 @@ impl AwsAuthConfig { #[derive(Clone, Debug, PartialEq, Eq)] pub enum AwsAuthFlow { WebIdentity { - token: String, + token: SecretValue, role: String, session_name: String, }, AssumeRole { role: String, session_name: Option, + web_identity_role: Option, }, Profile { name: String, }, SessionToken { - access_key_id: String, - secret_access_key: String, - session_token: String, + access_key_id: SecretValue, + secret_access_key: SecretValue, + session_token: SecretValue, }, StaticKeys { - access_key_id: String, - secret_access_key: String, + access_key_id: SecretValue, + secret_access_key: SecretValue, region_name: String, }, DefaultChain, } -fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String { - let mut hasher = Sha256::new(); - hasher.update(format!("{config:?}:{flow:?}")); - format!("{:x}", hasher.finalize()) -} - -impl AwsAuthService { - fn get_cached_credentials(&self, key: &str) -> Option { - self.static_credentials - .get(key) - .or_else(|| self.ambient_credentials.get(key)) - } - - fn set_cached_credentials(&self, key: String, credentials: Credentials, ttl: Duration) { - if ttl == STATIC_CREDENTIALS_TTL { - self.static_credentials.insert(key, credentials); - } else { - self.ambient_credentials.insert(key, credentials); - } - } -} - fn role_identity(arn: &str) -> Option<(&str, &str, &str)> { let mut parts = arn.splitn(6, ':'); let ("arn", partition, _, _, account, resource) = ( @@ -170,6 +186,9 @@ pub fn classify_auth( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> AwsAuthFlow { + let web_identity_role = env_lookup(AWS_WEB_IDENTITY_TOKEN_FILE) + .filter(|token_file| !token_file.is_empty()) + .and_then(|_| env_lookup(AWS_ROLE_ARN)); let config = config.with_environment(env_lookup); if let (Some(token), Some(role), Some(session_name)) = ( config.web_identity_token.clone(), @@ -186,6 +205,7 @@ pub fn classify_auth( return AwsAuthFlow::AssumeRole { role, session_name: config.session_name.clone(), + web_identity_role, }; } if let Some(name) = config.profile_name { @@ -216,6 +236,30 @@ pub fn classify_auth( AwsAuthFlow::DefaultChain } +async fn profile_provider( + name: &str, + region: Option, + config: aws_config::provider_config::ProviderConfig, + builder: aws_config::profile::credentials::Builder, +) -> Result { + let region = match region { + Some(region) => Some(aws_types::region::Region::new(region)), + None => { + aws_config::default_provider::region::DefaultRegionChain::builder() + .profile_name(name) + .build() + .region() + .await + } + }; + Ok(SharedCredentialsProvider::new( + builder + .configure(&config.with_region(region)) + .profile_name(name) + .build(), + )) +} + impl AwsAuthService { pub async fn resolve_credentials( &self, @@ -224,183 +268,240 @@ impl AwsAuthService { ) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); - match flow { - AwsAuthFlow::SessionToken { - access_key_id, - secret_access_key, - session_token, - } => Ok(Credentials::new( - access_key_id, - secret_access_key, - Some(session_token), - None, - "litellm-static-session", - )), - AwsAuthFlow::StaticKeys { - access_key_id, - secret_access_key, - region_name, - } => { - let flow = AwsAuthFlow::StaticKeys { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - region_name, - }; - let key = cache_key(&resolved, &flow); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let credentials = Credentials::new( - access_key_id, - secret_access_key, - None, - None, - "litellm-static", - ); - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL), - ); - Ok(credentials) - } - AwsAuthFlow::Profile { name } => { - let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() - .profile_name(name) - .build(); + self.fresh_credentials(&resolved, &flow).await + } + + /// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings + /// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads. + pub async fn default_chain_credentials(&self, region: &str) -> Result { + let config = AwsAuthConfig { + region_name: Some(region.to_owned()), + ..Default::default() + }; + self.fresh_credentials(&config, &AwsAuthFlow::DefaultChain) + .await + } + + async fn fresh_credentials( + &self, + resolved: &AwsAuthConfig, + flow: &AwsAuthFlow, + ) -> Result { + let cached = self.cached_credentials(resolved, flow).await?; + if fresh(&cached) { + return Ok(cached); + } + self.credentials.invalidate(resolved).await; + let refetched = self.cached_credentials(resolved, flow).await?; + if fresh(&refetched) { + return Ok(refetched); + } + self.credentials.invalidate(resolved).await; + Err(Error::ExpiredCredentials) + } + + async fn cached_credentials( + &self, + resolved: &AwsAuthConfig, + flow: &AwsAuthFlow, + ) -> Result { + self.credentials + .try_get_with(resolved.clone(), async { + let provider = self + .providers + .try_get_with( + resolved.clone(), + self.loader.load(resolved.clone(), flow.clone()), + ) + .await + .map_err(|error| (*error).clone())?; provider .provide_credentials() .await - .map_err(Error::credentials(CredentialFlow::Profile)) - } - AwsAuthFlow::AssumeRole { role, session_name } => { - if is_already_running_as_role(&role, &resolved).await? { - let ambient_flow = AwsAuthFlow::DefaultChain; - let key = cache_key(&resolved, &ambient_flow); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::DefaultChain))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - return Ok(credentials); - } - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - if let (Some(access_key_id), Some(secret_access_key)) = - (resolved.access_key_id, resolved.secret_access_key) - { - loader = loader.credentials_provider(Credentials::new( - access_key_id, - secret_access_key, - resolved.session_token, - None, - "litellm-role-source", - )); - } - let sdk_config = loader.load().await; - let builder = aws_config::sts::AssumeRoleProvider::builder(role); - let builder = match session_name { - Some(name) => builder.session_name(name), - None => builder.session_name(default_session_name()), - }; - let builder = match resolved.external_id { - Some(id) => builder.external_id(id), - None => builder, - }; - let provider = builder.configure(&sdk_config).build().await; - provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::AssumeRole)) - } - AwsAuthFlow::WebIdentity { - token, - role, - session_name, - } => { - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let client = aws_sdk_sts::Client::new(&sdk_config); - let response = client - .assume_role_with_web_identity() - .role_arn(role) - .role_session_name(session_name) - .web_identity_token(token) - .send() - .await - .map_err(Error::credentials(CredentialFlow::WebIdentity))?; - let credentials = response - .credentials() - .ok_or(Error::MissingWebIdentityCredentials)?; - let expiration = - SystemTime::try_from(*credentials.expiration()).map_err(|source| { - Error::WebIdentityExpiration(litellm_error::Source::new(source)) - })?; - Ok(Credentials::new( - credentials.access_key_id(), - credentials.secret_access_key(), - Some(credentials.session_token().to_string()), - Some(expiration), - "litellm-web-identity", - )) - } - AwsAuthFlow::DefaultChain => { - let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::DefaultChain))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&AwsAuthFlow::DefaultChain) - .unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - Ok(credentials) + .map_err(Error::credentials(flow.credential_flow())) + }) + .await + .map_err(|error| (*error).clone()) + } +} + +fn fresh(credentials: &Credentials) -> bool { + credentials + .expiry() + .is_none_or(|expiry| expiry > SystemTime::now() + EXPIRY_BUFFER) +} + +impl AwsAuthFlow { + fn credential_flow(&self) -> CredentialFlow { + match self { + Self::Profile { .. } => CredentialFlow::Profile, + Self::AssumeRole { .. } => CredentialFlow::AssumeRole, + Self::WebIdentity { .. } => CredentialFlow::WebIdentity, + Self::DefaultChain | Self::StaticKeys { .. } | Self::SessionToken { .. } => { + CredentialFlow::DefaultChain } } } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { +async fn build_provider( + resolved: AwsAuthConfig, + flow: AwsAuthFlow, +) -> Result { + match flow { + AwsAuthFlow::SessionToken { + access_key_id, + secret_access_key, + session_token, + } => Ok(SharedCredentialsProvider::new(Credentials::new( + access_key_id.expose(), + secret_access_key.expose(), + Some(session_token.expose().to_owned()), + None, + "litellm-static-session", + ))), + AwsAuthFlow::StaticKeys { + access_key_id, + secret_access_key, + .. + } => Ok(SharedCredentialsProvider::new(Credentials::new( + access_key_id.expose(), + secret_access_key.expose(), + None, + None, + "litellm-static", + ))), + AwsAuthFlow::Profile { name } => { + profile_provider( + &name, + resolved.region_name, + aws_config::provider_config::ProviderConfig::without_region(), + aws_config::profile::ProfileFileCredentialsProvider::builder(), + ) + .await + } + AwsAuthFlow::DefaultChain => { + let builder = + aws_config::default_provider::credentials::DefaultCredentialsChain::builder(); + let builder = match resolved.region_name { + Some(region) => builder.region(aws_types::region::Region::new(region)), + None => builder, + }; + Ok(SharedCredentialsProvider::new(builder.build().await)) + } + AwsAuthFlow::AssumeRole { + role, + session_name, + web_identity_role, + } => { + if is_already_running_as_role(&role, web_identity_role.as_deref(), &resolved).await? { + return Ok(SharedCredentialsProvider::new( + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await, + )); + } + let sdk_config = sdk_config(resolved.clone()).await; + let builder = aws_config::sts::AssumeRoleProvider::builder(role) + .session_name(session_name.unwrap_or_else(default_session_name)); + let builder = match resolved.external_id { + Some(id) => builder.external_id(id.expose()), + None => builder, + }; + Ok(SharedCredentialsProvider::new( + builder.configure(&sdk_config).build().await, + )) + } + AwsAuthFlow::WebIdentity { + token, + role, + session_name, + } => Ok(SharedCredentialsProvider::new(WebIdentityProvider { + client: aws_sdk_sts::Client::new(&sdk_config(resolved).await), + token, + role, + session_name, + })), + } +} + +async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig { + let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + let loader = match config.region_name { + Some(region) => loader.region(aws_types::region::Region::new(region)), + None => loader, + }; + let loader = match config.sts_endpoint { + Some(endpoint) => loader.endpoint_url(endpoint), + None => loader, + }; + let loader = match (config.access_key_id, config.secret_access_key) { + (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( + key.expose(), + secret.expose(), + config.session_token.map(|token| token.expose().to_owned()), + None, + "litellm-role-source", + )), + _ => loader, + }; + loader.load().await +} + +#[derive(Debug)] +struct WebIdentityProvider { + client: aws_sdk_sts::Client, + token: SecretValue, + role: String, + session_name: String, +} + +impl ProvideCredentials for WebIdentityProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + let response = self + .client + .assume_role_with_web_identity() + .role_arn(&self.role) + .role_session_name(&self.session_name) + .web_identity_token(self.token.expose()) + .send() + .await + .map_err(CredentialsError::provider_error)?; + let credentials = response.credentials().ok_or_else(|| { + CredentialsError::provider_error(Error::MissingWebIdentityCredentials) + })?; + let expiration = SystemTime::try_from(*credentials.expiration()) + .map_err(CredentialsError::provider_error)?; + Ok(Credentials::new( + credentials.access_key_id(), + credentials.secret_access_key(), + Some(credentials.session_token().to_owned()), + Some(expiration), + "litellm-web-identity", + )) + }) + } +} + +async fn is_already_running_as_role( + role: &str, + web_identity_role: Option<&str>, + config: &AwsAuthConfig, +) -> Result { + if config.access_key_id.is_some() || config.secret_access_key.is_some() { + return Ok(false); + } if role_identity(role).is_none() { return Ok(false); } - if let (Ok(current_role), Ok(token_file)) = ( - std::env::var(AWS_ROLE_ARN), - std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE), - ) && !token_file.is_empty() - { - return Ok(same_role_arns(role, ¤t_role)); + if let Some(current_role) = web_identity_role { + return Ok(same_role_arns(role, current_role)); } let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); @@ -473,16 +574,6 @@ pub fn sign_headers( headers: &HeaderMap, scope: SigningScope<'_>, ) -> Result { - let identity: Identity = scope.credentials.clone().into(); - let params = v4::SigningParams::builder() - .identity(&identity) - .region(scope.region) - .name(scope.service) - .time(scope.time) - .settings(SigningSettings::default()) - .build() - .map(SigningParams::from) - .map_err(Error::signing(SigningStage::Parameters))?; let header_refs = headers .iter() .map(|(name, value)| { @@ -494,16 +585,14 @@ pub fn sign_headers( }) }) .collect::, _>>()?; - let request = SignableRequest::new( + let instructions = sign_request( method, url, header_refs.into_iter(), - SignableBody::Bytes(body), - ) - .map_err(Error::signing(SigningStage::Request))?; - let (instructions, _) = sign(request, ¶ms) - .map_err(Error::signing(SigningStage::Signature))? - .into_parts(); + body, + SigningSettings::default(), + scope, + )?; instructions .headers() .map(|(name, value)| { @@ -516,6 +605,60 @@ pub fn sign_headers( .collect() } +pub fn presign_url( + url: &str, + expires_in: Duration, + scope: SigningScope<'_>, +) -> Result { + let mut signed = url::Url::parse(url).map_err(Error::signing(SigningStage::Request))?; + let mut settings = SigningSettings::default(); + settings.signature_location = SignatureLocation::QueryParams; + settings.expires_in = Some(expires_in); + let instructions = sign_request( + "GET", + signed.as_str(), + std::iter::empty(), + &[], + settings, + scope, + )?; + let (signature, auth): (Vec<_>, Vec<_>) = instructions + .params() + .iter() + .map(|(name, value)| (*name, value.as_ref())) + .partition(|(name, _)| *name == "X-Amz-Signature"); + signed + .query_pairs_mut() + .extend_pairs(auth.into_iter().chain(signature)); + Ok(signed.into()) +} + +fn sign_request<'a>( + method: &'a str, + url: &'a str, + headers: impl Iterator, + body: &'a [u8], + settings: SigningSettings, + scope: SigningScope<'_>, +) -> Result { + let identity: Identity = scope.credentials.clone().into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(scope.region) + .name(scope.service) + .time(scope.time) + .settings(settings) + .build() + .map(SigningParams::from) + .map_err(Error::signing(SigningStage::Parameters))?; + let request = SignableRequest::new(method, url, headers, SignableBody::Bytes(body)) + .map_err(Error::signing(SigningStage::Request))?; + let (instructions, _) = sign(request, ¶ms) + .map_err(Error::signing(SigningStage::Signature))? + .into_parts(); + Ok(instructions) +} + /// Model-id and region parsing shared by every Bedrock route. pub fn bedrock_model_id_and_region(model: &str) -> (String, Option) { let mut stripped = model; @@ -586,16 +729,26 @@ pub fn aws_auth_config( env_lookup: &dyn Fn(&str) -> Option, ) -> AwsAuthConfig { AwsAuthConfig { - access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup), - secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup), - session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup), + access_key_id: params + .resolve(&AwsParams::ACCESS_KEY_ID, env_lookup) + .map(SecretValue::new), + secret_access_key: params + .resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup) + .map(SecretValue::new), + session_token: params + .resolve(&AwsParams::SESSION_TOKEN, env_lookup) + .map(SecretValue::new), region_name: params.resolve(&AwsParams::REGION, env_lookup), session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup), profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup), role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup), - web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup), + web_identity_token: params + .resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup) + .map(SecretValue::new), sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup), - external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup), + external_id: params + .resolve(&AwsParams::EXTERNAL_ID, env_lookup) + .map(SecretValue::new), } } @@ -660,10 +813,133 @@ mod tests { use crate::constants::{AWS_REGION, BEDROCK_SERVICE}; use reqwest::header::CONTENT_TYPE; + #[cfg(unix)] + #[rstest::fixture] + fn process_profile() -> aws_runtime::env_config::file::EnvConfigFiles { + let credentials = serde_json::json!({ + "Version": 1, + "AccessKeyId": "process-access", + "SecretAccessKey": "process-secret", + "SessionToken": "process-session", + }); + aws_runtime::env_config::file::EnvConfigFiles::builder() + .include_default_config_file(false) + .include_default_credentials_file(false) + .with_contents( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + format!( + "[profile smoke]\ncredential_process = /usr/bin/printf '%s' '{credentials}'\n" + ), + ) + .build() + } + + #[cfg(unix)] + #[rstest::rstest] + #[tokio::test] + async fn profile_process_credentials_can_be_loaded( + process_profile: aws_runtime::env_config::file::EnvConfigFiles, + ) { + let provider = profile_provider( + "smoke", + Some("us-west-2".into()), + aws_config::provider_config::ProviderConfig::without_region(), + aws_config::profile::ProfileFileCredentialsProvider::builder() + .profile_files(process_profile), + ) + .await + .unwrap(); + let credentials = provider.provide_credentials().await.unwrap(); + + assert_eq!(credentials.access_key_id(), "process-access"); + assert_eq!(credentials.secret_access_key(), "process-secret"); + assert_eq!(credentials.session_token(), Some("process-session")); + } + + #[derive(Debug)] + struct StsConnector; + + impl aws_smithy_runtime_api::client::http::HttpConnector for StsConnector { + fn call( + &self, + request: aws_smithy_runtime_api::client::orchestrator::HttpRequest, + ) -> aws_smithy_runtime_api::client::http::HttpConnectorFuture { + assert!(request.uri().contains("sts.us-west-2.")); + aws_smithy_runtime_api::client::http::HttpConnectorFuture::new(async { + Ok( + aws_smithy_runtime_api::client::orchestrator::HttpResponse::new( + 200_u16.try_into().unwrap(), + aws_smithy_types::body::SdkBody::from( + r#" + + role-accessrole-secret + role-session2100-01-01T00:00:00Z + arn:aws:sts::123456789012:assumed-role/smoke/session + smoke:session + smoke"#, + ), + ), + ) + }) + } + } + + #[rstest::rstest] + #[tokio::test] + async fn role_profile_uses_region_when_acquiring_sts_credentials() { + use aws_smithy_runtime_api::client::http::{SharedHttpConnector, http_client_fn}; + let files = aws_runtime::env_config::file::EnvConfigFiles::builder() + .include_default_config_file(false) + .include_default_credentials_file(false) + .with_contents( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + "[profile smoke]\nrole_arn = arn:aws:iam::123456789012:role/smoke\nsource_profile = source\n\n[profile source]\naws_access_key_id = source-access\naws_secret_access_key = source-secret\n", + ).build(); + let config = aws_config::provider_config::ProviderConfig::without_region() + .with_http_client(http_client_fn(|_, _| { + SharedHttpConnector::new(StsConnector) + })); + let credentials = profile_provider( + "smoke", + Some("us-west-2".into()), + config, + aws_config::profile::ProfileFileCredentialsProvider::builder().profile_files(files), + ) + .await + .unwrap(); + let credentials = credentials.provide_credentials().await.unwrap(); + assert_eq!(credentials.access_key_id(), "role-access"); + assert_eq!(credentials.secret_access_key(), "role-secret"); + assert_eq!(credentials.session_token(), Some("role-session")); + assert!(credentials.expiry().is_some()); + } + fn no_env(_: &str) -> Option { None } + #[rstest::rstest] + #[case::access_key(Some("source-access"), None)] + #[case::secret_key(None, Some("source-secret"))] + #[case::source_pair(Some("source-access"), Some("source-secret"))] + #[tokio::test] + async fn explicit_source_keys_cannot_reuse_an_ambient_role( + #[case] access_key: Option<&str>, + #[case] secret_key: Option<&str>, + ) { + let role = "arn:aws:iam::123456789012:role/target"; + let config = AwsAuthConfig { + access_key_id: access_key.map(SecretValue::new), + secret_access_key: secret_key.map(SecretValue::new), + ..Default::default() + }; + assert!( + !is_already_running_as_role(role, Some(role), &config) + .await + .unwrap() + ); + } + #[test] fn secret_names_cover_environment_reads() { let seen = std::sync::Arc::new(std::sync::Mutex::new( @@ -706,7 +982,10 @@ mod tests { }; assert_eq!( - aws_auth_config(¶ms, &env).secret_access_key.as_deref(), + aws_auth_config(¶ms, &env) + .secret_access_key + .as_ref() + .map(SecretValue::expose), expected ); assert_eq!( @@ -831,6 +1110,53 @@ mod tests { ); } + #[rstest::rstest] + #[case::running_with_web_identity("/var/run/token", Some("arn:aws:iam::1:role/current"))] + #[case::empty_token_file("", None)] + #[tokio::test] + async fn assume_role_reads_the_ambient_role_from_the_secret_snapshot( + #[case] token_file: &str, + #[case] expected: Option<&str>, + ) { + use litellm_secrets::source::SecretSource; + + struct WebIdentitySecrets<'a>(&'a str); + + impl SecretSource for WebIdentitySecrets<'_> { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> futures_util::future::BoxFuture< + 'a, + Result, litellm_secrets::Error>, + > { + Box::pin(async move { + Ok(match name { + AWS_WEB_IDENTITY_TOKEN_FILE => Some(SecretValue::new(self.0)), + AWS_ROLE_ARN => Some(SecretValue::new("arn:aws:iam::1:role/current")), + _ => None, + }) + }) + } + } + + let snapshot = WebIdentitySecrets(token_file) + .resolve(AwsParams::secret_names()) + .await + .unwrap(); + let config = AwsAuthConfig { + role_name: Some("arn:aws:iam::1:role/target".into()), + ..Default::default() + }; + let AwsAuthFlow::AssumeRole { + web_identity_role, .. + } = classify_auth(config, &|name| snapshot.get(name)) + else { + panic!("a role name selects AssumeRole"); + }; + assert_eq!(web_identity_role.as_deref(), expected); + } + #[tokio::test] async fn static_credentials_do_not_use_network() { let credentials = AwsAuthService::default() @@ -849,110 +1175,231 @@ mod tests { assert_eq!(credentials.session_token(), None); } - #[test] - fn cache_policy_matches_python_flows() { - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::StaticKeys { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - region_name: "us-east-1".into(), - }), - Some(STATIC_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::DefaultChain), - Some(AMBIENT_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::SessionToken { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - session_token: "token".into(), - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::Profile { - name: "profile".into() - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::AssumeRole { - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: None, - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::WebIdentity { - token: "token".into(), - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: "session".into(), - }), - None - ); + struct CountingLoader { + loads: Arc, + calls: Arc, + short_lifetime: Duration, + short_calls: usize, + } + + impl ProviderLoader for CountingLoader { + fn load(&self, config: AwsAuthConfig, _: AwsAuthFlow) -> ProviderFuture<'_> { + let loads = self.loads.clone(); + let provider = CountingProvider { + calls: self.calls.clone(), + short_lifetime: self.short_lifetime, + short_calls: self.short_calls, + key: config.access_key_id.unwrap(), + }; + Box::pin(async move { + loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(SharedCredentialsProvider::new(provider)) + }) + } + } + + #[derive(Debug)] + struct CountingProvider { + calls: Arc, + short_lifetime: Duration, + short_calls: usize, + key: SecretValue, + } + + impl ProvideCredentials for CountingProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async move { + tokio::task::yield_now().await; + let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let lifetime = if call < self.short_calls { + self.short_lifetime + } else { + Duration::from_secs(3600) + }; + Ok(Credentials::new( + self.key.expose(), + "credential-value", + Some(format!("session-{call}")), + Some(SystemTime::now() + lifetime), + "counting", + )) + }) + } + } + + #[derive(Default)] + struct RecordingLoader { + loads: std::sync::Mutex>, + } + + impl ProviderLoader for RecordingLoader { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { + self.loads.lock().unwrap().push((config, flow)); + Box::pin(async { + Ok(SharedCredentialsProvider::new(Credentials::new( + "ak", + "sk", + None, + None, + "recording", + ))) + }) + } } #[rstest::rstest] - fn cache_round_trip_preserves_credentials() { - let auth = AwsAuthService::default(); - let key = format!("cache-test-{}", std::process::id()); - let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test"); - auth.set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL); + #[tokio::test] + async fn default_chain_credentials_do_not_share_a_configured_role_in_the_same_region() { + let loader = Arc::new(RecordingLoader::default()); + let auth = AwsAuthService::new(loader.clone()); + let bedrock_env = |name: &str| match name { + AWS_ROLE_NAME => Some("arn:aws:iam::123456789012:role/bedrock".to_string()), + AWS_REGION_NAME => Some("us-east-1".to_string()), + _ => None, + }; + auth.resolve_credentials(AwsAuthConfig::default(), &bedrock_env) + .await + .unwrap(); + + auth.default_chain_credentials("us-east-1").await.unwrap(); + auth.default_chain_credentials("us-east-1").await.unwrap(); + + let loads = loader.loads.lock().unwrap(); + assert_eq!(loads.len(), 2); assert_eq!( - auth.get_cached_credentials(&key) - .map(|value| value.access_key_id().to_string()), - Some("cache-ak".to_string()) + loads[1], + ( + AwsAuthConfig { + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + AwsAuthFlow::DefaultChain + ) ); } #[rstest::rstest] #[tokio::test] - async fn cloned_services_reuse_credentials_but_independent_services_do_not() { - let auth = AwsAuthService::default(); + async fn near_expiry_credentials_are_refreshed_without_rebuilding_the_provider() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let loads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: loads.clone(), + calls: calls.clone(), + short_lifetime: Duration::from_secs(30), + short_calls: 1, + })); let config = AwsAuthConfig { - access_key_id: Some("configured-key".into()), - secret_access_key: Some("configured-secret".into()), - region_name: Some("us-east-1".into()), - ..AwsAuthConfig::default() + access_key_id: Some("identity".into()), + ..Default::default() }; - let flow = classify_auth(config.clone(), &no_env); - let cached = Credentials::new("cached-key", "cached-secret", None, None, "test"); - auth.set_cached_credentials( - cache_key(&config, &flow), - cached.clone(), - STATIC_CREDENTIALS_TTL, - ); - - let reused = auth + let first = auth + .resolve_credentials(config.clone(), &no_env) + .await + .unwrap(); + let second = auth .clone() .resolve_credentials(config.clone(), &no_env) .await .unwrap(); - let independent = AwsAuthService::default() - .resolve_credentials(config.clone(), &no_env) - .await - .unwrap(); - let different = AwsAuthConfig { - access_key_id: Some("different-key".into()), - ..config.clone() - }; - let other_identity = auth - .resolve_credentials(different.clone(), &no_env) - .await - .unwrap(); + assert_eq!(first.session_token(), Some("session-1")); + assert_eq!(first.session_token(), second.session_token()); + assert_eq!(loads.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } - assert_eq!(reused.access_key_id(), cached.access_key_id()); - assert_eq!(reused.secret_access_key(), cached.secret_access_key()); + #[rstest::rstest] + #[case::recovered_by_one_refetch(1, Some("session-1"))] + #[case::still_expired_after_refetch(2, None)] + #[tokio::test] + async fn expired_credentials_are_refetched_once( + #[case] short_calls: usize, + #[case] session: Option<&str>, + ) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: Arc::new(AtomicUsize::new(0)), + calls: calls.clone(), + short_lifetime: Duration::ZERO, + short_calls, + })); + let config = AwsAuthConfig { + access_key_id: Some("identity".into()), + ..Default::default() + }; + let result = auth.resolve_credentials(config, &no_env).await; assert_eq!( - Some(independent.access_key_id()), - config.access_key_id.as_deref() + result.as_ref().ok().and_then(Credentials::session_token), + session ); - assert_eq!( - Some(other_identity.access_key_id()), - different.access_key_id.as_deref() + assert!(session.is_some() || matches!(result, Err(Error::ExpiredCredentials))); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[tokio::test] + async fn concurrent_calls_share_acquisition_and_distinct_identities_stay_isolated() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let loads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: loads.clone(), + calls: calls.clone(), + short_lifetime: Duration::from_secs(3600), + short_calls: 0, + })); + let first = AwsAuthConfig { + access_key_id: Some("first".into()), + ..Default::default() + }; + let second = AwsAuthConfig { + access_key_id: Some("second".into()), + ..Default::default() + }; + let (a, b, c) = tokio::join!( + auth.resolve_credentials(first.clone(), &no_env), + auth.resolve_credentials(first, &no_env), + auth.resolve_credentials(second, &no_env) ); + let a = a.unwrap(); + let b = b.unwrap(); + let c = c.unwrap(); + assert_eq!(a.session_token(), b.session_token()); + assert_ne!(a.access_key_id(), c.access_key_id()); + assert_eq!(loads.load(Ordering::SeqCst), 2); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[rstest::rstest] + fn auth_config_debug_never_discloses_credentials() { + let config = AwsAuthConfig { + access_key_id: Some("credential-value".into()), + secret_access_key: Some("credential-value".into()), + session_token: Some("credential-value".into()), + web_identity_token: Some("credential-value".into()), + external_id: Some("credential-value".into()), + ..Default::default() + }; + let output = format!("{config:?}"); + assert!(!output.contains("credential-value")); + assert!(output.contains("[REDACTED]")); + } + + #[rstest::rstest] + #[case::session(AwsAuthFlow::SessionToken { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), session_token: "credential-value".into() })] + #[case::static_keys(AwsAuthFlow::StaticKeys { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), region_name: "region".into() })] + #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into() })] + fn auth_flow_debug_never_discloses_credentials(#[case] flow: AwsAuthFlow) { + let output = format!("{flow:?}"); + assert!(!output.contains("credential-value")); + assert!(output.contains("[REDACTED]")); } #[test] @@ -1142,8 +1589,8 @@ mod tests { let credentials = AwsAuthService::default() .resolve_credentials( AwsAuthConfig { - access_key_id: Some(access_key_id), - secret_access_key: Some(secret_access_key), + access_key_id: Some(access_key_id.into()), + secret_access_key: Some(secret_access_key.into()), region_name: Some("us-west-2".to_string()), ..Default::default() }, @@ -1191,4 +1638,44 @@ mod tests { failures.join("; ") ); } + + #[rstest::rstest] + #[case::provisioned( + None, + "https://cache-1/?Action=connect&User=iam-user", + "cache-1/?Action=connect&User=iam-user&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIDEXAMPLE%2F20261029%2Fus-east-1%2Felasticache%2Faws4_request&X-Amz-Date=20261029T120000Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=dfa58492a6ae9ba065088724170a52437329d03f3e9bc9a4c90a80cc8a18e2f6" + )] + #[case::serverless_with_a_session_token( + Some("session-token"), + "https://cache-1/?Action=connect&User=iam-user&ResourceType=ServerlessCache", + "cache-1/?Action=connect&User=iam-user&ResourceType=ServerlessCache&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIDEXAMPLE%2F20261029%2Fus-east-1%2Felasticache%2Faws4_request&X-Amz-Date=20261029T120000Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Security-Token=session-token&X-Amz-Signature=e48bdbb3bf9d5e8f1d325bf33d5260e333df1f6b8868e8edd1e4c3cc01b4c3f9" + )] + fn presigned_urls_match_botocore_sigv4_query_auth( + #[case] session_token: Option<&str>, + #[case] url: &str, + #[case] expected: &str, + ) { + // Expected values from botocore 1.43.117 SigV4QueryAuth with get_current_datetime patched to + // 2026-10-29T12:00:00Z (generated 2026-10-29), the signer Python's ElastiCache IAM provider uses + let credentials = Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + session_token.map(str::to_owned), + None, + "test", + ); + let signed = presign_url( + url, + Duration::from_secs(900), + SigningScope { + region: "us-east-1", + service: "elasticache", + credentials: &credentials, + time: UNIX_EPOCH + Duration::from_secs(1_793_275_200), + }, + ) + .unwrap(); + + assert_eq!(signed.strip_prefix("https://"), Some(expected)); + } } diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 789fde156bd..0e4662eda9e 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -11,8 +11,8 @@ pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0]; pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0]; pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0]; pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0]; -pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; -pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; +pub const AWS_ROLE_ARN: &str = AwsParams::ROLE_ARN_ENV; +pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV; pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0]; pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0]; pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; diff --git a/litellm-rust/crates/auth-aws/src/error.rs b/litellm-rust/crates/auth-aws/src/error.rs index 5f88fc8a171..5445de1e6c3 100644 --- a/litellm-rust/crates/auth-aws/src/error.rs +++ b/litellm-rust/crates/auth-aws/src/error.rs @@ -2,6 +2,8 @@ use litellm_error::Source; #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { + #[error("AWS credential provider returned expired credentials")] + ExpiredCredentials, #[error("could not load AWS {flow} credentials")] Credentials { flow: CredentialFlow, diff --git a/litellm-rust/crates/auth-aws/src/signer.rs b/litellm-rust/crates/auth-aws/src/signer.rs index dff917d7c13..2d9ad9ba123 100644 --- a/litellm-rust/crates/auth-aws/src/signer.rs +++ b/litellm-rust/crates/auth-aws/src/signer.rs @@ -75,7 +75,8 @@ impl RequestSigner for SigV4Signer { ) .map_err(|error| match error { Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error), - Error::Credentials { .. } + Error::ExpiredCredentials + | Error::Credentials { .. } | Error::MissingWebIdentityCredentials | Error::WebIdentityExpiration(_) | Error::Signing { .. } => litellm_http::Error::signature(error), diff --git a/litellm-rust/crates/auth-azure/Cargo.toml b/litellm-rust/crates/auth-azure/Cargo.toml index 7dcb0e513a7..36c10a74c80 100644 --- a/litellm-rust/crates/auth-azure/Cargo.toml +++ b/litellm-rust/crates/auth-azure/Cargo.toml @@ -8,17 +8,19 @@ repository.workspace = true [dependencies] litellm-secret-value.workspace = true litellm-auth-types.workspace = true +litellm-error.workspace = true moka.workspace = true serde_json.workspace = true sha2.workspace = true strum.workspace = true +tokio.workspace = true url.workspace = true azure_core = "1.0.0" azure_identity = { version = "1.0.0", features = ["tokio"] } [dev-dependencies] -litellm-error.workspace = true +litellm-secrets.workspace = true rstest.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/litellm-rust/crates/auth-azure/src/lib.rs b/litellm-rust/crates/auth-azure/src/lib.rs index 9ed505e4160..6319406ce9c 100644 --- a/litellm-rust/crates/auth-azure/src/lib.rs +++ b/litellm-rust/crates/auth-azure/src/lib.rs @@ -3,5 +3,5 @@ mod native; mod resolve; mod types; -pub use resolve::{AzureAuthService, SECRET_NAMES}; +pub use resolve::{AzureAuthService, AzureIdentity, SECRET_NAMES}; pub use types::{AzureAuthInputs, ConfigValue}; diff --git a/litellm-rust/crates/auth-azure/src/native.rs b/litellm-rust/crates/auth-azure/src/native.rs index f40745ab7a2..03021428ea7 100644 --- a/litellm-rust/crates/auth-azure/src/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -59,6 +59,7 @@ pub(crate) enum NativeAzureRequest { pub(crate) struct ValidatedAzureRequest { request: NativeAzureRequest, credential_source: InputSource, + probe_imds: bool, } impl ValidatedAzureRequest { @@ -68,9 +69,18 @@ impl ValidatedAzureRequest { Ok(Self { request, credential_source, + probe_imds: false, }) } + pub(crate) fn with_imds_probe(self, enabled: bool) -> Self { + Self { + probe_imds: enabled + && matches!(self.request, NativeAzureRequest::ManagedIdentity { .. }), + ..self + } + } + pub(crate) fn credential_source(&self) -> InputSource { self.credential_source } @@ -87,9 +97,34 @@ impl ValidatedAzureRequest { } } +pub(crate) enum CredentialAttempt { + Resolved(ResolvedCredential), + Unavailable(Error), + Failed(Error), +} + +impl CredentialAttempt { + pub(crate) fn into_result(self) -> Result { + match self { + Self::Resolved(credential) => Ok(credential), + Self::Unavailable(error) | Self::Failed(error) => Err(error), + } + } +} + pub(crate) struct NativeAzureTokenAcquirer { cache: AzureCredentialProviderCache, transport: Option, + imds_available: tokio::sync::Mutex, +} + +enum ImdsAvailability { + Unknown, + Available, + Unavailable { + retry_at: tokio::time::Instant, + error: Error, + }, } impl Default for NativeAzureTokenAcquirer { @@ -103,6 +138,7 @@ impl NativeAzureTokenAcquirer { Self { cache: AzureCredentialProviderCache::new(cache_capacity), transport: None, + imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown), } } @@ -114,10 +150,51 @@ impl NativeAzureTokenAcquirer { Self { cache: AzureCredentialProviderCache::new(cache_capacity), transport: Some(transport), + imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown), } } - pub(crate) async fn acquire( + pub(crate) async fn acquire(&self, request: ValidatedAzureRequest) -> CredentialAttempt { + let discover_imds = request.probe_imds; + if discover_imds && let Err(error) = self.discover_imds().await { + return CredentialAttempt::Unavailable(error); + } + match self.acquire_token(request).await { + Ok(credential) => CredentialAttempt::Resolved(credential), + Err(error) => { + let missing_identity = discover_imds && unassigned_identity(&error); + if missing_identity { + CredentialAttempt::Unavailable(error) + } else { + CredentialAttempt::Failed(error) + } + } + } + } + + async fn discover_imds(&self) -> Result<(), Error> { + let mut availability = self.imds_available.lock().await; + match &*availability { + ImdsAvailability::Available => return Ok(()), + ImdsAvailability::Unavailable { retry_at, error } + if tokio::time::Instant::now() < *retry_at => + { + return Err(error.clone()); + } + ImdsAvailability::Unknown | ImdsAvailability::Unavailable { .. } => {} + } + let result = probe_imds(self.transport.clone().unwrap_or_default()).await; + *availability = match &result { + Ok(()) => ImdsAvailability::Available, + Err(error) => ImdsAvailability::Unavailable { + retry_at: tokio::time::Instant::now() + Duration::from_secs(30), + error: error.clone(), + }, + }; + result + } + + async fn acquire_token( &self, request: ValidatedAzureRequest, ) -> Result { @@ -148,6 +225,54 @@ impl NativeAzureTokenAcquirer { } } +fn unassigned_identity(error: &(dyn std::error::Error + 'static)) -> bool { + std::iter::successors(Some(error), |cause| cause.source()).any(|cause| { + cause + .downcast_ref::() + .or_else(|| { + cause + .downcast_ref::()? + .downcast_ref::() + }) + .is_some_and(|error| { + error.http_status() == Some(azure_core::http::StatusCode::BadRequest) + }) + }) +} + +async fn probe_imds(transport: azure_core::http::Transport) -> Result<(), Error> { + use azure_core::http::{Context, Method, Request, Url}; + + let endpoint = + Url::parse("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2019-08-01") + .map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed("Azure IMDS probe endpoint", error)) + })?; + let mut request = Request::new(endpoint, Method::Get); + let response = tokio::time::timeout( + Duration::from_secs(1), + transport.send(&Context::new(), &mut request), + ) + .await + .map_err(|error| { + Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error)) + })? + .map_err(|error| { + Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error)) + })?; + // Headerless IMDS discovery expects 400, checked 2026-11-01 + // https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service#errors-and-debugging + if response.status() != azure_core::http::StatusCode::BadRequest { + return Err(Error::CredentialAcquisition( + ErrorDetail::UnexpectedStatus { + operation: "Azure IMDS availability probe", + status: u16::from(response.status()), + }, + )); + } + Ok(()) +} + impl NativeAzureRequest { fn scope(&self) -> &str { match self { @@ -493,6 +618,7 @@ impl ClientAssertion for StaticAssertion { #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; + use std::time::Duration; use azure_core::http::headers::Headers; use azure_core::http::{AsyncRawResponse, HttpClient, Request, StatusCode, Transport}; @@ -569,6 +695,296 @@ mod tests { } } + #[derive(Debug)] + struct AvailableImdsClient { + probe_status: StatusCode, + probe_delay: Duration, + recover_after: Option, + response_status: Option, + probes: std::sync::atomic::AtomicUsize, + } + + impl Default for AvailableImdsClient { + fn default() -> Self { + Self { + probe_status: StatusCode::BadRequest, + probe_delay: Duration::ZERO, + recover_after: None, + response_status: None, + probes: std::sync::atomic::AtomicUsize::new(0), + } + } + } + + impl HttpClient for AvailableImdsClient { + fn execute_request<'life0, 'life1, 'async_trait>( + &'life0 self, + request: &'life1 Request, + ) -> std::pin::Pin< + Box> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + if request + .headers() + .get_optional_str(&azure_core::http::headers::HeaderName::from_static( + "metadata", + )) + .is_none() + { + let probe = self + .probes + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.recover_after.is_some_and(|failures| probe >= failures) { + return Ok(AsyncRawResponse::from_bytes( + StatusCode::BadRequest, + Headers::new(), + "metadata header required", + )); + } + tokio::time::sleep(self.probe_delay).await; + return Ok(AsyncRawResponse::from_bytes( + self.probe_status, + Headers::new(), + "metadata header required", + )); + } + if let Some(status) = self.response_status { + return Ok(AsyncRawResponse::from_bytes( + status, + Headers::new(), + "identity endpoint failure", + )); + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let expires = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600; + Ok(AsyncRawResponse::from_bytes( + StatusCode::Ok, + Headers::new(), + serde_json::json!({ + "access_token": "managed-token", "expires_on": expires.to_string(), + "token_type": "Bearer", "resource": "https://service.test", + }) + .to_string(), + )) + }) + } + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn available_imds_retains_full_sdk_token_acquisition() { + let acquirer = NativeAzureTokenAcquirer::with_transport( + 4, + Transport::new(Arc::new(AvailableImdsClient { + probe_status: StatusCode::BadRequest, + response_status: None, + ..Default::default() + })), + ); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + let credential = acquirer.acquire(request).await.into_result().unwrap(); + assert_eq!(credential.secret().expose(), "managed-token"); + } + + #[rstest::rstest] + #[case::another_metadata_service(StatusCode::NotFound)] + #[case::token_required(StatusCode::Unauthorized)] + #[case::proxy_rejection(StatusCode::Forbidden)] + #[case::unavailable(StatusCode::ServiceUnavailable)] + #[tokio::test(start_paused = true)] + async fn unrelated_metadata_responses_do_not_start_managed_identity_acquisition( + #[case] status: StatusCode, + ) { + let client = Arc::new(AvailableImdsClient { + probe_status: status, + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + assert!(matches!( + acquirer.acquire(request.clone()).await, + super::CredentialAttempt::Unavailable(_) + )); + assert!(matches!( + acquirer.acquire(request).await, + super::CredentialAttempt::Unavailable(_) + )); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn an_imds_timeout_is_not_repeated_on_the_next_request() { + let client = Arc::new(AvailableImdsClient { + probe_delay: Duration::from_secs(10), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + assert!(matches!( + acquirer.acquire(request.clone()).await, + super::CredentialAttempt::Unavailable(_) + )); + let second = tokio::time::timeout(Duration::from_millis(10), acquirer.acquire(request)) + .await + .unwrap(); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[rstest::fixture] + fn imds_request() -> ValidatedAzureRequest { + ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true) + } + + #[rstest::rstest] + #[case::timeout(StatusCode::BadRequest, Duration::from_secs(10))] + #[case::service_unavailable(StatusCode::ServiceUnavailable, Duration::ZERO)] + #[tokio::test(start_paused = true)] + async fn failed_imds_discovery_recovers_without_restarting_the_acquirer( + imds_request: ValidatedAzureRequest, + #[case] status: StatusCode, + #[case] delay: Duration, + ) { + let client = Arc::new(AvailableImdsClient { + probe_status: status, + probe_delay: delay, + recover_after: Some(1), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request.clone()), + ); + assert!(matches!(first, super::CredentialAttempt::Unavailable(_))); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + + tokio::time::advance(Duration::from_secs(30)).await; + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request.clone()), + ); + assert_eq!( + first.into_result().unwrap().secret().expose(), + "managed-token" + ); + assert_eq!( + second.into_result().unwrap().secret().expose(), + "managed-token" + ); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + + tokio::time::advance(Duration::from_secs(60)).await; + assert_eq!( + acquirer + .acquire(imds_request) + .await + .into_result() + .unwrap() + .secret() + .expose(), + "managed-token", + ); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn a_persistent_imds_outage_reuses_each_failed_probe( + imds_request: ValidatedAzureRequest, + ) { + let client = Arc::new(AvailableImdsClient { + probe_delay: Duration::from_secs(10), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + assert!(matches!( + acquirer.acquire(imds_request.clone()).await, + super::CredentialAttempt::Unavailable(_), + )); + tokio::time::advance(Duration::from_secs(30)).await; + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request), + ); + assert!(matches!(first, super::CredentialAttempt::Unavailable(_))); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[case::default_identity_not_assigned(true, StatusCode::BadRequest, true)] + #[case::explicit_identity_not_assigned(false, StatusCode::BadRequest, false)] + #[case::default_identity_forbidden(true, StatusCode::Forbidden, false)] + #[case::default_identity_service_failure(true, StatusCode::InternalServerError, false)] + #[tokio::test(start_paused = true)] + async fn only_unavailable_default_identities_allow_fallback( + #[case] discovery: bool, + #[case] status: StatusCode, + #[case] unavailable: bool, + ) { + let acquirer = NativeAzureTokenAcquirer::with_transport( + 4, + Transport::new(Arc::new(AvailableImdsClient { + probe_status: StatusCode::BadRequest, + response_status: Some(status), + ..Default::default() + })), + ); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(discovery); + let outcome = acquirer.acquire(request).await; + let is_unavailable = matches!(outcome, super::CredentialAttempt::Unavailable(_)); + let error = outcome.into_result().unwrap_err(); + assert_eq!( + is_unavailable, + unavailable, + "{}", + litellm_error::Report::new(&error) + ); + } + + #[rstest::rstest] #[tokio::test] async fn client_secret_uses_sdk_protocol_and_reuses_cached_credential() { let transport = Arc::new(RecordingTokenClient::default()); @@ -582,8 +998,12 @@ mod tests { "https://login.test", ); - let first = acquirer.acquire(request.clone()).await.unwrap(); - let second = acquirer.acquire(request).await.unwrap(); + let first = acquirer + .acquire(request.clone()) + .await + .into_result() + .unwrap(); + let second = acquirer.acquire(request).await.into_result().unwrap(); assert_eq!(first.secret().expose(), "native-token"); assert_eq!(second.secret().expose(), "native-token"); @@ -599,6 +1019,7 @@ mod tests { ); } + #[rstest::rstest] #[tokio::test] async fn credential_provider_cache_isolates_every_client_secret_identity_field() { let transport = Arc::new(RecordingTokenClient::default()); @@ -645,10 +1066,10 @@ mod tests { ), ]; - acquirer.acquire(base.clone()).await.unwrap(); - acquirer.acquire(base).await.unwrap(); + acquirer.acquire(base.clone()).await.into_result().unwrap(); + acquirer.acquire(base).await.into_result().unwrap(); for request in variants.into_iter().skip(1) { - acquirer.acquire(request).await.unwrap(); + acquirer.acquire(request).await.into_result().unwrap(); } assert_eq!(transport.requests.lock().unwrap().len(), 6); diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index a92630af26a..9ff4eda540a 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -1,3 +1,4 @@ +use crate::native::CredentialAttempt; use litellm_auth_types::{ CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, Sourced, TokenProviderHandle, @@ -12,25 +13,31 @@ use std::sync::Arc; use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; use super::types::{AzureAuthInputs, AzureCredentialType, ConfigValue, DEFAULT_AZURE_SCOPE}; -const AZURE_AD_TOKEN_ENV: &str = "AZURE_AD_TOKEN"; -const AZURE_TENANT_ID_ENV: &str = "AZURE_TENANT_ID"; -const AZURE_CLIENT_ID_ENV: &str = "AZURE_CLIENT_ID"; -const AZURE_CLIENT_SECRET_ENV: &str = "AZURE_CLIENT_SECRET"; -const AZURE_SCOPE_ENV: &str = "AZURE_SCOPE"; -const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST"; -const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL"; -const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; +#[derive(strum::IntoStaticStr, strum::VariantArray, strum::VariantNames)] +enum AzureSecretName { + #[strum(serialize = "AZURE_AD_TOKEN")] + AdToken, + #[strum(serialize = "AZURE_TENANT_ID")] + TenantId, + #[strum(serialize = "AZURE_CLIENT_ID")] + ClientId, + #[strum(serialize = "AZURE_CLIENT_SECRET")] + ClientSecret, + #[strum(serialize = "AZURE_SCOPE")] + Scope, + #[strum(serialize = "AZURE_AUTHORITY_HOST")] + AuthorityHost, + #[strum(serialize = "AZURE_CREDENTIAL")] + Credential, + #[strum(serialize = "AZURE_FEDERATED_TOKEN_FILE")] + FederatedTokenFile, + #[strum(serialize = "IDENTITY_ENDPOINT")] + IdentityEndpoint, + #[strum(serialize = "MSI_ENDPOINT")] + MsiEndpoint, +} -pub const SECRET_NAMES: &[&str] = &[ - AZURE_AD_TOKEN_ENV, - AZURE_TENANT_ID_ENV, - AZURE_CLIENT_ID_ENV, - AZURE_CLIENT_SECRET_ENV, - AZURE_SCOPE_ENV, - AZURE_AUTHORITY_HOST_ENV, - AZURE_CREDENTIAL_ENV, - AZURE_FEDERATED_TOKEN_FILE_ENV, -]; +pub const SECRET_NAMES: &[&str] = ::VARIANTS; #[derive(Clone, Debug)] pub(crate) enum AzureCredentialPlan { @@ -57,14 +64,14 @@ trait AzureTokenAcquirer: Send + Sync { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>>; + ) -> Pin + Send + '_>>; } impl AzureTokenAcquirer for NativeAzureTokenAcquirer { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>> { + ) -> Pin + Send + '_>> { Box::pin(NativeAzureTokenAcquirer::acquire(self, request)) } } @@ -83,12 +90,36 @@ impl AzureAuthService { Self { native } } + pub async fn identity_token( + &self, + identity: &AzureIdentity, + scope: &str, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result, Error> { + let plan = identity_plan(identity, scope, env_lookup)?; + let inputs = AzureAuthInputs::default_credential_for_scope(scope); + Ok(self + .run_plan(plan, &inputs, env_lookup) + .await? + .map(Sourced::into_value)) + } + pub async fn get_azure_ad_token( &self, inputs: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result>, Error> { - match select_auth_plan(inputs, env_lookup)? { + self.run_plan(select_auth_plan(inputs, env_lookup)?, inputs, env_lookup) + .await + } + + async fn run_plan( + &self, + plan: AzureCredentialPlan, + inputs: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result>, Error> { + match plan { AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)), AzureCredentialPlan::Caller(caller) => { let credential = caller.acquire().await?; @@ -123,6 +154,7 @@ impl AzureAuthService { self.native .acquire(request) .await + .into_result() .map(|credential| Sourced::new(credential, source)) .map(Some) } @@ -131,6 +163,7 @@ impl AzureAuthService { self.native .acquire(request) .await + .into_result() .map(|credential| Some(Sourced::new(credential, source))) } AzureCredentialPlan::Chain(requests) => { @@ -138,8 +171,16 @@ impl AzureAuthService { for request in requests { let source = request.credential_source(); match self.native.acquire(request).await { - Ok(credential) => return Ok(Some(Sourced::new(credential, source))), - Err(error) => failures.push(error), + CredentialAttempt::Resolved(credential) => { + return Ok(Some(Sourced::new(credential, source))); + } + CredentialAttempt::Unavailable(error) => failures.push(error), + CredentialAttempt::Failed(error) => { + return Err(Error::CredentialChain { + last: Box::new(error), + earlier: failures, + }); + } } } match failures.pop() { @@ -155,39 +196,161 @@ impl AzureAuthService { } } +/// The identity azure-identity builds for one explicit credential class, without LiteLLM's provider settings +/// (`AZURE_AD_TOKEN`, `AZURE_CREDENTIAL`, `AZURE_SCOPE`) that `get_azure_ad_token` also reads. +#[derive(Clone, Debug)] +pub enum AzureIdentity { + ClientSecret { + tenant_id: String, + client_id: String, + client_secret: SecretValue, + }, + ManagedIdentity { + client_id: String, + }, + Default, +} + +fn identity_plan( + identity: &AzureIdentity, + scope: &str, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let deployment = |value: &str| Sourced::new(value.to_owned(), InputSource::Deployment); + let environment = |name: &str| { + env_lookup(name) + .filter(|value| !value.trim().is_empty()) + .map(|value| Sourced::new(value, InputSource::Environment)) + }; + let scope = deployment(scope); + let authority = environment(AzureSecretName::AuthorityHost.into()); + match identity { + AzureIdentity::ClientSecret { + tenant_id, + client_id, + client_secret, + } => Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( + NativeAzureRequest::ClientSecret { + tenant_id: deployment(tenant_id), + client_id: deployment(client_id), + client_secret: Sourced::new(client_secret.clone(), InputSource::Deployment), + scope, + authority, + }, + )?)), + AzureIdentity::ManagedIdentity { client_id } => Ok(AzureCredentialPlan::Native( + ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: Some(deployment(client_id)), + scope, + selection_source: InputSource::Deployment, + })?, + )), + AzureIdentity::Default => { + let tenant_id = environment(AzureSecretName::TenantId.into()); + let client_id = environment(AzureSecretName::ClientId.into()); + let from_environment = match ( + tenant_id.clone(), + client_id.clone(), + environment(AzureSecretName::ClientSecret.into()), + ) { + (Some(tenant_id), Some(client_id), Some(client_secret)) => { + Some(NativeAzureRequest::ClientSecret { + tenant_id, + client_id, + client_secret: client_secret.map(SecretValue::new), + scope: scope.clone(), + authority: authority.clone(), + }) + } + _ => None, + }; + let probe_imds = env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none() + && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none(); + let AzureCredentialPlan::Chain(chain) = select_native_plan( + Some(Sourced::new( + AzureCredentialType::DefaultAzureCredential, + InputSource::Deployment, + )), + tenant_id, + client_id, + environment(AzureSecretName::FederatedTokenFile.into()), + scope, + authority, + InputSource::Environment, + )? + else { + unreachable!("DefaultAzureCredential always plans a chain") + }; + let environment_request = from_environment + .map(ValidatedAzureRequest::new) + .transpose()?; + Ok(AzureCredentialPlan::Chain( + environment_request + .into_iter() + .chain(chain) + .map(|request| request.with_imds_probe(probe_imds)) + .collect(), + )) + } + } +} + pub(crate) fn select_auth_plan( inputs: &AzureAuthInputs, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup); - let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup); - let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup); - let client_secret = - configured_secret(&inputs.client_secret, AZURE_CLIENT_SECRET_ENV, env_lookup); - let scope = configured_string(&inputs.azure_scope, AZURE_SCOPE_ENV, env_lookup) - .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment)); - let authority = configured_string( - &inputs.azure_authority_host, - AZURE_AUTHORITY_HOST_ENV, + let token = configured_secret( + &inputs.azure_ad_token, + AzureSecretName::AdToken.into(), env_lookup, ); - let selector = configured_string(&inputs.azure_credential, AZURE_CREDENTIAL_ENV, env_lookup) - .map(|value| { - value - .value() - .parse::() - .map(|selector| Sourced::new(selector, value.source())) - }) - .transpose() - .map_err(|error| { - Error::InvalidConfiguration(ErrorDetail::failed( - "Azure credential selector parsing", - error, - )) - })?; + let tenant_id = configured_string( + &inputs.tenant_id, + AzureSecretName::TenantId.into(), + env_lookup, + ); + let client_id = configured_string( + &inputs.client_id, + AzureSecretName::ClientId.into(), + env_lookup, + ); + let client_secret = configured_secret( + &inputs.client_secret, + AzureSecretName::ClientSecret.into(), + env_lookup, + ); + let scope = configured_string( + &inputs.azure_scope, + AzureSecretName::Scope.into(), + env_lookup, + ) + .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment)); + let authority = configured_string( + &inputs.azure_authority_host, + AzureSecretName::AuthorityHost.into(), + env_lookup, + ); + let selector = configured_string( + &inputs.azure_credential, + AzureSecretName::Credential.into(), + env_lookup, + ) + .map(|value| { + value + .value() + .parse::() + .map(|selector| Sourced::new(selector, value.source())) + }) + .transpose() + .map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed( + "Azure credential selector parsing", + error, + )) + })?; let federated_token_file = configured_string( &inputs.federated_token_file, - AZURE_FEDERATED_TOKEN_FILE_ENV, + AzureSecretName::FederatedTokenFile.into(), env_lookup, ); @@ -237,7 +400,12 @@ pub(crate) fn select_auth_plan( return Ok(AzureCredentialPlan::Missing); } - select_native_plan( + let probe_imds = selector + .as_ref() + .is_none_or(|selector| *selector.value() == AzureCredentialType::DefaultAzureCredential) + && env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none() + && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none(); + match select_native_plan( selector, tenant_id, client_id, @@ -245,7 +413,15 @@ pub(crate) fn select_auth_plan( scope, authority, inputs.enable_azure_ad_token_refresh.source(), - ) + )? { + AzureCredentialPlan::Chain(requests) => Ok(AzureCredentialPlan::Chain( + requests + .into_iter() + .map(|request| request.with_imds_probe(probe_imds)) + .collect(), + )), + plan => Ok(plan), + } } fn select_native_plan( @@ -509,7 +685,10 @@ mod tests { #[derive(Debug)] struct FileResolver; + use crate::native::CredentialAttempt; + struct ChainAcquirer { + terminal_on: Option<&'static str>, requests: Mutex>, succeed_on: Option<&'static str>, } @@ -518,20 +697,25 @@ mod tests { fn acquire( &self, request: ValidatedAzureRequest, - ) -> std::pin::Pin> + Send + '_>> - { + ) -> std::pin::Pin + Send + '_>> { let kind = request.kind(); self.requests.lock().unwrap().push(kind); Box::pin(async move { if self.succeed_on == Some(kind) { - Ok(ResolvedCredential::AccessToken { + CredentialAttempt::Resolved(ResolvedCredential::AccessToken { token: SecretValue::new("chain-token"), expires_on: None, }) } else { - Err(Error::CredentialAcquisition( - litellm_auth_types::ErrorDetail::Unresolved { subject: kind }, - )) + let error = + Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::Unresolved { + subject: kind, + }); + if self.terminal_on == Some(kind) { + CredentialAttempt::Failed(error) + } else { + CredentialAttempt::Unavailable(error) + } } }) } @@ -690,9 +874,210 @@ mod tests { assert_eq!(resolved, Some(SecretValue::new("rotated-assertion"))); } + #[derive(Debug)] + struct UnavailableImdsClient; + + impl azure_core::http::HttpClient for UnavailableImdsClient { + fn execute_request<'life0, 'life1, 'async_trait>( + &'life0 self, + _request: &'life1 azure_core::http::Request, + ) -> std::pin::Pin< + Box< + dyn Future> + + Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(std::future::pending()) + } + } + + struct OfflineChainAcquirer(super::NativeAzureTokenAcquirer); + + impl AzureTokenAcquirer for OfflineChainAcquirer { + fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(async move { + if request.kind() == "developer-tools" { + return CredentialAttempt::Resolved(ResolvedCredential::AccessToken { + token: SecretValue::new("cli-token"), + expires_on: None, + }); + } + self.0.acquire(request).await + }) + } + } + + #[rstest::rstest] + #[case::default_off_azure("DefaultAzureCredential", None, true)] + #[case::explicit_managed_identity("ManagedIdentityCredential", None, false)] + #[case::app_service("DefaultAzureCredential", Some("http://app-service.test"), false)] + #[tokio::test(start_paused = true)] + async fn only_default_imds_discovery_is_bounded( + #[case] selector: &str, + #[case] identity_endpoint: Option<&str>, + #[case] expect_cli: bool, + ) { + let native = super::NativeAzureTokenAcquirer::with_transport( + 4, + azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)), + ); + let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native))); + let inputs = AzureAuthInputs { + azure_credential: crate::types::ConfigValue::Value(Sourced::new( + selector.into(), + InputSource::Deployment, + )), + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + let environment = |name: &str| { + (name == "IDENTITY_ENDPOINT") + .then_some(identity_endpoint) + .flatten() + .map(str::to_string) + }; + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(2), + service.get_azure_ad_token(&inputs, &environment), + ) + .await; + + assert_eq!(outcome.is_ok(), expect_cli); + if expect_cli { + assert_eq!( + outcome.unwrap().unwrap().unwrap().value().secret().expose(), + "cli-token" + ); + } + } + + #[rstest::rstest] + #[case::app_service(Some("IDENTITY_ENDPOINT"), false)] + #[case::legacy_app_service(Some("MSI_ENDPOINT"), false)] + #[case::off_azure(None, true)] + #[tokio::test(start_paused = true)] + async fn default_chain_uses_endpoints_from_the_secret_snapshot( + #[case] endpoint: Option<&str>, + #[case] expect_cli: bool, + #[values(false, true)] consumer_identity: bool, + ) { + use litellm_secrets::source::SecretSource; + + struct EndpointSecrets<'a>(Option<&'a str>); + + impl SecretSource for EndpointSecrets<'_> { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> std::pin::Pin< + Box< + dyn Future, litellm_secrets::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { + Ok((self.0 == Some(name)).then(|| SecretValue::new("http://app-service.test"))) + }) + } + } + + let snapshot = EndpointSecrets(endpoint) + .resolve(SECRET_NAMES) + .await + .unwrap(); + let native = super::NativeAzureTokenAcquirer::with_transport( + 4, + azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)), + ); + let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native))); + let lookup = |name: &str| snapshot.get(name); + let scope = "https://service.test/.default"; + let outcome = tokio::time::timeout(std::time::Duration::from_secs(2), async { + if consumer_identity { + service + .identity_token(&super::AzureIdentity::Default, scope, &lookup) + .await + } else { + service + .get_azure_ad_token( + &AzureAuthInputs::default_credential_for_scope(scope), + &lookup, + ) + .await + .map(|token| token.map(Sourced::into_value)) + } + }) + .await; + + assert_eq!(outcome.is_ok(), expect_cli); + if expect_cli { + assert_eq!( + outcome.unwrap().unwrap().unwrap().secret().expose(), + "cli-token" + ); + } + } + + #[rstest::rstest] + #[case::litellm_provider_settings_are_ignored( + &[("AZURE_AD_TOKEN", "provider-token"), ("AZURE_CREDENTIAL", "ClientSecretCredential")], + &["managed-identity", "developer-tools"] + )] + #[case::environment_client_secret_comes_first( + &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_CLIENT_SECRET", "secret")], + &["client-secret", "managed-identity", "developer-tools"] + )] + #[case::aks_workload_identity( + &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token")], + &["workload-identity", "managed-identity", "developer-tools"] + )] + #[tokio::test] + async fn the_default_identity_follows_azure_identity_defaults( + #[case] environment: &[(&str, &str)], + #[case] expected: &[&str], + ) { + let acquirer = Arc::new(ChainAcquirer { + terminal_on: None, + requests: Mutex::new(Vec::new()), + succeed_on: Some("developer-tools"), + }); + let service = AzureAuthService::with_acquirer(acquirer.clone()); + let lookup = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; + + let credential = service + .identity_token( + &super::AzureIdentity::Default, + "https://redis.azure.com/.default", + &lookup, + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(credential.secret().expose(), "chain-token"); + assert_eq!(*acquirer.requests.lock().unwrap(), expected); + } + + #[rstest::rstest] #[tokio::test] async fn default_chain_uses_declared_order_and_stops_after_success() { let acquirer = Arc::new(ChainAcquirer { + terminal_on: None, requests: Mutex::new(Vec::new()), succeed_on: Some("developer-tools"), }); @@ -715,9 +1100,13 @@ mod tests { ); } + #[rstest::rstest] + #[case::all_unavailable(false)] + #[case::terminal_failure_after_unavailable(true)] #[tokio::test] - async fn chain_reports_each_acquisition_failure() { + async fn chain_reports_each_acquisition_failure(#[case] terminal: bool) { let acquirer = Arc::new(ChainAcquirer { + terminal_on: terminal.then_some("developer-tools"), requests: Mutex::new(Vec::new()), succeed_on: None, }); @@ -744,6 +1133,27 @@ mod tests { ); } + #[rstest::rstest] + #[tokio::test] + async fn a_configured_identity_failure_does_not_fall_back_to_a_developer_identity() { + let acquirer = Arc::new(ChainAcquirer { + terminal_on: Some("managed-identity"), + requests: Mutex::new(Vec::new()), + succeed_on: Some("developer-tools"), + }); + let service = AzureAuthService::with_acquirer(acquirer.clone()); + let inputs = AzureAuthInputs { + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + let error = service + .get_azure_ad_token(&inputs, &|_| None) + .await + .unwrap_err(); + assert_eq!(*acquirer.requests.lock().unwrap(), ["managed-identity"]); + assert!(matches!(error, Error::CredentialChain { earlier, .. } if earlier.is_empty())); + } + #[derive(Debug)] struct CallerToken(&'static str); diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index ad60f0db817..90aab8d281c 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true repository.workspace = true [features] -google-sdk = ["dep:google-cloud-auth", "dep:http"] +google-sdk = ["dep:http"] [dependencies] litellm-secret-value.workspace = true @@ -21,10 +21,14 @@ tokio.workspace = true veil.workspace = true gcp_auth = "0.12.7" -google-cloud-auth = { workspace = true, optional = true } +google-cloud-auth.workspace = true http = { workspace = true, optional = true } [dev-dependencies] +time = { workspace = true, features = ["formatting"] } +axum = { workspace = true, features = ["json"] } litellm-error.workspace = true rstest.workspace = true +rustls.workspace = true tempfile.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/litellm-rust/crates/auth-gcp/src/auth.rs b/litellm-rust/crates/auth-gcp/src/auth.rs index 650d5831796..5976ef82344 100644 --- a/litellm-rust/crates/auth-gcp/src/auth.rs +++ b/litellm-rust/crates/auth-gcp/src/auth.rs @@ -1,20 +1,11 @@ -use std::{future::Future, pin::Pin, sync::Arc}; +use std::sync::Arc; -use gcp_auth::{CustomServiceAccount, TokenProvider}; -use litellm_auth_types::{CredentialPlacement, Error, ErrorDetail, http::apply_credential}; -use moka::future::Cache; -use serde_json::Value; -use sha2::{Digest, Sha256}; +use litellm_auth_types::{CredentialPlacement, Error, http::apply_credential}; use crate::{ - config::{ - CredentialSource, VertexConfig, credential_source, get_vertex_ai_project, - is_base64_encoded_json, names_a_credentials_file, non_empty_env, - }, - constants::{ - CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT, VERTEX_AI_API_KEY_ENV, - VERTEXAI_API_KEY_ENV, - }, + config::{VertexConfig, credential_source, get_vertex_ai_project, non_empty_env}, + constants::{VERTEX_AI_API_KEY_ENV, VERTEXAI_API_KEY_ENV}, + identity::{GcpAuth, GcpProviderLoader, GcpTokenSource}, }; pub struct VertexEnvironment { @@ -22,28 +13,15 @@ pub struct VertexEnvironment { pub project_id: String, } -struct VertexAccessToken { - token: String, - project_id: String, -} - -#[derive(Clone)] +#[derive(Clone, Default)] pub struct VertexAuth { - providers: Cache>, - loader: Arc, -} - -impl Default for VertexAuth { - fn default() -> Self { - Self::new(Arc::new(GcpProviderLoader)) - } + identity: GcpAuth, } impl VertexAuth { - pub fn new(loader: Arc) -> Self { + pub fn new(loader: Arc) -> Self { Self { - providers: Cache::builder().max_capacity(64).build(), - loader, + identity: GcpAuth::new(loader), } } @@ -52,7 +30,10 @@ impl VertexAuth { config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - self.load_provider(config, env_lookup).await?.token().await + self.identity + .access_token(credential_source(config, env_lookup)) + .await + .map(|token| token.expose().to_owned()) } pub async fn validate_environment( @@ -77,10 +58,18 @@ impl VertexAuth { (true, _) => None, (false, Some(token)) => Some(token), (false, None) => { - let access = self.get_access_token(config, env_lookup).await?; + let provider = self.load_provider(config, env_lookup).await?; + let (token, project_id) = match project_id { + Some(project_id) => (provider.token().await?, project_id), + None => tokio::try_join!(provider.token(), provider.project_id())?, + }; return Ok(VertexEnvironment { - headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?, - project_id: project_id.unwrap_or(access.project_id), + headers: apply_credential( + headers, + token.expose(), + CredentialPlacement::Bearer, + )?, + project_id, }); } }; @@ -104,248 +93,171 @@ impl VertexAuth { }) } - async fn get_access_token( - &self, - config: &VertexConfig, - env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result { - let provider = self.load_provider(config, env_lookup).await?; - let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?; - Ok(VertexAccessToken { token, project_id }) - } - async fn load_provider( &self, config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, Error> { - let source = credential_source(config, env_lookup); - let key = source.cache_key(); - self.providers - .try_get_with(key, self.loader.load(source)) + ) -> Result, Error> { + self.identity + .load_provider(credential_source(config, env_lookup)) .await - .map_err(|error| (*error).clone()) } } -pub trait VertexTokenSource: Send + Sync { - fn project_id(&self) -> VertexAuthFuture<'_, String>; - fn token(&self) -> VertexAuthFuture<'_, String>; -} - -pub trait VertexProviderLoader: Send + Sync { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc>; -} - -pub type VertexAuthFuture<'a, T> = Pin> + Send + 'a>>; - -struct GcpTokenSource(Arc); - -impl VertexTokenSource for GcpTokenSource { - fn project_id(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async move { - self.0 - .project_id() - .await - .map(|project| project.to_string()) - .map_err(auth_acquisition_error) - }) - } - - fn token(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async move { - self.0 - .token(&[CLOUD_PLATFORM_SCOPE]) - .await - .map(|token| token.as_str().to_string()) - .map_err(auth_acquisition_error) - }) - } -} - -struct GcpProviderLoader; - -impl VertexProviderLoader for GcpProviderLoader { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc> { - Box::pin(async move { - let provider: Arc = match source { - CredentialSource::Inline(configured) => Arc::new( - CustomServiceAccount::from_json(validate_request_credentials( - configured.expose(), - )?) - .map_err(auth_acquisition_error)?, - ), - CredentialSource::Trusted(configured) => { - Arc::new(load_trusted_service_account(configured.expose())?) - } - CredentialSource::ApplicationCredentials(path) => { - Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?) - } - CredentialSource::Adc => { - gcp_auth::provider().await.map_err(auth_acquisition_error)? - } - }; - Ok(Arc::new(GcpTokenSource(provider)) as Arc) - }) - } -} - -fn validate_request_credentials(configured: &str) -> Result<&str, Error> { - let token_uri = serde_json::from_str::(configured) - .ok() - .and_then(|credentials| { - credentials - .get("token_uri") - .and_then(Value::as_str) - .map(str::to_string) - }); - if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { - return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { - field: "request-controlled vertex_credentials token_uri".into(), - expected: "the canonical Google OAuth token endpoint", - })); - } - Ok(configured) -} - -impl CredentialSource { - fn cache_key(&self) -> CredentialCacheKey { - match self { - Self::Inline(configured) => { - CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into()) - } - Self::Trusted(configured) => { - CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into()) - } - Self::ApplicationCredentials(path) => { - CredentialCacheKey::ApplicationCredentials(path.clone()) - } - Self::Adc => CredentialCacheKey::Adc, - } - } -} - -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -enum CredentialCacheKey { - Inline([u8; 32]), - Trusted([u8; 32]), - ApplicationCredentials(String), - Adc, -} - -fn load_trusted_service_account(configured: &str) -> Result { - if is_base64_encoded_json(configured) { - return Err(Error::InvalidConfiguration( - ErrorDetail::Base64EncodedJson { - field: "vertex_credentials", - }, - )); - } - if names_a_credentials_file(configured) { - CustomServiceAccount::from_file(configured) - } else { - CustomServiceAccount::from_json(configured) - } - .map_err(auth_acquisition_error) -} - -fn auth_acquisition_error(error: gcp_auth::Error) -> Error { - Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed( - "Vertex AI credentials", - error, - )) -} - #[cfg(test)] mod tests { + use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use crate::{ + CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth, + VertexConfig, get_vertex_ai_location, secret_names, + }; use litellm_secret_value::SecretValue; + use rstest::rstest; + use serde_json::{Value, json}; - use super::*; - - #[rstest::rstest] - #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)] - #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)] - #[case::missing_endpoint("{}", false)] - fn request_credentials_require_canonical_token_endpoint( - #[case] credentials: &str, - #[case] accepted: bool, - ) { - assert_eq!(validate_request_credentials(credentials).is_ok(), accepted); + struct FakeProvider { + project: Option<&'static str>, + calls: Arc, } - #[rstest::rstest] - #[case::missing_file( - "/nonexistent/vertex-key.json", - "failed to open application credentials file" - )] - #[case::malformed_json( - " {\"type\": \"service_account\",", - "failed to deserialize ApplicationCredentials" - )] - #[case::quoted_json( - "'{\"type\": \"service_account\"}'", - "failed to deserialize ApplicationCredentials" - )] - #[tokio::test] - async fn trusted_credentials_are_read_as_a_file_unless_they_are_json( - #[case] configured: &str, - #[case] expected: &str, - ) { - let Err(error) = GcpProviderLoader - .load(CredentialSource::Trusted(SecretValue::new(configured))) - .await - else { - panic!("{configured:?} should not load"); - }; - let report = litellm_error::Report::new(&error).to_string(); - assert!(report.contains(expected), "{report}"); - } + impl GcpTokenSource for FakeProvider { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + self.project.map(str::to_owned).ok_or( + litellm_auth_types::Error::CredentialAcquisition( + litellm_auth_types::ErrorDetail::Unresolved { + subject: "credential project", + }, + ), + ) + }) + } - #[rstest::rstest] - #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")] - #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")] - #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")] - #[tokio::test] - async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) { - assert!( - trusted_load_error(configured) - .await - .contains("is base64-encoded JSON, not the JSON itself") - ); - } - - async fn trusted_load_error(configured: &str) -> String { - match GcpProviderLoader - .load(CredentialSource::Trusted(SecretValue::new(configured))) - .await - { - Ok(_) => panic!("{configured:?} should not load"), - Err(error) => litellm_error::Report::new(&error).to_string(), + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(SecretValue::new("adc-token")) }) } } - #[rstest::rstest] - #[tokio::test] - async fn a_relative_key_file_named_like_json_is_read_as_a_file() { - let key = tempfile::Builder::new() - .prefix("{vertex") - .suffix(".json") - .tempfile_in(".") - .unwrap(); - std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap(); - let relative_name = key.path().file_name().unwrap().to_str().unwrap(); - let absolute_path = std::fs::canonicalize(key.path()).unwrap(); - assert_eq!( - trusted_load_error(relative_name).await, - trusted_load_error(absolute_path.to_str().unwrap()).await - ); + struct FakeLoader { + loads: Arc, + provider: Arc, } - #[rstest::rstest] - fn inline_and_trusted_credentials_never_share_a_cache_entry() { - assert_ne!( - CredentialSource::Inline(SecretValue::new("same-value")).cache_key(), - CredentialSource::Trusted(SecretValue::new("same-value")).cache_key() - ); + impl GcpProviderLoader for FakeLoader { + fn load(&self, _source: CredentialSource) -> GcpAuthFuture<'_, Arc> { + let loads = self.loads.clone(); + let provider = self.provider.clone(); + Box::pin(async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(provider) + }) + } + } + + fn config(value: Value) -> VertexConfig { + VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new()) + .unwrap() + } + + fn auth(calls: Arc, loads: Arc) -> VertexAuth { + let provider: Arc = Arc::new(FakeProvider { + calls, + project: Some("adc-project"), + }); + VertexAuth::new(Arc::new(FakeLoader { loads, provider })) + } + + #[rstest] + #[tokio::test] + async fn secret_names_cover_environment_reads() { + let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) + .await + .unwrap(); + get_vertex_ai_location(&VertexConfig::default(), &env); + let allowed = secret_names() + .iter() + .map(|name| (*name).to_owned()) + .collect::>(); + assert!(seen.lock().unwrap().is_subset(&allowed)); + } + + #[rstest] + #[tokio::test] + async fn explicit_token_and_header_do_not_acquire_adc() { + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone()); + let configured = config(json!({"vertex_project":"project-1"})); + let explicit = auth + .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None) + .await + .unwrap(); + assert_eq!(explicit.headers[0].1, "Bearer access-token"); + let existing = auth + .validate_environment( + vec![("authorization".into(), "Bearer existing".into())], + None, + &configured, + &|_| None, + ) + .await + .unwrap(); + assert_eq!(existing.headers[0].1, "Bearer existing"); + assert_eq!(loads.load(Ordering::SeqCst), 0); + } + + #[rstest] + #[tokio::test] + async fn provider_is_reused_across_authentication_calls() { + let calls = Arc::new(AtomicUsize::new(0)); + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(calls.clone(), loads.clone()); + let config = VertexConfig::default(); + let environment = |_: &str| None; + let (first, second) = tokio::try_join!( + auth.validate_environment(Vec::new(), None, &config, &environment), + auth.validate_environment(Vec::new(), None, &config, &environment), + ) + .unwrap(); + assert_eq!(first.project_id, "adc-project"); + assert_eq!(first.headers[0].1, "Bearer adc-token"); + assert_eq!(first.project_id, second.project_id); + assert_eq!(first.headers, second.headers); + assert_eq!(loads.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 4); + } + + #[rstest] + #[tokio::test] + async fn an_explicit_project_does_not_require_project_metadata_in_the_credentials() { + let auth = VertexAuth::new(Arc::new(FakeLoader { + loads: Arc::new(AtomicUsize::new(0)), + provider: Arc::new(FakeProvider { + calls: Arc::new(AtomicUsize::new(0)), + project: None, + }), + })); + let configured = VertexConfig::new(None, Some("configured-project".into()), None); + let environment = auth + .validate_environment(Vec::new(), None, &configured, &|_| None) + .await + .unwrap(); + assert_eq!(environment.project_id, "configured-project"); + assert_eq!(environment.headers[0].1, "Bearer adc-token"); } } diff --git a/litellm-rust/crates/auth-gcp/src/config.rs b/litellm-rust/crates/auth-gcp/src/config.rs index 133e1c26903..048e40b0a2f 100644 --- a/litellm-rust/crates/auth-gcp/src/config.rs +++ b/litellm-rust/crates/auth-gcp/src/config.rs @@ -146,6 +146,20 @@ pub enum CredentialSource { Adc, } +impl CredentialSource { + pub fn from_vertex_environment(env_lookup: &dyn Fn(&str) -> Option) -> Self { + VertexParams::CREDENTIALS + .resolve(&|_| None, env_lookup) + .map(|configured| Self::Trusted(SecretValue::new(configured))) + .unwrap_or_else(|| Self::application_default(env_lookup)) + } + + pub fn application_default(env_lookup: &dyn Fn(&str) -> Option) -> Self { + non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) + .map_or(Self::Adc, Self::ApplicationCredentials) + } +} + pub(crate) fn credential_source( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, @@ -158,12 +172,7 @@ pub(crate) fn credential_source( } }; } - if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) { - return CredentialSource::Trusted(SecretValue::new(configured)); - } - non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) - .map(CredentialSource::ApplicationCredentials) - .unwrap_or(CredentialSource::Adc) + CredentialSource::from_vertex_environment(env_lookup) } fn optional_credentials( @@ -319,6 +328,24 @@ mod tests { )); } + #[rstest::rstest] + fn application_default_ignores_vertex_credentials() { + let env = |name: &str| match name { + VERTEXAI_CREDENTIALS_ENV => Some("environment-json".to_string()), + GOOGLE_APPLICATION_CREDENTIALS_ENV => Some("adc.json".to_string()), + _ => None, + }; + assert!( + matches!(CredentialSource::application_default(&env), CredentialSource::ApplicationCredentials(path) if path == "adc.json") + ); + assert!(matches!( + CredentialSource::application_default( + &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into()) + ), + CredentialSource::Adc + )); + } + #[rstest::rstest] fn params_become_a_deployment_sourced_config() { let params = VertexParams { diff --git a/litellm-rust/crates/auth-gcp/src/identity.rs b/litellm-rust/crates/auth-gcp/src/identity.rs new file mode 100644 index 00000000000..5144221b218 --- /dev/null +++ b/litellm-rust/crates/auth-gcp/src/identity.rs @@ -0,0 +1,445 @@ +use crate::{ + config::{CredentialSource, is_base64_encoded_json, names_a_credentials_file}, + constants::{CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT}, +}; +use gcp_auth::{CustomServiceAccount, TokenProvider}; +use litellm_auth_types::{Error, ErrorDetail}; +use litellm_secret_value::SecretValue; +use moka::future::Cache; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{future::Future, pin::Pin, sync::Arc}; + +#[derive(Clone)] +pub struct GcpAuth { + providers: Cache>, + loader: Arc, +} + +impl Default for GcpAuth { + fn default() -> Self { + Self::new(Arc::new(NativeProviderLoader)) + } +} + +impl GcpAuth { + pub fn new(loader: Arc) -> Self { + Self { + providers: Cache::builder().max_capacity(64).build(), + loader, + } + } + + pub async fn access_token(&self, source: CredentialSource) -> Result { + self.load_provider(source).await?.token().await + } + + pub(crate) async fn load_provider( + &self, + source: CredentialSource, + ) -> Result, Error> { + self.providers + .try_get_with(source.cache_key(), self.loader.load(source)) + .await + .map_err(|error| (*error).clone()) + } +} + +pub trait GcpTokenSource: Send + Sync { + fn project_id(&self) -> GcpAuthFuture<'_, String>; + fn token(&self) -> GcpAuthFuture<'_, SecretValue>; +} + +pub trait GcpProviderLoader: Send + Sync { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc>; +} + +pub type GcpAuthFuture<'a, T> = Pin> + Send + 'a>>; + +struct ServiceAccountTokenSource(Arc); + +impl GcpTokenSource for ServiceAccountTokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async move { + self.0 + .project_id() + .await + .map(|project| project.to_string()) + .map_err(auth_acquisition_error) + }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + self.0 + .token(&[CLOUD_PLATFORM_SCOPE]) + .await + .map(|token| SecretValue::new(token.as_str())) + .map_err(auth_acquisition_error) + }) + } +} + +struct SdkTokenSource { + credentials: google_cloud_auth::credentials::AccessTokenCredentials, + project_id: Option, +} + +impl GcpTokenSource for SdkTokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async move { + self.project_id + .clone() + .ok_or(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "GCP resource project", + })) + }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + self.credentials + .access_token() + .await + .map(|token| SecretValue::new(token.token)) + .map_err(auth_acquisition_error) + }) + } +} + +async fn load_application_credentials(path: &str) -> Result, Error> { + let contents = tokio::fs::read_to_string(path) + .await + .map_err(auth_acquisition_error)?; + load_credentials_json(&contents) +} + +fn load_credentials_json(contents: &str) -> Result, Error> { + let configured: Value = serde_json::from_str(contents).map_err(auth_acquisition_error)?; + let kind = configured.get("type").and_then(Value::as_str); + let project_id = configured + .get("project_id") + .and_then(Value::as_str) + .map(str::to_string); + let credentials = match kind { + Some("authorized_user") => { + google_cloud_auth::credentials::user_account::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("impersonated_service_account") => { + google_cloud_auth::credentials::impersonated::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("external_account") => { + google_cloud_auth::credentials::external_account::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("service_account") | None => { + let provider = + CustomServiceAccount::from_json(contents).map_err(auth_acquisition_error)?; + return Ok(Arc::new(ServiceAccountTokenSource(Arc::new(provider)))); + } + Some(_) => { + return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { + field: "Google application credentials type".into(), + expected: "a supported Google credential type", + })); + } + } + .map_err(auth_acquisition_error)?; + Ok(Arc::new(SdkTokenSource { + credentials, + project_id, + })) +} + +struct NativeProviderLoader; + +impl GcpProviderLoader for NativeProviderLoader { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc> { + Box::pin(async move { + let provider: Arc = match source { + CredentialSource::Inline(configured) => Arc::new( + CustomServiceAccount::from_json(validate_request_credentials( + configured.expose(), + )?) + .map_err(auth_acquisition_error)?, + ), + CredentialSource::Trusted(configured) => { + return load_trusted_credentials(configured.expose()).await; + } + CredentialSource::ApplicationCredentials(path) => { + return load_application_credentials(&path).await; + } + CredentialSource::Adc => { + gcp_auth::provider().await.map_err(auth_acquisition_error)? + } + }; + Ok(Arc::new(ServiceAccountTokenSource(provider)) as Arc) + }) + } +} + +fn validate_request_credentials(configured: &str) -> Result<&str, Error> { + let token_uri = serde_json::from_str::(configured) + .ok() + .and_then(|credentials| { + credentials + .get("token_uri") + .and_then(Value::as_str) + .map(str::to_string) + }); + if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { + return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { + field: "request-controlled vertex_credentials token_uri".into(), + expected: "the canonical Google OAuth token endpoint", + })); + } + Ok(configured) +} + +impl CredentialSource { + fn cache_key(&self) -> CredentialCacheKey { + match self { + Self::Inline(configured) => { + CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into()) + } + Self::Trusted(configured) => { + CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into()) + } + Self::ApplicationCredentials(path) => { + CredentialCacheKey::ApplicationCredentials(path.clone()) + } + Self::Adc => CredentialCacheKey::Adc, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +enum CredentialCacheKey { + Inline([u8; 32]), + Trusted([u8; 32]), + ApplicationCredentials(String), + Adc, +} + +async fn load_trusted_credentials(configured: &str) -> Result, Error> { + if is_base64_encoded_json(configured) { + return Err(Error::InvalidConfiguration( + ErrorDetail::Base64EncodedJson { + field: "vertex_credentials", + }, + )); + } + if names_a_credentials_file(configured) { + load_application_credentials(configured).await + } else { + load_credentials_json(configured) + } +} + +fn auth_acquisition_error(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed( + "Google credentials", + error, + )) +} + +#[cfg(test)] +mod tests { + use litellm_secret_value::SecretValue; + + use super::*; + + #[rstest::rstest] + #[case::authorized_user("authorized_user", 2)] + #[case::external_account("external_account", 2)] + #[case::impersonated_service_account("impersonated_service_account", 4)] + #[tokio::test] + async fn explicit_application_credentials_refresh_expired_tokens( + #[case] kind: &str, + #[case] expected_requests: usize, + #[values(false, true)] inline: bool, + ) { + use axum::{Json, Router, routing::post}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let _ = rustls::crypto::ring::default_provider().install_default(); + let requests = Arc::new(AtomicUsize::new(0)); + let recorded = requests.clone(); + let impersonations = requests.clone(); + let app = Router::new().route("/token", post(move || { + let recorded = recorded.clone(); + async move { + Json(serde_json::json!({ + "access_token": format!("token-{}", recorded.fetch_add(1, Ordering::SeqCst)), + "expires_in": 3600, + "token_type": "Bearer", + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + })) + } + })); + let app = app.route("/v1/projects/-/serviceAccounts/test-principal:generateAccessToken", post(move || { + let recorded = impersonations.clone(); + async move { + let expiration = (time::OffsetDateTime::now_utc() + time::Duration::hours(1)).format(&time::format_description::well_known::Rfc3339).unwrap(); + Json(serde_json::json!({ + "accessToken": format!("impersonated-{}", recorded.fetch_add(1, Ordering::SeqCst)), + "expireTime": expiration, + })) + } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let file = tempfile::NamedTempFile::new().unwrap(); + let subject_token = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(subject_token.path(), "subject-token").unwrap(); + let configured = if kind == "external_account" { + serde_json::json!({ + "type": "external_account", "audience": "//iam.googleapis.com/projects/test/locations/global/workloadIdentityPools/test/providers/test", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": format!("http://{address}/token"), + "credential_source": { "file": subject_token.path().to_str().unwrap() }, "quota_project_id": "quota-project" + }) + } else { + let source = serde_json::json!({ + "type": "authorized_user", "client_id": "user-client", "client_secret": "user-secret", + "refresh_token": "user-refresh", "quota_project_id": "quota-project", "token_uri": format!("http://{address}/token") + }); + if kind == "impersonated_service_account" { + serde_json::json!({ + "type": kind, "source_credentials": source, "quota_project_id": "quota-project", + "service_account_impersonation_url": format!("http://{address}/v1/projects/-/serviceAccounts/test-principal:generateAccessToken") + }) + } else { + source + } + }; + std::fs::write(file.path(), configured.to_string()).unwrap(); + let source = if inline { + CredentialSource::Trusted(SecretValue::new(configured.to_string())) + } else { + CredentialSource::ApplicationCredentials(file.path().to_str().unwrap().into()) + }; + let provider = NativeProviderLoader.load(source).await.unwrap(); + + assert!(matches!( + provider.project_id().await, + Err(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "GCP resource project" + })) + )); + let original = provider.token().await.unwrap(); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(3601)).await; + tokio::time::resume(); + let refreshed = provider.token().await.unwrap(); + assert_ne!(original, refreshed); + assert_eq!(requests.load(Ordering::SeqCst), expected_requests); + server.abort(); + } + + #[rstest::rstest] + #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)] + #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)] + #[case::missing_endpoint("{}", false)] + fn request_credentials_require_canonical_token_endpoint( + #[case] credentials: &str, + #[case] accepted: bool, + ) { + assert_eq!(validate_request_credentials(credentials).is_ok(), accepted); + } + + #[rstest::rstest] + #[case::missing_file("/nonexistent/vertex-key.json", "Google credentials")] + #[case::malformed_json(" {\"type\": \"service_account\",", "Google credentials")] + #[case::quoted_json("'{\"type\": \"service_account\"}'", "Google credentials")] + #[tokio::test] + async fn trusted_credentials_are_read_as_a_file_unless_they_are_json( + #[case] configured: &str, + #[case] expected: &str, + ) { + let Err(error) = NativeProviderLoader + .load(CredentialSource::Trusted(SecretValue::new(configured))) + .await + else { + panic!("{configured:?} should not load"); + }; + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.contains(expected), "{report}"); + } + + #[rstest::rstest] + #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")] + #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")] + #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")] + #[tokio::test] + async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) { + assert!( + trusted_load_error(configured) + .await + .contains("is base64-encoded JSON, not the JSON itself") + ); + } + + async fn trusted_load_error(configured: &str) -> String { + match NativeProviderLoader + .load(CredentialSource::Trusted(SecretValue::new(configured))) + .await + { + Ok(_) => panic!("{configured:?} should not load"), + Err(error) => litellm_error::Report::new(&error).to_string(), + } + } + + #[rstest::rstest] + #[tokio::test] + async fn a_relative_key_file_named_like_json_is_read_as_a_file() { + let key = tempfile::Builder::new() + .prefix("{vertex") + .suffix(".json") + .tempfile_in(".") + .unwrap(); + std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap(); + let relative_name = key.path().file_name().unwrap().to_str().unwrap(); + let absolute_path = std::fs::canonicalize(key.path()).unwrap(); + assert_eq!( + trusted_load_error(relative_name).await, + trusted_load_error(absolute_path.to_str().unwrap()).await + ); + } + + #[rstest::rstest] + #[tokio::test] + async fn unknown_application_credential_types_are_rejected_without_service_account_fallback() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write( + file.path(), + r#"{"type":"unsupported","private_key":"credential-value"}"#, + ) + .unwrap(); + let result = GcpAuth::default() + .access_token(CredentialSource::ApplicationCredentials( + file.path().to_str().unwrap().into(), + )) + .await; + let error = result.unwrap_err(); + assert!(matches!(error, Error::InvalidConfiguration(_))); + assert!( + !litellm_error::Report::new(&error) + .to_string() + .contains("credential-value") + ); + } + + #[rstest::rstest] + fn inline_and_trusted_credentials_never_share_a_cache_entry() { + assert_ne!( + CredentialSource::Inline(SecretValue::new("same-value")).cache_key(), + CredentialSource::Trusted(SecretValue::new("same-value")).cache_key() + ); + } +} diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 2b389d45926..03c660e6eb8 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -1,16 +1,16 @@ mod auth; mod config; pub mod constants; +mod identity; #[cfg(feature = "google-sdk")] mod sdk; -pub use auth::{ - VertexAuth, VertexAuthFuture, VertexEnvironment, VertexProviderLoader, VertexTokenSource, -}; +pub use auth::{VertexAuth, VertexEnvironment}; pub use config::{ CredentialSource, VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, }; pub use constants::secret_names; +pub use identity::{GcpAuth, GcpAuthFuture, GcpProviderLoader, GcpTokenSource}; #[cfg(feature = "google-sdk")] pub use sdk::GoogleCredentials; diff --git a/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs b/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs deleted file mode 100644 index 5991749ab8a..00000000000 --- a/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, -}; - -use litellm_auth_gcp::{ - CredentialSource, VertexAuth, VertexAuthFuture, VertexConfig, VertexProviderLoader, - VertexTokenSource, get_vertex_ai_location, secret_names, -}; -use rstest::rstest; -use serde_json::{Value, json}; - -struct FakeProvider { - calls: Arc, -} - -impl VertexTokenSource for FakeProvider { - fn project_id(&self) -> VertexAuthFuture<'_, String> { - self.calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok("adc-project".into()) }) - } - - fn token(&self) -> VertexAuthFuture<'_, String> { - self.calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok("adc-token".into()) }) - } -} - -struct FakeLoader { - loads: Arc, - provider: Arc, -} - -impl VertexProviderLoader for FakeLoader { - fn load(&self, _source: CredentialSource) -> VertexAuthFuture<'_, Arc> { - let loads = self.loads.clone(); - let provider = self.provider.clone(); - Box::pin(async move { - loads.fetch_add(1, Ordering::SeqCst); - Ok(provider) - }) - } -} - -fn config(value: Value) -> VertexConfig { - VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new()) - .unwrap() -} - -fn auth(calls: Arc, loads: Arc) -> VertexAuth { - let provider: Arc = Arc::new(FakeProvider { calls }); - VertexAuth::new(Arc::new(FakeLoader { loads, provider })) -} - -#[rstest] -#[tokio::test] -async fn secret_names_cover_environment_reads() { - let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); - let recorded = seen.clone(); - let env = |name: &str| { - recorded.lock().unwrap().insert(name.to_string()); - None - }; - let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); - auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) - .await - .unwrap(); - get_vertex_ai_location(&VertexConfig::default(), &env); - assert!( - seen.lock() - .unwrap() - .iter() - .all(|name| secret_names().contains(&name.as_str())) - ); -} - -#[rstest] -#[tokio::test] -async fn explicit_token_and_header_do_not_acquire_adc() { - let loads = Arc::new(AtomicUsize::new(0)); - let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone()); - let configured = config(json!({"vertex_project":"project-1"})); - let explicit = auth - .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None) - .await - .unwrap(); - assert_eq!(explicit.headers[0].1, "Bearer access-token"); - let existing = auth - .validate_environment( - vec![("authorization".into(), "Bearer existing".into())], - None, - &configured, - &|_| None, - ) - .await - .unwrap(); - assert_eq!(existing.headers[0].1, "Bearer existing"); - assert_eq!(loads.load(Ordering::SeqCst), 0); -} - -#[rstest] -#[tokio::test] -async fn provider_is_reused_across_authentication_calls() { - let calls = Arc::new(AtomicUsize::new(0)); - let loads = Arc::new(AtomicUsize::new(0)); - let auth = auth(calls.clone(), loads.clone()); - for _ in 0..2 { - let environment = auth - .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None) - .await - .unwrap(); - assert_eq!(environment.project_id, "adc-project"); - assert_eq!(environment.headers[0].1, "Bearer adc-token"); - } - assert_eq!(loads.load(Ordering::SeqCst), 1); - assert_eq!(calls.load(Ordering::SeqCst), 4); -} diff --git a/litellm-rust/crates/auth-types/src/error.rs b/litellm-rust/crates/auth-types/src/error.rs index a8ba7b64e03..a2e5629bf3f 100644 --- a/litellm-rust/crates/auth-types/src/error.rs +++ b/litellm-rust/crates/auth-types/src/error.rs @@ -92,6 +92,11 @@ impl Error { #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum ErrorDetail { + #[error("{operation} returned HTTP {status}")] + UnexpectedStatus { + operation: &'static str, + status: u16, + }, #[error("{field} must be {expected}")] InvalidType { field: String, diff --git a/litellm-rust/crates/auth-types/src/params/aws.rs b/litellm-rust/crates/auth-types/src/params/aws.rs index 8075fc15e9f..de21fbf94c9 100644 --- a/litellm-rust/crates/auth-types/src/params/aws.rs +++ b/litellm-rust/crates/auth-types/src/params/aws.rs @@ -53,6 +53,9 @@ pub struct AwsParams { } impl AwsParams { + pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN"; + pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; + pub const ACCESS_KEY_ID: ParamSpec = spec( "access key id", ParamKind::Credential, @@ -149,6 +152,10 @@ impl AwsParams { AwsParams::SPECS .iter() .flat_map(|spec| spec.env.iter().copied()) + .chain([ + AwsParams::ROLE_ARN_ENV, + AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV, + ]) .fold(Vec::new(), |mut names, name| { if !names.contains(&name) { names.push(name); diff --git a/litellm-rust/crates/auth-types/tests/aws_params.rs b/litellm-rust/crates/auth-types/tests/aws_params.rs index ed9a4f78ec2..b438984c199 100644 --- a/litellm-rust/crates/auth-types/tests/aws_params.rs +++ b/litellm-rust/crates/auth-types/tests/aws_params.rs @@ -3,21 +3,13 @@ use rstest::rstest; use serde_json::{Map, Value}; #[rstest] -fn fields_name_every_param_once_and_in_declaration_order() { +fn every_named_param_roundtrips_without_loss() { let filled: Map = AwsParams::fields() .map(|name| (name.to_string(), Value::from(format!("value-of-{name}")))) .collect(); let typed = AwsParams::from_optional_params(&filled); let serialized = serde_json::to_value(&typed).unwrap(); - assert_eq!( - serialized - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect::>(), - AwsParams::fields().collect::>() - ); + assert_eq!(filled.len(), AwsParams::fields().count()); assert_eq!(serialized, Value::Object(filled.clone())); assert_eq!( serde_json::from_value::(Value::Object(filled)).unwrap(), diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index a6b08445bc8..b8c6720c234 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -8,7 +8,7 @@ use litellm_cache::{ use litellm_http::Client; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; -use crate::{GcpTokenSource, TokenSource}; +use crate::{GcsTokenSource, TokenSource}; pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; @@ -62,7 +62,7 @@ pub struct GcsCache { impl GcsCache { pub fn new(config: GcsConfig, client: Client, codec: S) -> Self { - let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); + let token = Arc::new(GcsTokenSource::new(config.path_service_account.clone())); Self::with_token_source(config, client, codec, token) } diff --git a/litellm-rust/crates/cache-gcs/src/lib.rs b/litellm-rust/crates/cache-gcs/src/lib.rs index cbb61cf0685..d40bca081a4 100644 --- a/litellm-rust/crates/cache-gcs/src/lib.rs +++ b/litellm-rust/crates/cache-gcs/src/lib.rs @@ -2,4 +2,4 @@ mod cache; mod token; pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix}; -pub use token::{GcpTokenSource, StaticTokenSource, TokenSource}; +pub use token::{GcsTokenSource, StaticTokenSource, TokenSource}; diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs index 852d8bbf035..4f639e7b865 100644 --- a/litellm-rust/crates/cache-gcs/src/token.rs +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -1,7 +1,6 @@ use std::{future::Future, pin::Pin}; -use litellm_auth_gcp::{VertexAuth, VertexConfig}; -use litellm_auth_types::{InputSource, Sourced}; +use litellm_auth_gcp::{CredentialSource, GcpAuth}; use litellm_secret_value::SecretValue; use litellm_cache::Error; @@ -10,28 +9,30 @@ pub trait TokenSource: Send + Sync + 'static { fn bearer_token(&self) -> Pin> + Send + '_>>; } -pub struct GcpTokenSource { - auth: VertexAuth, - config: VertexConfig, +pub struct GcsTokenSource { + auth: GcpAuth, + credentials: Option, } -impl GcpTokenSource { +impl GcsTokenSource { pub fn new(path_service_account: Option) -> Self { - let credentials = path_service_account - .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment)); Self { - auth: VertexAuth::default(), - config: VertexConfig::new(credentials, None, None), + auth: GcpAuth::default(), + credentials: path_service_account.filter(|path| !path.trim().is_empty()), } } } -impl TokenSource for GcpTokenSource { +impl TokenSource for GcsTokenSource { fn bearer_token(&self) -> Pin> + Send + '_>> { Box::pin(async move { self.auth - .access_token(&self.config, &|name| std::env::var(name).ok()) + .access_token(match &self.credentials { + Some(path) => CredentialSource::Trusted(SecretValue::new(path.clone())), + None => CredentialSource::application_default(&|name| std::env::var(name).ok()), + }) .await + .map(|token| token.expose().to_owned()) .map_err(Error::unavailable) }) } diff --git a/litellm-rust/crates/cache-redis-semantic/Cargo.toml b/litellm-rust/crates/cache-redis-semantic/Cargo.toml deleted file mode 100644 index 6a5c794bde7..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "litellm-cache-redis-semantic" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -litellm-cache.workspace = true -litellm-cache-redis.workspace = true -redis = { workspace = true, features = ["tls-rustls"] } -sha2.workspace = true - -[dev-dependencies] -litellm-cache-testing.workspace = true -redis-test.workspace = true -rstest.workspace = true -serde_json.workspace = true -tokio.workspace = true diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs deleted file mode 100644 index 42fc6ae585f..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs +++ /dev/null @@ -1,404 +0,0 @@ -use std::{ - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use litellm_cache::{ - BaseCache, CacheCodec, Error, SemanticCacheContext, - semantic::{Embedder, SemanticCache, SemanticLookup, prompt_from_context}, -}; -use litellm_cache_redis::{ - RedisTopology, - connection::{ConnectionRef, Connections}, -}; -use sha2::{Digest, Sha256}; - -use crate::{ - RedisSemanticConfig, - index::{CACHE_KEY_FIELD, Index, VECTOR_FIELD}, - reply::{bytes_field, first_document, number_field, string_field}, -}; - -struct Inner { - index: Index, - distance_threshold: f64, - clock: fn() -> f64, -} - -impl Inner { - fn new(config: RedisSemanticConfig, clock: fn() -> f64) -> Self { - Self { - index: Index::new(config.index_name), - distance_threshold: 1.0 - f64::from(config.similarity_threshold), - clock, - } - } - - fn store( - &self, - connection: &mut ConnectionRef<'_>, - tag: &str, - response: Vec, - prompt: &str, - vector: &[f32], - ttl: Option, - ) -> Result<(), Error> { - let index = self.index.ensure(connection, vector.len())?; - let entry_id = entry_id(prompt, tag); - let hash_key = format!("{index}:{entry_id}"); - redis::cmd("HSET") - .arg(&hash_key) - .arg("entry_id") - .arg(&entry_id) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(response) - .arg(VECTOR_FIELD) - .arg(vector_buffer(vector)) - .arg("inserted_at") - .arg(format!("{}", (self.clock)())) - .arg("updated_at") - .arg(format!("{}", (self.clock)())) - .arg(CACHE_KEY_FIELD) - .arg(tag) - .query::<()>(connection) - .map_err(Error::unavailable)?; - if let Some(ttl) = ttl { - redis::cmd("EXPIRE") - .arg(&hash_key) - .arg(ttl_seconds(ttl)) - .query::<()>(connection) - .map_err(Error::unavailable)?; - } - Ok(()) - } - - fn lookup( - &self, - connection: &mut ConnectionRef<'_>, - tag: &str, - vector: &[f32], - ) -> Result>, Error> { - let index = self.index.ensure(connection, vector.len())?; - let query = format!( - "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]", - escape_tag(tag) - ); - let result = redis::cmd("FT.SEARCH") - .arg(&index) - .arg(query) - .arg("RETURN") - .arg(8) - .arg("entry_id") - .arg("prompt") - .arg("response") - .arg("inserted_at") - .arg("updated_at") - .arg("metadata") - .arg(CACHE_KEY_FIELD) - .arg("vector_distance") - .arg("SORTBY") - .arg("vector_distance") - .arg("ASC") - .arg("DIALECT") - .arg(2) - .arg("LIMIT") - .arg(0) - .arg(1) - .arg("PARAMS") - .arg(2) - .arg("vector") - .arg(vector_buffer(vector)) - .query::(connection) - .map_err(Error::unavailable)?; - let Some(fields) = first_document(&result) else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) { - return Ok(SemanticLookup::miss(Some(0.0))); - } - // redisvl's range query only returns entries within the distance threshold, so a - // farther hit reads as no result. - let Some(distance) = number_field(fields, "vector_distance") - .filter(|distance| *distance <= self.distance_threshold) - else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let Some(response) = bytes_field(fields, "response") else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - Ok(SemanticLookup { - value: Some(response), - similarity: Some(1.0 - distance), - }) - } -} - -/// `RedisSemanticCache`: a redisvl-compatible semantic index on Redis Stack. Values go through -/// the injected codec, so the response layer decides what a cached entry is. -pub struct RedisSemanticCache { - connections: Arc>, - embedder: E, - codec: S, - inner: Arc, -} - -impl RedisSemanticCache { - pub fn new( - url: &str, - embedder: E, - codec: S, - config: RedisSemanticConfig, - ) -> Result { - Ok(Self { - connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), - embedder, - codec, - inner: Arc::new(Inner::new(config, timestamp)), - }) - } -} - -impl RedisSemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn with_connection( - connection: C, - embedder: E, - codec: S, - config: RedisSemanticConfig, - ) -> Self { - Self { - connections: Arc::new(Connections::fixed(connection)), - embedder, - codec, - inner: Arc::new(Inner::new(config, timestamp)), - } - } - - pub fn with_clock(self, clock: fn() -> f64) -> Self { - let config = RedisSemanticConfig { - index_name: self.index_name().to_owned(), - similarity_threshold: self.similarity_threshold(), - }; - Self { - inner: Arc::new(Inner::new(config, clock)), - ..self - } - } - - pub fn embedder(&self) -> &E { - &self.embedder - } - - pub fn index_name(&self) -> &str { - self.inner.index.name() - } - - pub fn similarity_threshold(&self) -> f32 { - (1.0 - self.inner.distance_threshold) as f32 - } - - fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str { - context.scope.as_deref().unwrap_or(key) - } - - fn decode(&self, lookup: SemanticLookup>) -> Result, Error> { - Ok(SemanticLookup { - value: lookup - .value - .map(|bytes| self.codec.decode(&bytes)) - .transpose()?, - similarity: lookup.similarity, - }) - } -} - -impl BaseCache for RedisSemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Value = S::Value; - type Context = SemanticCacheContext; - - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &Self::Context, - ) -> Result<(), Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(()); - }; - let response = self.codec.encode(&value)?; - let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let tag = Self::tag(key, context); - self.connections.execute(|connection| { - self.inner - .store(connection, tag, response, &prompt, &vector, context.ttl) - }) - } - - fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { - self.get_cache_with_similarity(key, context) - .map(|lookup| lookup.value) - } - - async fn async_set_cache( - &self, - key: &str, - value: Self::Value, - context: Self::Context, - ) -> Result<(), Error> { - let Some(prompt) = prompt_from_context(&context) else { - return Ok(()); - }; - let response = self.codec.encode(&value)?; - let vector = self - .embedder - .async_embed(&prompt, context.metadata.as_ref()) - .await?; - let tag = Self::tag(key, &context).to_owned(); - let inner = Arc::clone(&self.inner); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - inner.store(connection, &tag, response, &prompt, &vector, context.ttl) - }) - .await - } - - async fn async_get_cache( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - self.async_get_cache_with_similarity(key, context) - .await - .map(|lookup| lookup.value) - } -} - -/// Python stamps a similarity of `0.0` when there is no prompt or no hit in the key's scope. -impl SemanticCache for RedisSemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn get_cache_with_similarity( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let tag = Self::tag(key, context); - let lookup = self - .connections - .execute(|connection| self.inner.lookup(connection, tag, &vector))?; - self.decode(lookup) - } - - async fn async_get_cache_with_similarity( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let vector = self - .embedder - .async_embed(&prompt, context.metadata.as_ref()) - .await?; - let tag = Self::tag(key, context).to_owned(); - let inner = Arc::clone(&self.inner); - let lookup = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - inner.lookup(connection, &tag, &vector) - }) - .await?; - self.decode(lookup) - } -} - -fn timestamp() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs_f64()) - .unwrap_or_default() -} - -fn entry_id(prompt: &str, tag: &str) -> String { - let mut digest = Sha256::new(); - digest.update(prompt.as_bytes()); - digest.update(CACHE_KEY_FIELD.as_bytes()); - digest.update(tag.as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn vector_buffer(vector: &[f32]) -> Vec { - vector - .iter() - .flat_map(|component| component.to_le_bytes()) - .collect() -} - -fn escape_tag(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for ch in value.chars() { - if matches!( - ch, - ',' | '.' - | '<' - | '>' - | '{' - | '}' - | '[' - | ']' - | '\\' - | '"' - | '\'' - | ':' - | ';' - | '!' - | '@' - | '#' - | '$' - | '%' - | '^' - | '&' - | '*' - | '(' - | ')' - | '-' - | '+' - | '=' - | '~' - | '|' - | '/' - | ' ' - | '?' - ) { - escaped.push('\\'); - } - escaped.push(ch); - } - escaped -} - -fn ttl_seconds(ttl: Duration) -> u64 { - ttl.as_secs() - .saturating_add(u64::from(ttl.subsec_nanos() > 0)) - .max(1) -} diff --git a/litellm-rust/crates/cache-redis-semantic/src/config.rs b/litellm-rust/crates/cache-redis-semantic/src/config.rs deleted file mode 100644 index 6b810628be7..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/config.rs +++ /dev/null @@ -1,8 +0,0 @@ -/// `RedisSemanticCache.DEFAULT_REDIS_INDEX_NAME`. -pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; - -#[derive(Clone, Debug)] -pub struct RedisSemanticConfig { - pub index_name: String, - pub similarity_threshold: f32, -} diff --git a/litellm-rust/crates/cache-redis-semantic/src/index.rs b/litellm-rust/crates/cache-redis-semantic/src/index.rs deleted file mode 100644 index 23401938a0e..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/index.rs +++ /dev/null @@ -1,209 +0,0 @@ -use std::sync::OnceLock; - -use litellm_cache::Error; -use litellm_cache_redis::connection::ConnectionRef; - -use crate::reply::{number_value, string_value}; - -pub(crate) const CACHE_KEY_FIELD: &str = "litellm_cache_key"; -pub(crate) const VECTOR_FIELD: &str = "prompt_vector"; - -/// The redisvl `SemanticCache` index, resolved once per cache: the configured name when its -/// schema fits, else `_isolated`, recreated when that one is stale too. -pub(crate) struct Index { - name: String, - resolved: OnceLock, -} - -impl Index { - pub(crate) fn new(name: String) -> Self { - Self { - name, - resolved: OnceLock::new(), - } - } - - pub(crate) fn name(&self) -> &str { - &self.name - } - - pub(crate) fn ensure( - &self, - connection: &mut ConnectionRef<'_>, - dims: usize, - ) -> Result { - if let Some(name) = self.resolved.get() { - return Ok(name.clone()); - } - let name = match index_compatible(connection, &self.name, dims)? { - Some(true) => self.name.clone(), - Some(false) => self.isolated(connection, dims)?, - None => match create_index(connection, &self.name, dims) { - Ok(()) => self.name.clone(), - Err(error) => match index_compatible(connection, &self.name, dims)? { - Some(true) => self.name.clone(), - Some(false) => self.isolated(connection, dims)?, - None => return Err(error), - }, - }, - }; - let _ = self.resolved.set(name.clone()); - Ok(name) - } - - fn isolated(&self, connection: &mut ConnectionRef<'_>, dims: usize) -> Result { - let name = format!("{}_isolated", self.name); - match index_compatible(connection, &name, dims)? { - Some(true) => Ok(name), - Some(false) => { - redis::cmd("FT.DROPINDEX") - .arg(&name) - .query::<()>(connection) - .map_err(Error::unavailable)?; - create_index(connection, &name, dims)?; - Ok(name) - } - None => { - create_index(connection, &name, dims)?; - Ok(name) - } - } - } -} - -fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> { - redis::cmd("FT.CREATE") - .arg(name) - .arg("ON") - .arg("HASH") - .arg("PREFIX") - .arg(1) - .arg(name) - .arg("SCORE") - .arg(1.0) - .arg("SCHEMA") - .arg("prompt") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("response") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("inserted_at") - .arg("NUMERIC") - .arg("updated_at") - .arg("NUMERIC") - .arg(VECTOR_FIELD) - .arg("VECTOR") - .arg("FLAT") - .arg(6) - .arg("TYPE") - .arg("FLOAT32") - .arg("DIM") - .arg(dims) - .arg("DISTANCE_METRIC") - .arg("COSINE") - .arg(CACHE_KEY_FIELD) - .arg("TAG") - .arg("SEPARATOR") - .arg(",") - .query::<()>(connection) - .map_err(Error::unavailable) -} - -fn index_compatible( - connection: &mut ConnectionRef<'_>, - name: &str, - dims: usize, -) -> Result, Error> { - let info = match redis::cmd("FT.INFO") - .arg(name) - .query::(connection) - { - Ok(info) => info, - Err(error) if unknown_index(&error) => return Ok(None), - Err(error) => return Err(Error::unavailable(error)), - }; - Ok(Some(schema_compatible(&info, dims))) -} - -fn unknown_index(error: &redis::RedisError) -> bool { - error - .code() - .is_some_and(|code| code.eq_ignore_ascii_case("unknown")) - && error - .detail() - .is_some_and(|detail| detail.to_ascii_lowercase().starts_with("index")) -} - -struct Attribute { - name: Option, - field_type: Option, - dim: Option, - data_type: Option, - distance_metric: Option, -} - -fn attribute(value: &redis::Value) -> Option { - let redis::Value::Array(pairs) = value else { - return None; - }; - let mut attribute = Attribute { - name: None, - field_type: None, - dim: None, - data_type: None, - distance_metric: None, - }; - for pair in pairs.as_chunks::<2>().0 { - match string_value(&pair[0]).as_deref() { - Some("identifier") => attribute.name = string_value(&pair[1]), - Some("type") => attribute.field_type = string_value(&pair[1]), - Some("dim") => attribute.dim = number_value(&pair[1]), - Some("data_type") => attribute.data_type = string_value(&pair[1]), - Some("distance_metric") => attribute.distance_metric = string_value(&pair[1]), - _ => {} - } - } - Some(attribute) -} - -fn schema_compatible(info: &redis::Value, dims: usize) -> bool { - let redis::Value::Array(entries) = info else { - return false; - }; - let attributes = entries - .as_chunks::<2>() - .0 - .iter() - .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes")) - .map(|pair| &pair[1]); - let Some(redis::Value::Array(attributes)) = attributes else { - return false; - }; - let fields = attributes.iter().filter_map(attribute).collect::>(); - let has_field = |name: &str, field_type: &str| { - fields.iter().any(|field| { - field.name.as_deref() == Some(name) && field.field_type.as_deref() == Some(field_type) - }) - }; - has_field("prompt", "TEXT") - && has_field("response", "TEXT") - && has_field("inserted_at", "NUMERIC") - && has_field("updated_at", "NUMERIC") - && has_field(CACHE_KEY_FIELD, "TAG") - && fields.iter().any(|field| { - field.name.as_deref() == Some(VECTOR_FIELD) - && field.field_type.as_deref() == Some("VECTOR") - && field.dim == Some(dims as f64) - && field - .data_type - .as_deref() - .is_some_and(|data| data.eq_ignore_ascii_case("float32")) - && field - .distance_metric - .as_deref() - .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine")) - }) -} diff --git a/litellm-rust/crates/cache-redis-semantic/src/lib.rs b/litellm-rust/crates/cache-redis-semantic/src/lib.rs deleted file mode 100644 index 251323df6cf..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod cache; -mod config; -mod index; -mod reply; - -pub use cache::RedisSemanticCache; -pub use config::{DEFAULT_INDEX_NAME, RedisSemanticConfig}; diff --git a/litellm-rust/crates/cache-redis-semantic/src/reply.rs b/litellm-rust/crates/cache-redis-semantic/src/reply.rs deleted file mode 100644 index 24cbd573cde..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/reply.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub(crate) fn string_value(value: &redis::Value) -> Option { - match value { - redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), - redis::Value::SimpleString(text) => Some(text.clone()), - redis::Value::VerbatimString { text, .. } => Some(text.clone()), - _ => None, - } -} - -pub(crate) fn number_value(value: &redis::Value) -> Option { - match value { - redis::Value::Int(number) => Some(*number as f64), - redis::Value::Double(number) => Some(*number), - _ => string_value(value).and_then(|text| text.parse().ok()), - } -} - -pub(crate) fn first_document(result: &redis::Value) -> Option<&[redis::Value]> { - let redis::Value::Array(items) = result else { - return None; - }; - let [count, _document_id, fields, ..] = items.as_slice() else { - return None; - }; - if !matches!(count, redis::Value::Int(count) if *count > 0) { - return None; - } - match fields { - redis::Value::Array(fields) => Some(fields.as_slice()), - _ => None, - } -} - -fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> { - fields - .as_chunks::<2>() - .0 - .iter() - .find(|pair| string_value(&pair[0]).as_deref() == Some(name)) - .map(|pair| &pair[1]) -} - -pub(crate) fn string_field(fields: &[redis::Value], name: &str) -> Option { - field_value(fields, name).and_then(string_value) -} - -pub(crate) fn number_field(fields: &[redis::Value], name: &str) -> Option { - field_value(fields, name).and_then(number_value) -} - -pub(crate) fn bytes_field(fields: &[redis::Value], name: &str) -> Option> { - match field_value(fields, name)? { - redis::Value::BulkString(bytes) => Some(bytes.clone()), - redis::Value::SimpleString(text) => Some(text.clone().into_bytes()), - _ => None, - } -} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs deleted file mode 100644 index cc42f4c3924..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ /dev/null @@ -1,778 +0,0 @@ -mod support; - -use std::time::Duration; - -use litellm_cache::{ - BaseCache, Error, JsonCodec, SemanticCacheContext, - semantic::{SemanticCache, SemanticLookup}, -}; -use litellm_cache_redis_semantic::{DEFAULT_INDEX_NAME, RedisSemanticCache, RedisSemanticConfig}; -use redis_test::{MockCmd, MockRedisConnection}; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use sha2::{Digest, Sha256}; -use support::FakeEmbedder; - -const INDEX: &str = DEFAULT_INDEX_NAME; -const PROMPT: &str = "hello prompt"; -const CLOCK: fn() -> f64 = || 1700000000.5; -const VECTOR: [f32; 3] = [0.1, 0.2, 0.3]; - -type MockCache = RedisSemanticCache, MockRedisConnection>; - -#[fixture] -fn config() -> RedisSemanticConfig { - RedisSemanticConfig { - index_name: INDEX.into(), - similarity_threshold: 0.9, - } -} - -#[fixture] -fn entry() -> Value { - json!({"timestamp": 1.0, "response": {"answer": "yes"}}) -} - -#[fixture] -fn context() -> SemanticCacheContext { - messages_context(vec![json!({"role": "user", "content": PROMPT})]) -} - -fn messages_context(messages: Vec) -> SemanticCacheContext { - SemanticCacheContext { - messages: Some(Value::Array(messages)), - ..Default::default() - } -} - -fn cache(commands: Vec, embedder: FakeEmbedder) -> MockCache { - RedisSemanticCache::with_connection( - MockRedisConnection::new(commands).assert_all_commands_consumed(), - embedder, - JsonCodec::new(), - config(), - ) - .with_clock(CLOCK) -} - -fn encoded(value: &Value) -> Vec { - serde_json::to_vec(value).unwrap() -} - -fn vector_bytes(vector: &[f32]) -> Vec { - vector - .iter() - .flat_map(|component| component.to_le_bytes()) - .collect() -} - -fn entry_id(prompt: &str, tag: &str) -> String { - let mut digest = Sha256::new(); - digest.update(prompt.as_bytes()); - digest.update(b"litellm_cache_key"); - digest.update(tag.as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn s(value: &str) -> redis::Value { - redis::Value::BulkString(value.as_bytes().to_vec()) -} - -fn unknown_index_error() -> redis::RedisError { - support::server_error("Unknown index name") -} - -fn info_missing(index: &str) -> MockCmd { - MockCmd::new( - redis::cmd("FT.INFO").arg(index), - Err::(unknown_index_error()), - ) -} - -fn info(index: &str, value: redis::Value) -> MockCmd { - MockCmd::new(redis::cmd("FT.INFO").arg(index), Ok(value)) -} - -fn attribute(name: &str, field_type: &str, extra: Vec) -> redis::Value { - let mut parts = vec![ - s("identifier"), - s(name), - s("attribute"), - s(name), - s("type"), - s(field_type), - ]; - parts.extend(extra); - redis::Value::Array(parts) -} - -fn index_info(attributes: Vec) -> redis::Value { - redis::Value::Array(vec![ - s("index_name"), - s(INDEX), - s("attributes"), - redis::Value::Array(attributes), - ]) -} - -fn vector_attribute_with(dims: i64, data_type: &str, distance_metric: &str) -> redis::Value { - attribute( - "prompt_vector", - "VECTOR", - vec![ - s("algorithm"), - s("FLAT"), - s("data_type"), - s(data_type), - s("dim"), - redis::Value::Int(dims), - s("distance_metric"), - s(distance_metric), - ], - ) -} - -fn info_with_vector(vector: redis::Value) -> redis::Value { - index_info(vec![ - attribute("prompt", "TEXT", vec![]), - attribute("response", "TEXT", vec![]), - attribute("inserted_at", "NUMERIC", vec![]), - attribute("updated_at", "NUMERIC", vec![]), - vector, - attribute("litellm_cache_key", "TAG", vec![]), - ]) -} - -fn compatible_info(dims: i64) -> redis::Value { - info_with_vector(vector_attribute_with(dims, "FLOAT32", "COSINE")) -} - -fn unscoped_info(dims: i64) -> redis::Value { - index_info(vec![ - attribute("prompt", "TEXT", vec![]), - attribute("response", "TEXT", vec![]), - attribute("inserted_at", "NUMERIC", vec![]), - attribute("updated_at", "NUMERIC", vec![]), - vector_attribute_with(dims, "FLOAT32", "COSINE"), - ]) -} - -fn create_index(name: &str, dims: usize) -> MockCmd { - MockCmd::new(create_index_command(name, dims), Ok("OK")) -} - -fn create_index_command(name: &str, dims: usize) -> redis::Cmd { - let mut command = redis::cmd("FT.CREATE"); - command - .arg(name) - .arg("ON") - .arg("HASH") - .arg("PREFIX") - .arg(1) - .arg(name) - .arg("SCORE") - .arg(1.0) - .arg("SCHEMA") - .arg("prompt") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("response") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("inserted_at") - .arg("NUMERIC") - .arg("updated_at") - .arg("NUMERIC") - .arg("prompt_vector") - .arg("VECTOR") - .arg("FLAT") - .arg(6) - .arg("TYPE") - .arg("FLOAT32") - .arg("DIM") - .arg(dims) - .arg("DISTANCE_METRIC") - .arg("COSINE") - .arg("litellm_cache_key") - .arg("TAG") - .arg("SEPARATOR") - .arg(","); - command -} - -fn hset(index: &str, prompt: &str, tag: &str, vector: &[f32], value: &Value) -> MockCmd { - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{index}:{}", entry_id(prompt, tag))) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(value)) - .arg("prompt_vector") - .arg(vector_bytes(vector)) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ) -} - -fn search( - index: &str, - tag: &str, - vector: &[f32], - reply: redis::RedisResult, -) -> MockCmd { - let mut command = redis::cmd("FT.SEARCH"); - command - .arg(index) - .arg(format!( - "(@litellm_cache_key:{{{tag}}})=>[KNN 1 @prompt_vector $vector AS vector_distance]" - )) - .arg("RETURN") - .arg(8) - .arg("entry_id") - .arg("prompt") - .arg("response") - .arg("inserted_at") - .arg("updated_at") - .arg("metadata") - .arg("litellm_cache_key") - .arg("vector_distance") - .arg("SORTBY") - .arg("vector_distance") - .arg("ASC") - .arg("DIALECT") - .arg(2) - .arg("LIMIT") - .arg(0) - .arg(1) - .arg("PARAMS") - .arg(2) - .arg("vector") - .arg(vector_bytes(vector)); - MockCmd::new(command, reply) -} - -fn hit(tag: &str, distance: &str, response: Vec) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::Int(1), - s("litellm_semantic_cache_index:stored-id"), - redis::Value::Array(vec![ - s("entry_id"), - s("stored-id"), - s("prompt"), - s(PROMPT), - s("response"), - redis::Value::BulkString(response), - s("inserted_at"), - s("1700000000.5"), - s("updated_at"), - s("1700000000.5"), - s("litellm_cache_key"), - s(tag), - s("vector_distance"), - s(distance), - ]), - ]) -} - -fn empty_result() -> redis::Value { - redis::Value::Array(vec![redis::Value::Int(0)]) -} - -#[rstest] -#[case::creates_index_and_expires(false, Some(Duration::from_secs(5)), Some(5))] -#[case::existing_index_without_ttl(true, None, None)] -#[case::fractional_ttl_rounds_up(true, Some(Duration::from_millis(1500)), Some(2))] -fn store_writes_the_redisvl_hash( - #[case] index_exists: bool, - #[case] ttl: Option, - #[case] expire: Option, - entry: Value, - context: SemanticCacheContext, -) { - let hash_key = format!("{INDEX}:{}", entry_id(PROMPT, "key1")); - let mut commands = if index_exists { - vec![info(INDEX, compatible_info(3))] - } else { - vec![info_missing(INDEX), create_index(INDEX, 3)] - }; - commands.push(hset(INDEX, PROMPT, "key1", &VECTOR, &entry)); - commands.extend( - expire.map(|seconds| MockCmd::new(redis::cmd("EXPIRE").arg(&hash_key).arg(seconds), Ok(1))), - ); - let cache = cache(commands, FakeEmbedder::new(&[])); - - cache - .set_cache("key1", entry, &SemanticCacheContext { ttl, ..context }) - .unwrap(); -} - -#[rstest] -#[case::below_distance_threshold("key1", "0.05", None, Ok(Some(entry())))] -#[case::above_distance_threshold("key1", "0.5", None, Ok(None))] -#[case::other_cache_key("other", "0.05", None, Ok(None))] -#[case::malformed_response("key1", "0.05", Some(b"not json!".as_slice()), Err(Error::InvalidEntry(None)))] -fn lookup_applies_threshold_scope_and_codec( - #[case] stored_tag: &str, - #[case] distance: &str, - #[case] response: Option<&[u8]>, - #[case] expected: Result, Error>, - entry: Value, - context: SemanticCacheContext, -) { - let response = response.map_or_else(|| encoded(&entry), <[u8]>::to_vec); - let cache = cache( - vec![ - info(INDEX, compatible_info(3)), - search( - INDEX, - "key1", - &VECTOR, - Ok(hit(stored_tag, distance, response)), - ), - ], - FakeEmbedder::new(&[]), - ); - - let kind = |result: Result, Error>| { - result.map_err(|error| std::mem::discriminant(&error)) - }; - assert_eq!(kind(cache.get_cache("key1", &context)), kind(expected)); -} - -#[rstest] -#[case::hit(context(), Some(hit("key1", "0.05", encoded(&entry()))), Some(entry()), Some(1.0 - 0.05))] -#[case::beyond_distance_threshold(context(), Some(hit("key1", "0.5", encoded(&entry()))), None, Some(0.0))] -#[case::no_results(context(), Some(empty_result()), None, Some(0.0))] -#[case::other_cache_key(context(), Some(hit("other", "0.05", encoded(&entry()))), None, Some(0.0))] -#[case::no_prompt(SemanticCacheContext::default(), None, None, Some(0.0))] -#[tokio::test] -async fn lookup_reports_python_semantic_similarity( - #[case] context: SemanticCacheContext, - #[case] reply: Option, - #[case] value: Option, - #[case] similarity: Option, - #[values(false, true)] use_async: bool, -) { - let commands = reply.map_or_else(Vec::new, |reply| { - vec![ - info(INDEX, compatible_info(3)), - search(INDEX, "key1", &VECTOR, Ok(reply)), - ] - }); - let cache = cache(commands, FakeEmbedder::new(&[])); - - let lookup = if use_async { - cache - .async_get_cache_with_similarity("key1", &context) - .await - } else { - cache.get_cache_with_similarity("key1", &context) - }; - - assert_eq!(lookup, Ok(SemanticLookup { value, similarity })); -} - -#[rstest] -#[tokio::test] -async fn missing_prompt_is_a_noop_that_never_embeds(entry: Value) { - let embedder = FakeEmbedder::new(&[]); - let calls = embedder.calls.clone(); - let cache = cache(Vec::new(), embedder); - let context = SemanticCacheContext::default(); - - cache.set_cache("key1", entry.clone(), &context).unwrap(); - assert_eq!(cache.get_cache("key1", &context).unwrap(), None); - cache - .async_set_cache("key1", entry, context.clone()) - .await - .unwrap(); - assert_eq!(cache.async_get_cache("key1", &context).await.unwrap(), None); - assert!(calls.lock().unwrap().is_empty()); -} - -#[rstest] -fn scope_overrides_key_as_filter_tag(entry: Value, context: SemanticCacheContext) { - let cache = cache( - vec![ - info(INDEX, compatible_info(3)), - hset(INDEX, PROMPT, "scope-a", &VECTOR, &entry), - search( - INDEX, - "scope\\-a", - &VECTOR, - Ok(hit("scope-a", "0.05", encoded(&entry))), - ), - ], - FakeEmbedder::new(&[]), - ); - let context = SemanticCacheContext { - scope: Some("scope-a".into()), - ..context - }; - - cache.set_cache("key1", entry.clone(), &context).unwrap(); - assert_eq!(cache.get_cache("key1", &context).unwrap(), Some(entry)); -} - -#[rstest] -#[case::unscoped_schema(unscoped_info(3))] -#[case::wrong_distance_metric(info_with_vector(vector_attribute_with(3, "FLOAT32", "L2")))] -#[case::wrong_data_type(info_with_vector(vector_attribute_with(3, "FLOAT64", "COSINE")))] -fn incompatible_schema_falls_back_to_isolated_index( - #[case] base_info: redis::Value, - entry: Value, - context: SemanticCacheContext, -) { - let isolated = format!("{INDEX}_isolated"); - let cache = cache( - vec![ - info(INDEX, base_info), - info_missing(&isolated), - create_index(&isolated, 3), - hset(&isolated, PROMPT, "key1", &VECTOR, &entry), - ], - FakeEmbedder::new(&[]), - ); - - cache.set_cache("key1", entry, &context).unwrap(); -} - -#[rstest] -fn create_index_race_rechecks_schema_and_stores(entry: Value, context: SemanticCacheContext) { - let cache = cache( - vec![ - info_missing(INDEX), - MockCmd::new( - create_index_command(INDEX, 3), - Err::<&str, _>(support::server_error("Index already exists")), - ), - info(INDEX, compatible_info(3)), - hset(INDEX, PROMPT, "key1", &VECTOR, &entry), - ], - FakeEmbedder::new(&[]), - ); - - cache.set_cache("key1", entry, &context).unwrap(); -} - -#[rstest] -#[case::punctuation_and_spaces("a:b, c|d", "a\\:b\\,\\ c\\|d")] -#[case::braces_and_dots("{x}.y", "\\{x\\}\\.y")] -#[case::plain("key1", "key1")] -fn tag_special_characters_are_escaped_in_search_filter( - #[case] tag: &str, - #[case] escaped: &str, - context: SemanticCacheContext, -) { - let cache = cache( - vec![ - info(INDEX, compatible_info(3)), - search(INDEX, escaped, &VECTOR, Ok(empty_result())), - ], - FakeEmbedder::new(&[]), - ); - - assert_eq!(cache.get_cache(tag, &context).unwrap(), None); -} - -#[rstest] -#[case::content_parts( - SemanticCacheContext { - messages: Some(json!([ - {"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, - {"role": "assistant", "content": "reply"}, - ])), - ..Default::default() - }, - "firstsecondreply" -)] -#[case::responses_string_input( - SemanticCacheContext { input: Some(json!(" plain input ")), ..Default::default() }, - "plain input" -)] -#[case::responses_nested_input( - SemanticCacheContext { - input: Some(json!([{"content": [{"type": "input_text", "text": "nested"}]}, "tail"])), - ..Default::default() - }, - "nested\ntail" -)] -#[case::responses_output_text( - SemanticCacheContext { input: Some(json!({"output_text": " result text "})), ..Default::default() }, - "result text" -)] -#[case::search_results( - messages_context(vec![json!({ - "role": "user", - "content": "question", - "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], - })]), - "questionsrctfound{\"a\":1}" -)] -#[case::empty_messages_fall_back_to_input( - SemanticCacheContext { messages: Some(json!([])), input: Some(json!("fallback")), ..Default::default() }, - "fallback" -)] -fn prompt_extraction_matches_python_message_and_input_shapes( - #[case] context: SemanticCacheContext, - #[case] prompt: &str, -) { - let embedder = FakeEmbedder::new(&[]); - let calls = embedder.calls.clone(); - let cache = cache( - vec![ - info(INDEX, compatible_info(3)), - search(INDEX, "key1", &VECTOR, Ok(empty_result())), - ], - embedder, - ); - - cache.get_cache("key1", &context).unwrap(); - - assert_eq!(*calls.lock().unwrap(), vec![(prompt.to_owned(), None)]); -} - -#[rstest] -#[case(None)] -#[case(Some(Duration::from_secs(9)))] -fn ttl_passes_through_context_only(#[case] ttl: Option) { - let cache = cache(Vec::new(), FakeEmbedder::new(&[])); - - assert_eq!( - cache.get_ttl(&SemanticCacheContext { - ttl, - ..Default::default() - }), - ttl - ); -} - -#[rstest] -#[tokio::test] -async fn async_paths_embed_with_metadata_then_run_blocking_redis_work( - entry: Value, - context: SemanticCacheContext, -) { - let embedder = FakeEmbedder::new(&[]); - let calls = embedder.calls.clone(); - let cache = cache( - vec![ - info(INDEX, compatible_info(3)), - hset(INDEX, PROMPT, "key1", &VECTOR, &entry), - search( - INDEX, - "key1", - &VECTOR, - Ok(hit("key1", "0.05", encoded(&entry))), - ), - ], - embedder, - ); - let context = SemanticCacheContext { - metadata: Some(json!({"tenant": "team"})), - ..context - }; - - cache - .async_set_cache("key1", entry.clone(), context.clone()) - .await - .unwrap(); - assert_eq!( - cache.async_get_cache("key1", &context).await.unwrap(), - Some(entry) - ); - assert_eq!( - *calls.lock().unwrap(), - vec![(PROMPT.to_owned(), context.metadata.clone()); 2] - ); -} - -#[rstest] -fn accessors_report_the_config(config: RedisSemanticConfig) { - let cache = cache(Vec::new(), FakeEmbedder::new(&[])); - - assert_eq!(cache.index_name(), config.index_name); - assert!((cache.similarity_threshold() - config.similarity_threshold).abs() < 1e-6); -} - -#[rstest] -fn shared_base_index_across_dimensions_replaces_the_isolated_index(entry: Value) { - // Pins parity with Python's `_isolated` + overwrite=True flow. - let prompt = "shared prompt"; - let isolated = format!("{INDEX}_isolated"); - let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); - - let vector_a = vec![0.1f32; 8]; - let worker_a = cache( - vec![ - info_missing(INDEX), - create_index(INDEX, 8), - hset(INDEX, prompt, "key1", &vector_a, &entry), - ], - FakeEmbedder::new(&[(prompt, &vector_a)]), - ); - worker_a - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - - let vector_b = vec![0.2f32; 4]; - let worker_b = cache( - vec![ - info(INDEX, compatible_info(8)), - info_missing(&isolated), - create_index(&isolated, 4), - hset(&isolated, prompt, "key1", &vector_b, &entry), - search( - &isolated, - "key1", - &vector_b, - Ok(hit("key1", "0.0", encoded(&entry))), - ), - search( - &isolated, - "key1", - &vector_b, - Err(redis::RedisError::from(( - redis::ErrorKind::Extension, - "Vector dimension mismatch", - ))), - ), - ], - FakeEmbedder::new(&[(prompt, &vector_b)]), - ); - worker_b - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - assert_eq!( - worker_b.get_cache("key1", &context()).unwrap(), - Some(entry.clone()) - ); - - let vector_c = vec![0.3f32; 16]; - let worker_c = cache( - vec![ - info(INDEX, compatible_info(8)), - info(&isolated, compatible_info(4)), - MockCmd::new(redis::cmd("FT.DROPINDEX").arg(&isolated), Ok("OK")), - create_index(&isolated, 16), - hset(&isolated, prompt, "key1", &vector_c, &entry), - ], - FakeEmbedder::new(&[(prompt, &vector_c)]), - ); - worker_c - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - - assert!(matches!( - worker_b.get_cache("key1", &context()), - Err(Error::Unavailable(Some(_))) - )); -} - -#[fixture] -fn redis_stack_url() -> Option { - std::env::var("LITELLM_REDIS_STACK_URL").ok() -} - -fn live_cache( - url: &str, - index_name: &str, - prompt: &str, - vector: Vec, -) -> RedisSemanticCache> { - RedisSemanticCache::new( - url, - FakeEmbedder::new(&[(prompt, vector.as_slice())]), - JsonCodec::::new(), - RedisSemanticConfig { - index_name: index_name.to_owned(), - similarity_threshold: 0.9, - }, - ) - .unwrap() -} - -#[rstest] -fn live_shared_index_is_replaced_across_dimensions(redis_stack_url: Option, entry: Value) { - let Some(url) = redis_stack_url else { - return; - }; - // Pins parity with Python's `_isolated` + overwrite=True flow. - let base = format!("rust_semantic_shared_{}", std::process::id()); - let isolated = format!("{base}_isolated"); - let prompt = "shared live prompt"; - let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); - - let worker_a = live_cache(&url, &base, prompt, vec![0.1f32; 8]); - worker_a - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - - let worker_b = live_cache(&url, &base, prompt, vec![0.2f32; 4]); - worker_b - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - assert_eq!( - worker_b.get_cache("key1", &context()).unwrap(), - Some(entry.clone()) - ); - - let worker_c = live_cache(&url, &base, prompt, vec![0.3f32; 16]); - worker_c - .set_cache("key1", entry.clone(), &context()) - .unwrap(); - - assert!(matches!( - worker_b.get_cache("key1", &context()), - Err(Error::Unavailable(Some(_))) - )); - - let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap(); - for index in [&base, &isolated] { - let _: Result<(), _> = redis::cmd("FT.DROPINDEX") - .arg(index) - .arg("DD") - .query(&mut connection); - } -} - -#[rstest] -fn live_store_lookup_and_ttl_against_redis_stack(redis_stack_url: Option, entry: Value) { - let Some(url) = redis_stack_url else { - return; - }; - let prompt = "rust semantic cache live prompt"; - let index_name = format!("rust_semantic_test_{}", std::process::id()); - let cache = live_cache(&url, &index_name, prompt, vec![0.1, 0.2, 0.3, 0.4]); - let context = SemanticCacheContext { - ttl: Some(Duration::from_secs(120)), - ..messages_context(vec![json!({"role": "user", "content": prompt})]) - }; - - cache - .set_cache("live-key", entry.clone(), &context) - .unwrap(); - assert_eq!(cache.get_cache("live-key", &context).unwrap(), Some(entry)); - assert_eq!(cache.get_cache("other-key", &context).unwrap(), None); - - let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap(); - let ttl: i64 = redis::Commands::ttl( - &mut connection, - format!("{index_name}:{}", entry_id(prompt, "live-key")), - ) - .unwrap(); - assert!( - ttl > 0, - "expected stored hash to carry an expiry, got {ttl}" - ); -} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/contract.rs b/litellm-rust/crates/cache-redis-semantic/tests/contract.rs deleted file mode 100644 index fb3cda9616e..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/tests/contract.rs +++ /dev/null @@ -1,63 +0,0 @@ -mod support; - -use litellm_cache::{JsonCodec, SemanticCacheContext, semantic::PreparedEmbedding}; -use litellm_cache_redis_semantic::{DEFAULT_INDEX_NAME, RedisSemanticCache, RedisSemanticConfig}; -use litellm_cache_testing as contract; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use support::FakeSearch; - -type Cache = RedisSemanticCache, FakeSearch>; - -const PREFIX: &str = "contract:"; - -#[fixture] -fn cache() -> Cache { - RedisSemanticCache::with_connection( - FakeSearch::default(), - PreparedEmbedding(vec![0.6, 0.8]), - JsonCodec::new(), - RedisSemanticConfig { - index_name: DEFAULT_INDEX_NAME.into(), - similarity_threshold: 0.9, - }, - ) -} - -#[fixture] -fn context() -> SemanticCacheContext { - SemanticCacheContext { - messages: Some(json!([{"role": "user", "content": "contract prompt"}])), - ..Default::default() - } -} - -#[rstest] -#[tokio::test] -async fn hit_and_miss(cache: Cache, context: SemanticCacheContext) { - contract::hit_and_miss(&cache, context, PREFIX, json!({"answer": 42})).await; -} - -#[rstest] -#[tokio::test] -async fn sync_async_equivalence(cache: Cache, context: SemanticCacheContext) { - contract::sync_async_equivalence(&cache, context, PREFIX, json!("first"), json!([2])).await; -} - -#[rstest] -#[tokio::test] -async fn overwrite_replaces(cache: Cache, context: SemanticCacheContext) { - contract::overwrite_replaces(&cache, context, PREFIX, json!(1), json!({"b": 2})).await; -} - -#[rstest] -#[tokio::test] -async fn pipeline_writes_every_entry(cache: Cache, context: SemanticCacheContext) { - contract::pipeline_writes_every_entry( - &cache, - context, - PREFIX, - vec![json!("a"), json!(2), json!({"c": true})], - ) - .await; -} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs deleted file mode 100644 index f112dbe96a7..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs +++ /dev/null @@ -1,302 +0,0 @@ -#![allow(dead_code)] - -use std::{ - collections::{BTreeMap, HashMap}, - sync::{Arc, Mutex}, -}; - -use litellm_cache::{Error, semantic::Embedder}; -use serde_json::Value; - -pub type EmbedCalls = Arc)>>>; - -/// Embeds known prompts to fixed vectors, anything else to `[0.1, 0.2, 0.3]`, and records every -/// prompt with its metadata. -pub struct FakeEmbedder { - vectors: HashMap>, - pub calls: EmbedCalls, -} - -impl FakeEmbedder { - pub fn new(vectors: &[(&str, &[f32])]) -> Self { - Self { - vectors: vectors - .iter() - .map(|(prompt, vector)| ((*prompt).to_owned(), vector.to_vec())) - .collect(), - calls: EmbedCalls::default(), - } - } -} - -impl Embedder for FakeEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - self.calls - .lock() - .unwrap() - .push((prompt.to_owned(), metadata.cloned())); - Ok(self - .vectors - .get(prompt) - .cloned() - .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) - } - - async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - self.embed(prompt, metadata) - } -} - -struct FakeIndex { - prefix: Vec, - dims: usize, - vector_field: String, -} - -#[derive(Default)] -struct SearchState { - indexes: HashMap, - hashes: BTreeMap, BTreeMap>>, -} - -/// An in-memory Redis Stack speaking the `FT.*`, `HSET` and `EXPIRE` subset the semantic cache -/// sends, with exact cosine KNN over the hashes under an index prefix. -#[derive(Clone, Default)] -pub struct FakeSearch { - state: Arc>, -} - -impl FakeSearch { - fn run(&self, args: Vec>) -> redis::RedisResult { - let mut state = self.state.lock().unwrap(); - let text = |index: usize| String::from_utf8_lossy(&args[index]).into_owned(); - match text(0).to_uppercase().as_str() { - "FT.CREATE" => { - let name = text(1); - if state.indexes.contains_key(&name) { - return Err(server_error("Index already exists")); - } - let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); - let prefix = args[position("PREFIX").unwrap() + 2].clone(); - let dims = text(position("DIM").unwrap() + 1).parse().unwrap(); - let vector_field = text(position("VECTOR").unwrap() - 1); - state.indexes.insert( - name, - FakeIndex { - prefix, - dims, - vector_field, - }, - ); - Ok(redis::Value::Okay) - } - "FT.INFO" => { - let index = state - .indexes - .get(&text(1)) - .ok_or_else(|| server_error("Unknown index name"))?; - Ok(index_info(index)) - } - "FT.DROPINDEX" => { - state.indexes.remove(&text(1)); - Ok(redis::Value::Okay) - } - "HSET" => { - let hash = state.hashes.entry(args[1].clone()).or_default(); - for pair in args[2..].chunks(2) { - hash.insert( - String::from_utf8_lossy(&pair[0]).into_owned(), - pair[1].clone(), - ); - } - Ok(redis::Value::Int(((args.len() - 2) / 2) as i64)) - } - "EXPIRE" => Ok(redis::Value::Int(i64::from( - state.hashes.contains_key(&args[1]), - ))), - "FT.SEARCH" => { - let index = state - .indexes - .get(&text(1)) - .ok_or_else(|| server_error("no such index"))?; - let query = text(2); - let tag = query_tag(&query); - let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); - let vector = floats(&args[params + 3]); - let best = state - .hashes - .iter() - .filter(|(key, _)| key.starts_with(&index.prefix)) - .filter(|(_, fields)| { - fields.get("litellm_cache_key").map(Vec::as_slice) == Some(tag.as_bytes()) - }) - .filter_map(|(key, fields)| { - let stored = floats(fields.get(&index.vector_field)?); - (stored.len() == index.dims) - .then(|| (key, fields, 1.0 - cosine(&vector, &stored))) - }) - .min_by(|left, right| left.2.total_cmp(&right.2)); - let Some((key, fields, distance)) = best else { - return Ok(redis::Value::Array(vec![redis::Value::Int(0)])); - }; - let mut reply = fields - .iter() - .filter(|(name, _)| **name != index.vector_field) - .flat_map(|(name, value)| [bulk(name.as_bytes()), bulk(value)]) - .collect::>(); - reply.extend([ - bulk(b"vector_distance"), - bulk(distance.to_string().as_bytes()), - ]); - Ok(redis::Value::Array(vec![ - redis::Value::Int(1), - bulk(key), - redis::Value::Array(reply), - ])) - } - "PING" => Ok(redis::Value::SimpleString("PONG".into())), - _ => Err(server_error("unsupported command")), - } - } -} - -impl redis::ConnectionLike for FakeSearch { - fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { - let mut commands = parse_commands(command); - self.run(commands.remove(0)) - } - - fn req_packed_commands( - &mut self, - commands: &[u8], - offset: usize, - count: usize, - ) -> redis::RedisResult> { - let replies = parse_commands(commands) - .into_iter() - .map(|args| self.run(args)) - .collect::>>()?; - Ok(replies.into_iter().skip(offset).take(count).collect()) - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } -} - -pub fn server_error(reply: &str) -> redis::RedisError { - redis::parse_redis_value(format!("-{reply}\r\n").as_bytes()) - .expect("error reply") - .extract_error() - .expect_err("server error") -} - -fn bulk(bytes: &[u8]) -> redis::Value { - redis::Value::BulkString(bytes.to_vec()) -} - -fn index_info(index: &FakeIndex) -> redis::Value { - let attribute = |name: &str, field_type: &str| { - redis::Value::Array(vec![ - bulk(b"identifier"), - bulk(name.as_bytes()), - bulk(b"type"), - bulk(field_type.as_bytes()), - ]) - }; - redis::Value::Array(vec![ - bulk(b"attributes"), - redis::Value::Array(vec![ - attribute("prompt", "TEXT"), - attribute("response", "TEXT"), - attribute("inserted_at", "NUMERIC"), - attribute("updated_at", "NUMERIC"), - attribute("litellm_cache_key", "TAG"), - redis::Value::Array(vec![ - bulk(b"identifier"), - bulk(index.vector_field.as_bytes()), - bulk(b"type"), - bulk(b"VECTOR"), - bulk(b"dim"), - redis::Value::Int(index.dims as i64), - bulk(b"data_type"), - bulk(b"FLOAT32"), - bulk(b"distance_metric"), - bulk(b"COSINE"), - ]), - ]), - ]) -} - -/// The tag inside `@litellm_cache_key:{...}`, with query escapes removed. -fn query_tag(query: &str) -> String { - let start = query.find("@litellm_cache_key:{").unwrap() + "@litellm_cache_key:{".len(); - let mut tag = String::new(); - let mut characters = query[start..].chars(); - while let Some(character) = characters.next() { - match character { - '\\' => tag.extend(characters.next()), - '}' => break, - character => tag.push(character), - } - } - tag -} - -fn floats(bytes: &[u8]) -> Vec { - bytes - .as_chunks::<4>() - .0 - .iter() - .map(|chunk| f32::from_le_bytes(*chunk)) - .collect() -} - -fn cosine(left: &[f32], right: &[f32]) -> f64 { - let dot = left - .iter() - .zip(right) - .map(|(left, right)| f64::from(*left) * f64::from(*right)) - .sum::(); - let norm = |vector: &[f32]| { - vector - .iter() - .map(|value| f64::from(*value).powi(2)) - .sum::() - .sqrt() - }; - dot / (norm(left) * norm(right)) -} - -/// Splits a packed RESP request into each command's arguments. -fn parse_commands(mut bytes: &[u8]) -> Vec>> { - let line = |bytes: &mut &[u8]| { - let end = bytes - .windows(2) - .position(|window| window == b"\r\n") - .unwrap(); - let text = String::from_utf8(bytes[1..end].to_vec()).unwrap(); - *bytes = &bytes[end + 2..]; - text.parse::().unwrap() - }; - let mut commands = Vec::new(); - while !bytes.is_empty() { - let count = line(&mut bytes); - let mut args = Vec::with_capacity(count); - for _ in 0..count { - let length = line(&mut bytes); - args.push(bytes[..length].to_vec()); - bytes = &bytes[length + 2..]; - } - commands.push(args); - } - commands -} diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml deleted file mode 100644 index f7621299506..00000000000 --- a/litellm-rust/crates/cache-redis/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "litellm-cache-redis" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -litellm-cache.workspace = true -litellm-error.workspace = true -redis = { workspace = true, features = ["cluster", "tls-rustls"] } -r2d2 = "0.8.10" -thiserror.workspace = true -tokio.workspace = true - -[dev-dependencies] -litellm-cache-testing.workspace = true -redis-test.workspace = true -rstest.workspace = true -serde_json.workspace = true diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs deleted file mode 100644 index b873a2c0282..00000000000 --- a/litellm-rust/crates/cache-redis/src/cache.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::{ - sync::{Arc, OnceLock}, - time::Duration, -}; - -use litellm_cache::{BatchEntry, CacheCodec, Error}; - -use crate::{ - connection::{ConnectionRef, Connections}, - topology::RedisTopology, -}; - -const DEFAULT_TTL: Duration = Duration::from_secs(600); - -pub struct RedisCache { - pub(crate) connections: Arc>, - pub(crate) default_ttl: Duration, - pub(crate) codec: S, - pub(crate) namespace: Option, - pub(crate) topology: RedisTopology, - /// The server's major version, read from `INFO` once, like Python's `redis_version`. - pub(crate) major_version: Arc>, -} - -impl RedisCache { - pub fn new(url: &str, default_ttl: Option, codec: S) -> Result { - Self::connect(url, &RedisTopology::Standalone, default_ttl, codec) - } - - pub fn connect( - url: &str, - topology: &RedisTopology, - default_ttl: Option, - codec: S, - ) -> Result { - let connections = Connections::open(url, topology)?; - Ok(Self { - connections: Arc::new(connections), - default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), - codec, - namespace: None, - topology: topology.clone(), - major_version: Arc::default(), - }) - } -} - -impl RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn with_connection(connection: C, default_ttl: Option, codec: S) -> Self { - Self { - connections: Arc::new(Connections::fixed(connection)), - default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), - codec, - namespace: None, - topology: RedisTopology::Standalone, - major_version: Arc::default(), - } - } - - pub fn with_namespace(self, namespace: Option) -> Self { - Self { - namespace: namespace.filter(|value| !value.is_empty()), - ..self - } - } - - pub fn namespace(&self) -> Option<&str> { - self.namespace.as_deref() - } - - pub fn topology(&self) -> &RedisTopology { - &self.topology - } - - pub(crate) fn namespaced_key(&self, key: &str) -> String { - namespaced_key(self.namespace.as_deref(), key) - } - - pub(crate) fn namespaced_keys(&self, keys: &[String]) -> Vec { - keys.iter().map(|key| self.namespaced_key(key)).collect() - } - - /// Whole seconds for `ttl`, falling back to the default TTL like Python's `get_ttl`. - pub(crate) fn ttl_or_default(&self, ttl: Option) -> u64 { - ttl_seconds(ttl.unwrap_or(self.default_ttl)) - } - - pub(crate) fn execute( - &self, - operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, - ) -> Result { - self.connections.execute(operation) - } - - /// `_parse_redis_major_version`: the major version from `INFO`, or - /// `DEFAULT_REDIS_MAJOR_VERSION` when `INFO` fails or its version does not parse. The first - /// answer is kept, as Python reads `redis_version` once at construction. - pub(crate) async fn major_version(&self) -> u32 { - if let Some(version) = self.major_version.get() { - return *version; - } - let info = self - .run(|connection| connection.node_text(&redis::cmd("INFO"))) - .await; - let version = info - .ok() - .and_then(|info| parse_major_version(&info)) - .unwrap_or_else(default_major_version); - *self.major_version.get_or_init(|| version) - } - - pub(crate) async fn run(&self, operation: F) -> Result - where - T: Send + 'static, - F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, - { - Connections::run_blocking(Arc::clone(&self.connections), operation).await - } - - pub(crate) fn namespaced_pattern(&self) -> Result { - let namespace = self.namespace.as_ref().ok_or(Error::UnscopedFlush)?; - let escaped: String = namespace - .chars() - .flat_map(|ch| { - if matches!(ch, '*' | '?' | '[' | ']' | '\\') { - vec!['\\', ch] - } else { - vec![ch] - } - }) - .collect(); - Ok(format!("{escaped}:*")) - } - - pub(crate) fn decode_response(&self, value: redis::Value) -> Result, Error> { - match value { - redis::Value::Nil => Ok(None), - redis::Value::BulkString(bytes) => self.codec.decode(&bytes).map(Some), - redis::Value::SimpleString(text) => self.codec.decode(text.as_bytes()).map(Some), - _ => Err(Error::InvalidEntry(None)), - } - } - - pub(crate) fn decode_batch_response( - &self, - value: redis::Value, - ) -> Result, Error> { - match self.decode_response(value) { - Ok(Some(value)) => Ok(BatchEntry::Hit(value)), - Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), - Err(error) => Err(error), - } - } -} - -pub(crate) fn namespaced_key(namespace: Option<&str>, key: &str) -> String { - match namespace { - Some(namespace) if !key.starts_with(&format!("{namespace}:")) => { - format!("{namespace}:{key}") - } - _ => key.into(), - } -} - -pub(crate) fn ttl_seconds(ttl: Duration) -> u64 { - ttl.as_secs() - .saturating_add(u64::from(ttl.subsec_nanos() > 0)) - .max(1) -} - -fn parse_major_version(info: &str) -> Option { - let version = info - .lines() - .find_map(|line| line.trim().strip_prefix("redis_version:"))? - .trim(); - match version.split_once('.') { - Some((major, _)) => major.parse().ok(), - None => version.parse::().ok().map(|major| major as u32), - } -} - -fn default_major_version() -> u32 { - std::env::var("DEFAULT_REDIS_MAJOR_VERSION") - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(7) -} diff --git a/litellm-rust/crates/cache-redis/src/claim.rs b/litellm-rust/crates/cache-redis/src/claim.rs deleted file mode 100644 index c57a1b03395..00000000000 --- a/litellm-rust/crates/cache-redis/src/claim.rs +++ /dev/null @@ -1,109 +0,0 @@ -use litellm_cache::{CacheCodec, ClaimCache, Error, ExactCacheContext}; -use redis::Commands; - -use crate::{cache::RedisCache, connection::ConnectionRef}; - -const CLAIM_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ", - "elseif current ~= ARGV[1] then return 0; end; ", - "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ", - "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1" -); -const CLAIM_ATTEMPTS: usize = 8; - -fn stored_bytes(value: redis::Value) -> Result>, Error> { - match value { - redis::Value::Nil => Ok(None), - redis::Value::BulkString(bytes) => Ok(Some(bytes)), - redis::Value::SimpleString(text) => Ok(Some(text.into_bytes())), - _ => Err(Error::InvalidEntry(None)), - } -} - -/// Eligibility is decided on decoded values, so a pin written by another encoder (Python's -/// `json.dumps` spacing or key order) still matches. The write is a compare-and-set on the -/// bytes that decision was made on, retried when another claimant wins the race. -fn claim( - connection: &mut ConnectionRef<'_>, - codec: &S, - key: &str, - candidate: S::Value, - eligible: &[S::Value], - ttl: u64, -) -> Result -where - S::Value: PartialEq, -{ - let payload = codec.encode(&candidate)?; - if payload.is_empty() { - return Err(Error::InvalidEntry(None)); - } - for _ in 0..CLAIM_ATTEMPTS { - let current = stored_bytes( - connection - .get::<_, redis::Value>(key) - .map_err(Error::unavailable)?, - )? - .filter(|bytes| !bytes.is_empty()); - let existing = current - .as_deref() - .and_then(|bytes| codec.decode(bytes).ok()) - .filter(|existing| eligible.is_empty() || eligible.contains(existing)); - let refresh = existing - .as_ref() - .is_some_and(|existing| !eligible.is_empty() || *existing == candidate); - let write: &[u8] = if existing.is_some() { b"" } else { &payload }; - let applied = redis::cmd("EVAL") - .arg(CLAIM_SCRIPT) - .arg(1) - .arg(key) - .arg(current.as_deref().unwrap_or_default()) - .arg(ttl) - .arg(write) - .arg(u8::from(refresh)) - .query::(connection) - .map_err(Error::unavailable)?; - if applied { - return Ok(existing.unwrap_or(candidate)); - } - } - Err(Error::unavailable(ClaimContended)) -} - -#[derive(Debug, thiserror::Error)] -#[error("claim kept losing the race after {CLAIM_ATTEMPTS} attempts")] -struct ClaimContended; - -impl ClaimCache for RedisCache -where - S: CacheCodec + Clone + 'static, - S::Value: PartialEq, - C: redis::ConnectionLike + Send + 'static, -{ - fn claim_cache( - &self, - key: &str, - candidate: S::Value, - eligible: &[S::Value], - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(context.ttl); - self.execute(|connection| claim(connection, &self.codec, &key, candidate, eligible, ttl)) - } - - async fn async_claim_cache( - &self, - key: &str, - candidate: S::Value, - eligible: Vec, - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(context.ttl); - let codec = self.codec.clone(); - self.run(move |connection| claim(connection, &codec, &key, candidate, &eligible, ttl)) - .await - } -} diff --git a/litellm-rust/crates/cache-redis/src/connection.rs b/litellm-rust/crates/cache-redis/src/connection.rs deleted file mode 100644 index 99befa16ffa..00000000000 --- a/litellm-rust/crates/cache-redis/src/connection.rs +++ /dev/null @@ -1,513 +0,0 @@ -use std::{ - sync::{Arc, Mutex}, - time::Duration, -}; - -use litellm_cache::Error; -use redis::{ - ConnectionAddr, ConnectionInfo, IntoConnectionInfo, - cluster::{ - ClusterClient, ClusterClientBuilder, ClusterConnection, ClusterPipeline, NodeAddress, - }, - cluster_routing::{ - MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo, SingleNodeRoutingInfo, - }, -}; - -use crate::topology::{RedisNode, RedisTopology}; - -pub(crate) const REDIS_TIMEOUT: Duration = Duration::from_secs(5); -const REDIS_POOL_SIZE: u32 = 16; - -#[allow(private_interfaces)] -pub enum Connections { - Pool(r2d2::Pool), - Cluster(r2d2::Pool), - Fixed(Mutex), -} - -impl Connections -where - C: redis::ConnectionLike + Send + 'static, -{ - pub fn execute( - &self, - operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, - ) -> Result { - match self { - Self::Pool(pool) => { - let mut pooled = pool.get().map_err(Error::unavailable)?; - let result = operation(&mut ConnectionRef::Node(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable(_))); - result - } - Self::Cluster(pool) => { - let mut pooled = pool.get().map_err(Error::unavailable)?; - let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable(_))); - result - } - Self::Fixed(connection) => { - let mut connection = connection.lock()?; - operation(&mut ConnectionRef::Node(&mut *connection)) - } - } - } - - pub async fn run_blocking(connections: Arc, operation: F) -> Result - where - T: Send + 'static, - F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, - { - tokio::task::spawn_blocking(move || connections.execute(operation)) - .await - .map_err(Error::unavailable)? - } - - pub fn fixed(connection: C) -> Self { - Self::Fixed(Mutex::new(connection)) - } - - pub fn open(url: &str, topology: &RedisTopology) -> Result { - match topology { - RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)), - RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool( - ClusterConnectionManager::open(url, startup_nodes)?, - )?)), - } - } - - /// Closes every idle pooled connection; the next operation opens a fresh one. Connections - /// checked out right now return to the pool, and a caller-owned connection stays open. - pub fn disconnect(&self) { - match self { - Self::Pool(pool) => close_idle(pool), - Self::Cluster(pool) => close_idle(pool), - Self::Fixed(_) => {} - } - } -} - -fn pool(manager: M) -> Result, Error> { - r2d2::Pool::builder() - .max_size(REDIS_POOL_SIZE) - .min_idle(Some(0)) - .connection_timeout(REDIS_TIMEOUT) - .test_on_check_out(false) - .build(manager) - .map_err(Error::unavailable) -} - -fn close_idle(pool: &r2d2::Pool) -where - M: r2d2::ManageConnection>, -{ - let mut idle = Vec::new(); - while let Some(mut connection) = pool.try_get() { - connection.failed = true; - idle.push(connection); - } -} - -pub(crate) struct PooledConnection { - connection: C, - failed: bool, -} - -/// Pools connections without a checkout PING, which would double every operation's round trips. -/// A timed-out command leaves its reply on the socket while redis still reports the connection -/// open, so any connection whose operation failed is discarded instead of being reused. -pub(crate) struct ConnectionManager(redis::Client); - -impl ConnectionManager { - fn open(url: &str) -> Result { - redis::Client::open(url) - .map(Self) - .map_err(Error::unavailable) - } -} - -impl r2d2::ManageConnection for ConnectionManager { - type Connection = PooledConnection; - type Error = redis::RedisError; - - fn connect(&self) -> Result { - let connection = self.0.get_connection()?; - connection.set_read_timeout(Some(REDIS_TIMEOUT))?; - connection.set_write_timeout(Some(REDIS_TIMEOUT))?; - Ok(PooledConnection { - connection, - failed: false, - }) - } - - fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> { - redis::cmd("PING").query::(&mut connection.connection)?; - Ok(()) - } - - fn has_broken(&self, connection: &mut Self::Connection) -> bool { - connection.failed || !redis::ConnectionLike::is_open(&connection.connection) - } -} - -#[derive(Debug, thiserror::Error)] -enum TopologyFault { - #[error("Redis Cluster needs at least one startup node")] - NoStartupNodes, - #[error("Redis Cluster nodes need a TCP address")] - NonTcpAddress, - #[error("Redis Cluster answered a fan-out command without a per-node map")] - FanOutReply, - #[error("scan target does not match the connection topology")] - ScanTarget, -} - -pub(crate) struct ClusterConnectionManager(ClusterClient); - -impl ClusterConnectionManager { - fn open(url: &str, startup_nodes: &[RedisNode]) -> Result { - if startup_nodes.is_empty() { - return Err(Error::unavailable(TopologyFault::NoStartupNodes)); - } - let info = url.into_connection_info().map_err(Error::unavailable)?; - let nodes = startup_nodes - .iter() - .map(|node| node_info(&info, node)) - .collect::, _>>()?; - ClusterClientBuilder::new(nodes) - .connection_timeout(REDIS_TIMEOUT) - .response_timeout(REDIS_TIMEOUT) - .build() - .map(Self) - .map_err(Error::unavailable) - } -} - -fn node_info(info: &ConnectionInfo, node: &RedisNode) -> Result { - let addr = match info.addr() { - ConnectionAddr::Tcp(..) => ConnectionAddr::Tcp(node.host.clone(), node.port), - ConnectionAddr::TcpTls { - insecure, - tls_params, - .. - } => ConnectionAddr::TcpTls { - host: node.host.clone(), - port: node.port, - insecure: *insecure, - tls_params: tls_params.clone(), - }, - _ => return Err(Error::unavailable(TopologyFault::NonTcpAddress)), - }; - Ok(info.clone().set_addr(addr)) -} - -impl r2d2::ManageConnection for ClusterConnectionManager { - type Connection = PooledConnection; - type Error = redis::RedisError; - - fn connect(&self) -> Result { - let connection = self.0.get_connection()?; - connection.set_read_timeout(Some(REDIS_TIMEOUT))?; - connection.set_write_timeout(Some(REDIS_TIMEOUT))?; - Ok(PooledConnection { - connection, - failed: false, - }) - } - - fn is_valid(&self, connection: &mut Self::Connection) -> Result<(), redis::RedisError> { - redis::cmd("PING").query::(&mut connection.connection)?; - Ok(()) - } - - fn has_broken(&self, connection: &mut Self::Connection) -> bool { - connection.failed || !redis::ConnectionLike::is_open(&connection.connection) - } -} - -pub enum ConnectionRef<'a> { - Node(&'a mut dyn redis::ConnectionLike), - Cluster(&'a mut ClusterConnection), -} - -impl redis::ConnectionLike for ConnectionRef<'_> { - fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult { - match self { - Self::Node(connection) => connection.req_packed_command(cmd), - Self::Cluster(connection) => connection.req_packed_command(cmd), - } - } - - fn req_packed_commands( - &mut self, - cmd: &[u8], - offset: usize, - count: usize, - ) -> redis::RedisResult> { - match self { - Self::Node(connection) => connection.req_packed_commands(cmd, offset, count), - Self::Cluster(connection) => connection.req_packed_commands(cmd, offset, count), - } - } - - fn get_db(&self) -> i64 { - match self { - Self::Node(connection) => connection.get_db(), - Self::Cluster(connection) => redis::ConnectionLike::get_db(*connection), - } - } - - fn supports_pipelining(&self) -> bool { - match self { - Self::Node(connection) => connection.supports_pipelining(), - Self::Cluster(connection) => redis::ConnectionLike::supports_pipelining(*connection), - } - } - - fn check_connection(&mut self) -> bool { - match self { - Self::Node(connection) => connection.check_connection(), - Self::Cluster(connection) => connection.check_connection(), - } - } - - fn is_open(&self) -> bool { - match self { - Self::Node(connection) => connection.is_open(), - Self::Cluster(connection) => redis::ConnectionLike::is_open(*connection), - } - } -} - -impl ConnectionRef<'_> { - /// Runs `pipeline` and decodes its non-ignored replies as `T`. A cluster connection refuses - /// `Pipeline::query`, so there a transaction goes to its keys' slot as one MULTI/EXEC and - /// anything else is split per node by `ClusterPipeline`; either way the raw replies are - /// handed back to `pipeline` to decode. - pub(crate) fn query_pipeline( - &mut self, - pipeline: &redis::Pipeline, - ) -> Result { - match self { - Self::Node(connection) => pipeline.query(*connection), - Self::Cluster(connection) if pipeline.is_transaction() => { - redis::ConnectionLike::req_packed_commands( - *connection, - &pipeline.get_packed_pipeline(), - pipeline.len() + 1, - 1, - ) - .and_then(|replies| pipeline.query(&mut Replies(Some(replies)))) - } - Self::Cluster(connection) => { - let mut cluster = ClusterPipeline::with_capacity(pipeline.len()); - for command in pipeline.cmd_iter() { - cluster.add_command(command.clone()); - } - cluster - .query(connection) - .and_then(|replies| pipeline.query(&mut Replies(Some(replies)))) - } - } - .map_err(Error::unavailable) - } - - pub(crate) fn scan( - &mut self, - pattern: &str, - count: usize, - mut visit: impl FnMut(&mut Self, Vec) -> Result, - ) -> Result<(), Error> { - let pages = match self { - Self::Node(connection) => { - let page = scan_command(0, pattern, count) - .query::(*connection) - .map_err(Error::unavailable)?; - vec![(None, page)] - } - Self::Cluster(connection) => connection - .route_command( - &scan_command(0, pattern, count), - RoutingInfo::MultiNode(( - MultipleNodeRoutingInfo::AllMasters, - Some(ResponsePolicy::Special), - )), - ) - .map_err(Error::unavailable) - .and_then(primary_pages)? - .into_iter() - .map(|(node, page)| (Some(node), page)) - .collect(), - }; - for (node, (mut cursor, mut keys)) in pages { - loop { - if !visit(self, keys)? { - return Ok(()); - } - if cursor == 0 { - break; - } - (cursor, keys) = self.scan_page(node.as_ref(), cursor, pattern, count)?; - } - } - Ok(()) - } - - pub(crate) fn ping(&mut self) -> Result { - let command = redis::cmd("PING"); - match self { - Self::Node(connection) => command - .query::(*connection) - .map(|response| response == "PONG"), - Self::Cluster(connection) => connection - .route_command( - &command, - RoutingInfo::MultiNode(( - MultipleNodeRoutingInfo::AllNodes, - Some(ResponsePolicy::AllSucceeded), - )), - ) - .map(|_| true), - } - } - - pub(crate) fn node_text(&mut self, command: &redis::Cmd) -> Result { - match self { - Self::Node(connection) => command.query(*connection).map_err(Error::unavailable), - Self::Cluster(connection) => { - let value = connection - .route_command( - command, - RoutingInfo::MultiNode(( - MultipleNodeRoutingInfo::AllNodes, - Some(ResponsePolicy::Special), - )), - ) - .map_err(Error::unavailable)?; - let redis::Value::Map(entries) = value else { - return Err(Error::unavailable(TopologyFault::FanOutReply)); - }; - let mut replies = entries - .into_iter() - .map(|(node, reply)| { - Ok(( - redis::from_redis_value::(node).map_err(Error::unavailable)?, - redis::from_redis_value::(reply).map_err(Error::unavailable)?, - )) - }) - .collect::, Error>>()?; - replies.sort(); - Ok(replies - .into_iter() - .map(|(_, reply)| reply) - .collect::>() - .join("\n")) - } - } - } - - pub(crate) fn flushall(&mut self) -> Result<(), Error> { - let command = redis::cmd("FLUSHALL"); - match self { - Self::Node(connection) => command.query(*connection).map_err(Error::unavailable), - Self::Cluster(connection) => connection - .route_command( - &command, - RoutingInfo::MultiNode(( - MultipleNodeRoutingInfo::AllMasters, - Some(ResponsePolicy::AllSucceeded), - )), - ) - .map(|_| ()) - .map_err(Error::unavailable), - } - } - - fn scan_page( - &mut self, - node: Option<&NodeAddress>, - cursor: u64, - pattern: &str, - count: usize, - ) -> Result { - let command = scan_command(cursor, pattern, count); - match (self, node) { - (Self::Node(connection), None) => { - command.query(*connection).map_err(Error::unavailable) - } - (Self::Cluster(connection), Some(node)) => connection - .route_command( - &command, - RoutingInfo::SingleNode(SingleNodeRoutingInfo::ByAddress { - host: node.host().to_string(), - port: node.port(), - }), - ) - .map_err(Error::unavailable) - .and_then(|value| redis::from_redis_value(value).map_err(Error::unavailable)), - _ => Err(Error::unavailable(TopologyFault::ScanTarget)), - } - } -} - -type ScanPage = (u64, Vec); - -fn primary_pages(value: redis::Value) -> Result, Error> { - let redis::Value::Map(entries) = value else { - return Err(Error::unavailable(TopologyFault::FanOutReply)); - }; - entries - .into_iter() - .map(|(node, page)| { - let node = redis::from_redis_value::(node).map_err(Error::unavailable)?; - let node = NodeAddress::try_from(node.as_str()).map_err(Error::unavailable)?; - let page = redis::from_redis_value::(page).map_err(Error::unavailable)?; - Ok((node, page)) - }) - .collect() -} - -fn scan_command(cursor: u64, pattern: &str, count: usize) -> redis::Cmd { - let mut command = redis::cmd("SCAN"); - command - .cursor_arg(cursor) - .arg("MATCH") - .arg(pattern) - .arg("COUNT") - .arg(count); - command -} - -/// Hands already received pipeline replies to `Pipeline::query`, so it applies its own -/// ignore and error handling to replies a cluster pipeline gathered from several nodes. -struct Replies(Option>); - -impl redis::ConnectionLike for Replies { - fn req_packed_command(&mut self, _: &[u8]) -> redis::RedisResult { - Err((redis::ErrorKind::Client, "replies hold a pipeline only").into()) - } - - fn req_packed_commands( - &mut self, - _: &[u8], - _: usize, - _: usize, - ) -> redis::RedisResult> { - self.0 - .take() - .ok_or_else(|| (redis::ErrorKind::Client, "replies were already read").into()) - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } -} diff --git a/litellm-rust/crates/cache-redis/src/counter.rs b/litellm-rust/crates/cache-redis/src/counter.rs deleted file mode 100644 index a4beb9f5712..00000000000 --- a/litellm-rust/crates/cache-redis/src/counter.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::time::Duration; - -use litellm_cache::{ - BoundedCounterCache, CacheCodec, CountReadCache, CounterCache, Error, ExactCacheContext, - IncrementOperation, -}; - -use crate::{ - cache::{RedisCache, ttl_seconds}, - connection::ConnectionRef, - store::mget, -}; - -const INCREMENT_SCRIPT: &str = concat!( - "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", - "if redis.call('TTL', KEYS[1]) == -1 then ", - "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" -); -const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!( - "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ", - "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ", - "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return count" -); -const SET_MAX_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if current == false or tonumber(current) < tonumber(ARGV[1]) then ", - "redis.call('SET', KEYS[1], ARGV[1]); ", - "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return ARGV[1]; end; return current" -); - -impl CounterCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn increment_cache( - &self, - key: &str, - amount: f64, - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(context.ttl); - self.execute(|connection| increment(connection, key, amount, ttl, false)) - } - - /// Python `_incrbyfloat_with_ttl`: without `refresh_ttl` the TTL is set only on a key that - /// has none, in one atomic script; with it, every increment re-arms the TTL. - async fn async_increment( - &self, - key: &str, - amount: f64, - context: ExactCacheContext, - refresh_ttl: bool, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(context.ttl); - self.run(move |connection| increment(connection, key, amount, ttl, refresh_ttl)) - .await - } - - async fn async_increment_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - if operations.is_empty() { - return Ok(Vec::new()); - } - let mut pipeline = redis::pipe(); - for operation in operations { - let key = self.namespaced_key(&operation.key); - pipeline.cmd("INCRBYFLOAT").arg(&key).arg(operation.amount); - if let Some(ttl) = operation.ttl { - pipeline - .cmd("EXPIRE") - .arg(key) - .arg(ttl_seconds(ttl)) - .ignore(); - } - } - self.run(move |connection| connection.query_pipeline(&pipeline)) - .await - } -} - -fn increment( - connection: &mut ConnectionRef<'_>, - key: String, - amount: f64, - ttl: u64, - refresh_ttl: bool, -) -> Result { - if !refresh_ttl { - return redis::cmd("EVAL") - .arg(INCREMENT_SCRIPT) - .arg(1) - .arg(key) - .arg(amount) - .arg(ttl) - .query(connection) - .map_err(Error::unavailable); - } - connection - .query_pipeline( - redis::pipe() - .cmd("INCRBYFLOAT") - .arg(&key) - .arg(amount) - .cmd("EXPIRE") - .arg(&key) - .arg(ttl) - .ignore(), - ) - .map(|(value,)| value) -} - -impl CountReadCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn batch_get_counts(&self, keys: &[String]) -> Result>, Error> { - let keys = self.namespaced_keys(keys); - self.execute(|connection| mget(connection, keys))? - .into_iter() - .map(count) - .collect() - } - - async fn async_batch_get_counts(&self, keys: Vec) -> Result>, Error> { - let keys = self.namespaced_keys(&keys); - self.run(move |connection| mget(connection, keys)) - .await? - .into_iter() - .map(count) - .collect() - } -} - -fn count(value: redis::Value) -> Result, Error> { - redis::from_redis_value(value).map_err(Error::invalid_entry) -} - -impl BoundedCounterCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn increment_with_floor(&self, key: &str, amount: i64, ttl: Duration) -> Result { - let key = self.namespaced_key(key); - let ttl = ttl_seconds(ttl); - self.execute(|connection| increment_with_floor(connection, key, amount, ttl)) - } - - async fn async_increment_with_floor( - &self, - key: &str, - amount: i64, - ttl: Duration, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = ttl_seconds(ttl); - self.run(move |connection| increment_with_floor(connection, key, amount, ttl)) - .await - } - - async fn async_set_max( - &self, - key: &str, - value: f64, - ttl: Option, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(ttl); - self.run(move |connection| { - redis::cmd("EVAL") - .arg(SET_MAX_SCRIPT) - .arg(1) - .arg(key) - .arg(value) - .arg(ttl) - .query(connection) - .map_err(Error::unavailable) - }) - .await - } -} - -fn increment_with_floor( - connection: &mut ConnectionRef<'_>, - key: String, - amount: i64, - ttl: u64, -) -> Result { - redis::cmd("EVAL") - .arg(INCREMENT_WITH_FLOOR_SCRIPT) - .arg(1) - .arg(key) - .arg(amount) - .arg(ttl) - .query(connection) - .map_err(Error::unavailable) -} diff --git a/litellm-rust/crates/cache-redis/src/keys.rs b/litellm-rust/crates/cache-redis/src/keys.rs deleted file mode 100644 index 2e8a9e60e65..00000000000 --- a/litellm-rust/crates/cache-redis/src/keys.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::time::Duration; - -use litellm_cache::{CacheCodec, Error, RefreshTtlCache, ScanCache, TtlCache}; - -use crate::cache::RedisCache; - -impl TtlCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_get_ttl(&self, key: &str) -> Result, Error> { - let key = self.namespaced_key(key); - let ttl = self - .run(move |connection| { - redis::cmd("TTL") - .arg(key) - .query::(connection) - .map_err(Error::unavailable) - }) - .await?; - Ok(u64::try_from(ttl).ok().map(Duration::from_secs)) - } -} - -impl RefreshTtlCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_refresh_ttl(&self, key: &str, ttl: Option) -> Result { - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(ttl); - self.run(move |connection| { - redis::cmd("EXPIRE") - .arg(key) - .arg(ttl) - .query(connection) - .map_err(Error::unavailable) - }) - .await - } -} - -impl ScanCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result, Error> { - let pattern = format!("{}*", self.namespaced_key(pattern)); - self.run(move |connection| { - let mut matches = Vec::new(); - connection.scan(&pattern, count, |_, keys| { - matches.extend(keys); - Ok(matches.len() < count) - })?; - matches.truncate(count); - Ok(matches) - }) - .await - } -} diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs deleted file mode 100644 index 037e39e5d40..00000000000 --- a/litellm-rust/crates/cache-redis/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod cache; -mod claim; -pub mod connection; -mod counter; -mod keys; -mod lifecycle; -mod queue; -mod script; -mod store; -mod topology; - -pub use cache::RedisCache; -pub use queue::{RedisLpopOperation, RedisLpopResult, RedisRpushOperation}; -pub use script::{RedisArg, RedisScript}; -pub use topology::{RedisNode, RedisTopology}; diff --git a/litellm-rust/crates/cache-redis/src/lifecycle.rs b/litellm-rust/crates/cache-redis/src/lifecycle.rs deleted file mode 100644 index 54fa8ecc3bf..00000000000 --- a/litellm-rust/crates/cache-redis/src/lifecycle.rs +++ /dev/null @@ -1,86 +0,0 @@ -use litellm_cache::{ - CacheCodec, CacheConnectionResult, CacheConnectionStatus, ClientInfoCache, ConnectionCache, - DisconnectCache, Error, PingCache, -}; - -use crate::{cache::RedisCache, topology::RedisTopology}; - -impl PingCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn sync_ping(&self) -> Result { - self.execute(|connection| connection.ping().map_err(Error::unavailable)) - } - - async fn ping(&self) -> Result { - self.run(|connection| connection.ping().map_err(Error::unavailable)) - .await - } -} - -impl ConnectionCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - /// Python `RedisCache.test_connection`, or `RedisClusterCache.test_connection` for a - /// cluster topology, which differs only in its messages. - async fn test_connection(&self) -> Result { - let label = match self.topology { - RedisTopology::Standalone => "Redis", - RedisTopology::Cluster { .. } => "Redis Cluster", - }; - let ping = self - .run(|connection| Ok(connection.ping().map_err(Error::unavailable))) - .await - .and_then(|ping| ping) - .map_err(|error| litellm_error::Report::new(&error).to_string()); - Ok(match ping { - Ok(true) => CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: format!("{label} connection test successful"), - error: None, - }, - Ok(false) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("{label} ping returned False"), - error: None, - }, - Err(error) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("{label} connection failed: {error}"), - error: Some(error), - }, - }) - } -} - -impl DisconnectCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn disconnect(&self) -> Result<(), Error> { - self.connections.disconnect(); - Ok(()) - } -} - -impl ClientInfoCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type ClientList = String; - type Info = String; - - fn client_list(&self) -> Result { - self.execute(|connection| connection.node_text(redis::cmd("CLIENT").arg("LIST"))) - } - - fn info(&self) -> Result { - self.execute(|connection| connection.node_text(&redis::cmd("INFO"))) - } -} diff --git a/litellm-rust/crates/cache-redis/src/queue.rs b/litellm-rust/crates/cache-redis/src/queue.rs deleted file mode 100644 index d55e4f49187..00000000000 --- a/litellm-rust/crates/cache-redis/src/queue.rs +++ /dev/null @@ -1,226 +0,0 @@ -use std::time::Duration; - -use litellm_cache::{CacheCodec, Error, PopOperation, PushOperation, QueueCache, SetCache}; - -use crate::{cache::RedisCache, script::RedisArg}; - -pub type RedisRpushOperation = PushOperation; -pub type RedisLpopOperation = PopOperation; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RedisLpopResult { - Missing, - Value(Vec), - Values(Vec>), -} - -impl SetCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type SetValue = RedisArg; - type SetResult = usize; - - async fn async_set_cache_sadd( - &self, - key: &str, - values: Vec, - ttl: Option, - ) -> Result { - if values.is_empty() { - return Err(Error::InvalidEntry(None)); - } - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(ttl); - let mut pipeline = redis::pipe(); - pipeline - .cmd("SADD") - .arg(&key) - .arg(values) - .cmd("EXPIRE") - .arg(&key) - .arg(ttl) - .ignore(); - self.run(move |connection| connection.query_pipeline(&pipeline)) - .await - .map(|(added,)| added) - } -} - -impl QueueCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type QueueValue = RedisArg; - type PopResult = RedisLpopResult; - - async fn async_rpush(&self, key: &str, values: Vec) -> Result { - if values.is_empty() { - return Err(Error::InvalidEntry(None)); - } - let key = self.namespaced_key(key); - self.run(move |connection| { - redis::cmd("RPUSH") - .arg(key) - .arg(values) - .query(connection) - .map_err(Error::unavailable) - }) - .await - } - - async fn async_rpush_and_trim( - &self, - key: &str, - values: Vec, - max_len: usize, - ) -> Result { - if values.is_empty() { - return Err(Error::InvalidEntry(None)); - } - let key = self.namespaced_key(key); - let start = i64::try_from(max_len).map_or(i64::MIN, |max_len| -max_len); - let mut pipeline = redis::pipe(); - pipeline - .atomic() - .cmd("RPUSH") - .arg(&key) - .arg(values) - .cmd("LTRIM") - .arg(&key) - .arg(start) - .arg(-1) - .ignore(); - self.run(move |connection| connection.query_pipeline(&pipeline)) - .await - .map(|(length,)| length) - } - - async fn async_rpush_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - if operations.is_empty() { - return Ok(Vec::new()); - } - let mut pipeline = redis::pipe(); - for operation in operations { - if operation.values.is_empty() { - return Err(Error::InvalidEntry(None)); - } - pipeline - .cmd("RPUSH") - .arg(self.namespaced_key(&operation.key)) - .arg(operation.values); - } - self.run(move |connection| connection.query_pipeline(&pipeline)) - .await - } - - async fn async_lpop(&self, key: &str, count: Option) -> Result { - if let Some(count) = count - && self.major_version().await < 7 - { - return self.lpop_one_at_a_time(key, count).await; - } - let command = lpop(self.namespaced_key(key), count); - let value = self - .run(move |connection| { - command - .query::(connection) - .map_err(Error::unavailable) - }) - .await?; - lpop_result(value, count.is_some()) - } - - async fn async_lpop_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - if operations.is_empty() { - return Ok(Vec::new()); - } - if operations.iter().any(|operation| operation.count.is_some()) - && self.major_version().await < 7 - { - let mut results = Vec::with_capacity(operations.len()); - for operation in &operations { - results.push(self.async_lpop(&operation.key, operation.count).await?); - } - return Ok(results); - } - let multiple = operations - .iter() - .map(|operation| operation.count.is_some()) - .collect::>(); - let mut pipeline = redis::pipe(); - for operation in operations { - pipeline.add_command(lpop(self.namespaced_key(&operation.key), operation.count)); - } - self.run(move |connection| connection.query_pipeline::>(&pipeline)) - .await? - .into_iter() - .zip(multiple) - .map(|(value, multiple)| lpop_result(value, multiple)) - .collect() - } -} - -fn lpop(key: String, count: Option) -> redis::Cmd { - let mut command = redis::cmd("LPOP"); - command.arg(key); - if let Some(count) = count { - command.arg(count); - } - command -} - -fn redis_bytes(value: redis::Value) -> Result, Error> { - match value { - redis::Value::BulkString(bytes) => Ok(bytes), - redis::Value::SimpleString(text) => Ok(text.into_bytes()), - _ => Err(Error::InvalidEntry(None)), - } -} - -impl RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - /// `handle_lpop_count_for_older_redis_versions`: `count` single-`LPOP` pipelines, keeping - /// only the values actually popped. - async fn lpop_one_at_a_time(&self, key: &str, count: usize) -> Result { - let key = self.namespaced_key(key); - let mut values = Vec::new(); - for _ in 0..count { - let mut pipeline = redis::pipe(); - pipeline.add_command(lpop(key.clone(), None)); - let replies = self - .run(move |connection| connection.query_pipeline::>(&pipeline)) - .await?; - for reply in replies { - if reply != redis::Value::Nil { - values.push(redis_bytes(reply)?); - } - } - } - Ok(RedisLpopResult::Values(values)) - } -} - -fn lpop_result(value: redis::Value, multiple: bool) -> Result { - match value { - redis::Value::Nil => Ok(RedisLpopResult::Missing), - redis::Value::Array(values) if multiple => values - .into_iter() - .map(redis_bytes) - .collect::, _>>() - .map(RedisLpopResult::Values), - value if !multiple => redis_bytes(value).map(RedisLpopResult::Value), - _ => Err(Error::InvalidEntry(None)), - } -} diff --git a/litellm-rust/crates/cache-redis/src/script.rs b/litellm-rust/crates/cache-redis/src/script.rs deleted file mode 100644 index 09987437ed4..00000000000 --- a/litellm-rust/crates/cache-redis/src/script.rs +++ /dev/null @@ -1,136 +0,0 @@ -use std::sync::Arc; - -use litellm_cache::{CacheCodec, CacheScript, Error, ScriptCache}; - -use crate::{ - cache::{RedisCache, namespaced_key}, - connection::Connections, -}; - -#[derive(Clone, Debug, PartialEq)] -pub enum RedisArg { - Bytes(Vec), - Integer(i64), - Float(f64), -} - -impl From<&str> for RedisArg { - fn from(value: &str) -> Self { - Self::Bytes(value.as_bytes().to_vec()) - } -} - -impl From for RedisArg { - fn from(value: String) -> Self { - Self::Bytes(value.into_bytes()) - } -} - -impl From> for RedisArg { - fn from(value: Vec) -> Self { - Self::Bytes(value) - } -} - -impl From for RedisArg { - fn from(value: i64) -> Self { - Self::Integer(value) - } -} - -impl From for RedisArg { - fn from(value: f64) -> Self { - Self::Float(value) - } -} - -impl redis::ToRedisArgs for RedisArg { - fn write_redis_args(&self, out: &mut W) - where - W: ?Sized + redis::RedisWrite, - { - match self { - Self::Bytes(value) => value.write_redis_args(out), - Self::Integer(value) => value.write_redis_args(out), - Self::Float(value) => value.write_redis_args(out), - } - } -} - -pub struct RedisScript { - connections: Arc>, - namespace: Option, - source: String, -} - -impl CacheScript for RedisScript -where - C: redis::ConnectionLike + Send + 'static, -{ - type Argument = RedisArg; - type Output = redis::Value; - - async fn invoke( - &self, - keys: Vec, - arguments: Vec, - ) -> Result { - let keys = keys - .into_iter() - .map(|key| namespaced_key(self.namespace.as_deref(), &key)) - .collect::>(); - let source = self.source.clone(); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - eval(connection, &source, keys, arguments) - }) - .await - } -} - -impl RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub async fn async_eval( - &self, - script: String, - keys: Vec, - arguments: Vec, - ) -> Result { - let keys = self.namespaced_keys(&keys); - self.run(move |connection| eval(connection, &script, keys, arguments)) - .await - } -} - -fn eval( - connection: &mut impl redis::ConnectionLike, - script: &str, - keys: Vec, - arguments: Vec, -) -> Result { - redis::cmd("EVAL") - .arg(script) - .arg(keys.len()) - .arg(keys) - .arg(arguments) - .query(connection) - .map_err(Error::unavailable) -} - -impl ScriptCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Script = RedisScript; - - fn async_register_script(&self, source: String) -> Self::Script { - RedisScript { - connections: Arc::clone(&self.connections), - namespace: self.namespace.clone(), - source, - } - } -} diff --git a/litellm-rust/crates/cache-redis/src/store.rs b/litellm-rust/crates/cache-redis/src/store.rs deleted file mode 100644 index daadc940ca9..00000000000 --- a/litellm-rust/crates/cache-redis/src/store.rs +++ /dev/null @@ -1,232 +0,0 @@ -use std::time::Duration; - -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, BulkDeleteCache, CacheCodec, DeleteCache, Error, - ExactCacheContext, FlushAllCache, FlushCache, TtlPipelineCache, -}; -use redis::Commands; - -use crate::{cache::RedisCache, connection::ConnectionRef}; - -impl BaseCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Value = S::Value; - type Context = ExactCacheContext; - - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl.or(Some(self.default_ttl)) - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &ExactCacheContext, - ) -> Result<(), Error> { - let payload = self.codec.encode(&value)?; - let ttl = self.ttl_or_default(context.ttl); - let key = self.namespaced_key(key); - self.execute(|connection| { - connection - .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(Error::unavailable) - }) - } - - fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { - let key = self.namespaced_key(key); - let value = self.execute(|connection| { - connection - .get::<_, redis::Value>(key) - .map_err(Error::unavailable) - })?; - self.decode_response(value) - } - - async fn async_set_cache( - &self, - key: &str, - value: Self::Value, - context: ExactCacheContext, - ) -> Result<(), Error> { - let payload = self.codec.encode(&value)?; - let key = self.namespaced_key(key); - let ttl = self.ttl_or_default(context.ttl); - self.run(move |connection| { - connection - .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(Error::unavailable) - }) - .await - } - - async fn async_get_cache( - &self, - key: &str, - _: &ExactCacheContext, - ) -> Result, Error> { - let key = self.namespaced_key(key); - let value = self - .run(move |connection| { - connection - .get::<_, redis::Value>(key) - .map_err(Error::unavailable) - }) - .await?; - self.decode_response(value) - } - - async fn async_set_cache_pipeline( - &self, - cache_list: Vec<(String, Self::Value)>, - context: ExactCacheContext, - ) -> Result<(), Error> { - self.async_set_cache_pipeline_with_ttls( - cache_list - .into_iter() - .map(|(key, value)| (key, value, context.ttl)) - .collect(), - ) - .await - } -} - -impl TtlPipelineCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_set_cache_pipeline_with_ttls( - &self, - entries: Vec<(String, Self::Value, Option)>, - ) -> Result<(), Error> { - if entries.is_empty() { - return Ok(()); - } - let mut pipeline = redis::pipe(); - for (key, value, ttl) in entries { - pipeline - .cmd("SETEX") - .arg(self.namespaced_key(&key)) - .arg(self.ttl_or_default(ttl)) - .arg(self.codec.encode(&value)?) - .ignore(); - } - self.run(move |connection| connection.query_pipeline(&pipeline)) - .await - } -} - -impl BatchCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn batch_get_cache( - &self, - keys: &[String], - _: &ExactCacheContext, - ) -> Result>, Error> { - let keys = self.namespaced_keys(keys); - self.execute(|connection| mget(connection, keys))? - .into_iter() - .map(|value| self.decode_batch_response(value)) - .collect() - } - - async fn async_batch_get_cache( - &self, - keys: Vec, - _: ExactCacheContext, - ) -> Result>, Error> { - let keys = self.namespaced_keys(&keys); - self.run(move |connection| mget(connection, keys)) - .await? - .into_iter() - .map(|value| self.decode_batch_response(value)) - .collect() - } -} - -pub(crate) fn mget( - connection: &mut ConnectionRef<'_>, - keys: Vec, -) -> Result, Error> { - redis::cmd("MGET") - .arg(keys) - .query(connection) - .map_err(Error::unavailable) -} - -impl DeleteCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn delete_cache(&self, key: &str) -> Result<(), Error> { - let key = self.namespaced_key(key); - self.execute(|connection| connection.del::<_, ()>(key).map_err(Error::unavailable)) - } - - async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { - let key = self.namespaced_key(key); - self.run(move |connection| connection.del::<_, ()>(key).map_err(Error::unavailable)) - .await - } -} - -impl BulkDeleteCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn delete_cache_keys(&self, keys: Vec) -> Result { - if keys.is_empty() { - return Ok(0); - } - let keys = self.namespaced_keys(&keys); - self.run(move |connection| connection.del(keys).map_err(Error::unavailable)) - .await - } -} - -impl FlushCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn flush_cache(&self) -> Result<(), Error> { - let pattern = self.namespaced_pattern()?; - self.execute(|connection| flush_matching(connection, &pattern)) - } - - async fn async_flush_cache(&self) -> Result<(), Error> { - let pattern = self.namespaced_pattern()?; - self.run(move |connection| flush_matching(connection, &pattern)) - .await - } -} - -impl FlushAllCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn flushall(&self) -> Result<(), Error> { - self.execute(|connection| connection.flushall()) - } -} - -fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<(), Error> { - connection.scan(pattern, 1000, |connection, keys| { - if !keys.is_empty() { - connection - .del::<_, usize>(keys) - .map_err(Error::unavailable)?; - } - Ok(true) - }) -} diff --git a/litellm-rust/crates/cache-redis/src/topology.rs b/litellm-rust/crates/cache-redis/src/topology.rs deleted file mode 100644 index 7f4ee48b222..00000000000 --- a/litellm-rust/crates/cache-redis/src/topology.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RedisNode { - pub host: String, - pub port: u16, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub enum RedisTopology { - #[default] - Standalone, - Cluster { - startup_nodes: Vec, - }, -} diff --git a/litellm-rust/crates/cache-redis/tests/cache.rs b/litellm-rust/crates/cache-redis/tests/cache.rs deleted file mode 100644 index 9ec85251733..00000000000 --- a/litellm-rust/crates/cache-redis/tests/cache.rs +++ /dev/null @@ -1,1164 +0,0 @@ -mod support; - -use std::time::Duration; - -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, BoundedCounterCache, BulkDeleteCache, CacheCodec, - CacheConnectionStatus, CacheScript, ClaimCache, ClientInfoCache, ConnectionCache, - CountReadCache, CounterCache, DeleteCache, DisconnectCache, Error, ExactCacheContext, - FlushAllCache, FlushCache, IncrementOperation, JsonCodec, PingCache, QueueCache, - RefreshTtlCache, ScanCache, ScriptCache, SetCache, TtlCache, TtlPipelineCache, get_cache, - set_cache, -}; -use litellm_cache_redis::{ - RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, -}; -use redis_test::{MockCmd, MockRedisConnection}; -use rstest::{fixture, rstest}; -use serde_json::json; -use support::TaggedByteCodec; - -type Mocked = RedisCache; - -fn mock(commands: Vec) -> MockRedisConnection { - MockRedisConnection::new(commands).assert_all_commands_consumed() -} - -fn tagged(commands: Vec) -> Mocked { - RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42)) -} - -fn json_cache(commands: Vec) -> Mocked> { - RedisCache::with_connection(mock(commands), None, JsonCodec::new()) -} - -fn team(commands: Vec) -> Mocked> { - json_cache(commands).with_namespace(Some("team".into())) -} - -#[fixture] -fn context() -> ExactCacheContext { - ExactCacheContext::default() -} - -fn scan(pattern: &str, cursor: u64, count: usize, reply: redis::Value) -> MockCmd { - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(cursor) - .arg("MATCH") - .arg(pattern) - .arg("COUNT") - .arg(count), - Ok(reply), - ) -} - -#[rstest] -fn constructor_rejects_invalid_urls() { - assert!(RedisCache::new("not a redis url", None, JsonCodec::::new()).is_err()); -} - -#[rstest] -#[case::zero_rounds_up_to_one(Some(Duration::ZERO), 1)] -#[case::fractions_round_up(Some(Duration::from_millis(1500)), 2)] -#[case::whole_seconds_are_kept(Some(Duration::from_secs(15)), 15)] -#[case::missing_ttl_uses_default(None, 600)] -fn writes_round_ttls_up_to_positive_seconds(#[case] ttl: Option, #[case] seconds: u64) { - let cache = tagged(vec![MockCmd::new( - redis::cmd("SETEX") - .arg("key") - .arg(seconds) - .arg([42u8, 7].as_slice()), - Ok("OK"), - )]); - cache - .set_cache("key", 7, &ExactCacheContext { ttl }) - .unwrap(); -} - -#[rstest] -fn generic_helpers_use_the_injected_codec_and_ttl() { - let cache = tagged(vec![ - MockCmd::new( - redis::cmd("SETEX") - .arg("counter") - .arg(2) - .arg([42u8, 7].as_slice()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])), - ]); - let context = ExactCacheContext { - ttl: Some(Duration::from_millis(1500)), - }; - set_cache(&cache, "counter", 7, &context).unwrap(); - assert_eq!(get_cache(&cache, "counter", &context).unwrap(), Some(7)); -} - -#[rstest] -fn commands_round_trip_entries_and_delete_only_namespaced_keys(context: ExactCacheContext) { - let value = json!({"deployment": "model-a", "cooldown_seconds": 30}); - let payload = JsonCodec::::new() - .encode(&value) - .unwrap(); - let cache = json_cache(vec![ - MockCmd::new( - redis::cmd("SETEX") - .arg("litellm-cache:key") - .arg(600) - .arg(payload.clone()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("GET").arg("litellm-cache:key"), Ok(payload)), - MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)), - ]) - .with_namespace(Some("litellm-cache".into())); - - cache.set_cache("key", value.clone(), &context).unwrap(); - assert_eq!(cache.get_cache("key", &context).unwrap(), Some(value)); - cache.delete_cache("key").unwrap(); -} - -#[rstest] -#[tokio::test] -async fn async_operations_preserve_codec_ttl_and_missing_values(context: ExactCacheContext) { - let cache = RedisCache::with_connection( - mock(vec![ - MockCmd::new( - redis::cmd("SETEX") - .arg("counter") - .arg(9) - .arg([42u8, 7].as_slice()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])), - MockCmd::new( - redis::cmd("SETEX") - .arg("batch") - .arg(2) - .arg([42u8, 8].as_slice()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("DEL").arg("counter"), Ok(1u32)), - MockCmd::new(redis::cmd("GET").arg("counter"), Ok(redis::Value::Nil)), - ]), - Some(Duration::from_secs(9)), - TaggedByteCodec(42), - ); - cache - .batch_cache_write("counter", 7, context.clone()) - .await - .unwrap(); - assert_eq!( - cache.async_get_cache("counter", &context).await.unwrap(), - Some(7) - ); - cache - .async_set_cache_pipeline( - vec![("batch".into(), 8)], - ExactCacheContext { - ttl: Some(Duration::from_millis(1500)), - }, - ) - .await - .unwrap(); - cache.async_delete_cache("counter").await.unwrap(); - assert_eq!( - cache.async_get_cache("counter", &context).await.unwrap(), - None - ); -} - -#[rstest] -#[tokio::test] -async fn codec_errors_propagate_without_writing_partial_batches(context: ExactCacheContext) { - let cache = tagged(vec![ - MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])), - MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])), - ]); - assert_eq!( - cache.set_cache("invalid", 255, &context), - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache.async_set_cache("invalid", 255, context.clone()).await, - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache - .async_set_cache_pipeline( - vec![("valid".into(), 7), ("invalid".into(), 255)], - context.clone(), - ) - .await, - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache - .async_set_cache_pipeline_with_ttls(vec![ - ("valid".into(), 7, None), - ("invalid".into(), 255, None), - ]) - .await, - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache.get_cache("invalid", &context), - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache.async_get_cache("invalid", &context).await, - Err(Error::InvalidEntry(None)) - ); -} - -#[rstest] -#[case::bare_key("key")] -#[case::already_prefixed("team:key")] -fn namespaces_are_added_once(#[case] key: &str, context: ExactCacheContext) { - let cache = team(vec![MockCmd::new( - redis::cmd("GET").arg("team:key"), - Ok(redis::Value::Nil), - )]); - assert_eq!(cache.get_cache(key, &context).unwrap(), None); -} - -#[rstest] -#[case::empty(Some(String::new()))] -#[case::missing(None)] -fn empty_namespaces_leave_keys_unprefixed( - #[case] namespace: Option, - context: ExactCacheContext, -) { - let cache = json_cache(vec![MockCmd::new( - redis::cmd("GET").arg("key"), - Ok(redis::Value::Nil), - )]) - .with_namespace(namespace); - assert_eq!(cache.namespace(), None); - assert_eq!(cache.get_cache("key", &context).unwrap(), None); -} - -#[rstest] -#[tokio::test] -async fn flush_requires_a_namespace() { - let unscoped = json_cache(Vec::new()); - assert_eq!(unscoped.flush_cache(), Err(Error::UnscopedFlush)); - assert_eq!( - unscoped.async_flush_cache().await, - Err(Error::UnscopedFlush) - ); -} - -#[rstest] -#[case::plain_namespace("litellm-cache", "litellm-cache:*", "litellm-cache:key")] -#[case::glob_metacharacters_are_escaped("team*", "team\\*:*", "team*:key")] -fn flush_scans_and_deletes_only_namespaced_keys( - #[case] namespace: &str, - #[case] pattern: &str, - #[case] key: &str, -) { - let cache = json_cache(vec![ - scan(pattern, 0, 1000, redis_test::redis_value!(["0", [key]])), - MockCmd::new(redis::cmd("DEL").arg(key), Ok(1u32)), - ]) - .with_namespace(Some(namespace.into())); - cache.flush_cache().unwrap(); -} - -#[rstest] -#[tokio::test] -async fn async_flush_deletes_each_scan_page_separately() { - let cache = team(vec![ - scan( - "team:*", - 0, - 1000, - redis_test::redis_value!(["7", ["team:a", "team:b"]]), - ), - MockCmd::new(redis::cmd("DEL").arg("team:a").arg("team:b"), Ok(2u32)), - scan( - "team:*", - 7, - 1000, - redis_test::redis_value!(["0", ["team:c"]]), - ), - MockCmd::new(redis::cmd("DEL").arg("team:c"), Ok(1u32)), - ]); - cache.async_flush_cache().await.unwrap(); -} - -#[rstest] -fn flushall_ignores_the_namespace() { - team(vec![MockCmd::new(redis::cmd("FLUSHALL"), Ok("OK"))]) - .flushall() - .unwrap(); -} - -#[rstest] -#[tokio::test] -async fn batch_reads_keep_order_and_treat_invalid_values_as_invalid_entries( - context: ExactCacheContext, -) { - let cache = tagged(vec![MockCmd::new( - redis::cmd("MGET").arg("hit").arg("miss").arg("invalid"), - Ok(vec![ - redis::Value::BulkString(vec![42, 7]), - redis::Value::Nil, - redis::Value::BulkString(vec![99, 7]), - ]), - )]); - - assert_eq!( - cache - .async_batch_get_cache(vec!["hit".into(), "miss".into(), "invalid".into()], context) - .await - .unwrap(), - vec![BatchEntry::Hit(7), BatchEntry::Miss, BatchEntry::Invalid] - ); -} - -#[rstest] -#[tokio::test] -async fn ttl_pipeline_keeps_each_entry_ttl_and_defaults_missing_ones() { - let mut pipeline = redis::pipe(); - pipeline - .cmd("SETEX") - .arg("ns:team_id:t1") - .arg(60u64) - .arg(r#"{"team_id":"t1"}"#) - .cmd("SETEX") - .arg("ns:u1") - .arg(7u64) - .arg(r#"{"user_id":"u1"}"#) - .cmd("SETEX") - .arg("ns:org_id:o1") - .arg(300u64) - .arg(r#"{"a":1}"#); - let cache = RedisCache::with_connection( - mock(vec![MockCmd::with_values( - pipeline, - Ok(vec!["OK", "OK", "OK"]), - )]), - Some(Duration::from_secs(300)), - JsonCodec::::new(), - ) - .with_namespace(Some("ns".into())); - - cache - .async_set_cache_pipeline_with_ttls(vec![ - ( - "team_id:t1".into(), - json!({"team_id": "t1"}), - Some(Duration::from_secs(60)), - ), - ( - "u1".into(), - json!({"user_id": "u1"}), - Some(Duration::from_secs(7)), - ), - ("org_id:o1".into(), json!({"a": 1}), None), - ]) - .await - .unwrap(); -} - -#[rstest] -#[tokio::test] -async fn empty_pipelines_skip_the_round_trip(context: ExactCacheContext) { - let cache = json_cache(Vec::new()); - cache - .async_set_cache_pipeline(Vec::new(), context) - .await - .unwrap(); - cache - .async_set_cache_pipeline_with_ttls(Vec::new()) - .await - .unwrap(); - assert_eq!(cache.delete_cache_keys(Vec::new()).await.unwrap(), 0); - assert_eq!( - cache.async_rpush_pipeline(Vec::new()).await.unwrap(), - Vec::::new() - ); - assert_eq!( - cache.async_lpop_pipeline(Vec::new()).await.unwrap(), - Vec::::new() - ); - assert_eq!( - cache.async_increment_pipeline(Vec::new()).await.unwrap(), - Vec::::new() - ); -} - -#[rstest] -#[tokio::test] -async fn count_reads_parse_integers_and_keep_missing_counters() { - let mget = || { - MockCmd::new( - redis::cmd("MGET").arg("team:count").arg("team:missing"), - Ok(redis_test::redis_value!(["7", nil])), - ) - }; - let cache = team(vec![mget(), mget()]); - let keys = vec!["count".to_string(), "missing".to_string()]; - - assert_eq!(cache.batch_get_counts(&keys).unwrap(), [Some(7), None]); - assert_eq!( - cache.async_batch_get_counts(keys).await.unwrap(), - [Some(7), None] - ); -} - -#[rstest] -#[tokio::test] -async fn pings_run_on_both_paths() { - let cache = team(vec![ - MockCmd::new(redis::cmd("PING"), Ok("PONG")), - MockCmd::new(redis::cmd("PING"), Ok("PONG")), - ]); - assert!(cache.sync_ping().unwrap()); - assert!(cache.ping().await.unwrap()); -} - -#[rstest] -#[case::remaining(12, Some(Duration::from_secs(12)))] -#[case::no_expiry(-1, None)] -#[case::missing(-2, None)] -#[tokio::test] -async fn ttl_reads_hide_negative_replies(#[case] reply: i64, #[case] ttl: Option) { - let cache = team(vec![MockCmd::new( - redis::cmd("TTL").arg("team:key"), - Ok(reply), - )]); - assert_eq!(cache.async_get_ttl("key").await.unwrap(), ttl); -} - -#[rstest] -#[case::explicit_ttl(Some(Duration::from_secs(30)), 30, 1, true)] -#[case::default_ttl(None, 600, 1, true)] -#[case::missing_key(Some(Duration::from_secs(30)), 30, 0, false)] -#[tokio::test] -async fn refresh_ttl_expires_existing_keys_only( - #[case] ttl: Option, - #[case] seconds: u64, - #[case] reply: i64, - #[case] refreshed: bool, -) { - let cache = team(vec![MockCmd::new( - redis::cmd("EXPIRE").arg("team:key").arg(seconds), - Ok(reply), - )]); - assert_eq!( - cache.async_refresh_ttl("key", ttl).await.unwrap(), - refreshed - ); -} - -#[rstest] -#[tokio::test] -async fn scan_stops_at_count_and_bulk_delete_reports_existing_keys() { - let cache = team(vec![ - scan( - "team:job-*", - 0, - 25, - redis_test::redis_value!(["4", ["team:job-a"]]), - ), - scan( - "team:job-*", - 4, - 25, - redis_test::redis_value!(["0", ["team:job-b"]]), - ), - MockCmd::new( - redis::cmd("DEL").arg("team:job-a").arg("team:job-b"), - Ok(2u32), - ), - ]); - assert_eq!( - cache.async_scan_iter("job-", 25).await.unwrap(), - ["team:job-a", "team:job-b"] - ); - assert_eq!( - cache - .delete_cache_keys(vec!["job-a".into(), "job-b".into()]) - .await - .unwrap(), - 2 - ); -} - -#[rstest] -#[tokio::test] -async fn sets_add_members_and_arm_the_default_ttl() { - let mut pipeline = redis::pipe(); - pipeline - .cmd("SADD") - .arg("team:members") - .arg("a") - .arg("b") - .cmd("EXPIRE") - .arg("team:members") - .arg(600u64) - .ignore(); - let cache = team(vec![MockCmd::with_values( - pipeline, - Ok(vec![redis::Value::Int(2), redis::Value::Int(1)]), - )]); - assert_eq!( - cache - .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None) - .await - .unwrap(), - 2 - ); - assert_eq!( - cache - .async_set_cache_sadd("members", Vec::new(), None) - .await, - Err(Error::InvalidEntry(None)) - ); -} - -#[rstest] -#[tokio::test] -async fn queues_push_and_pop_namespaced_lists() { - let cache = team(vec![ - MockCmd::new( - redis::cmd("RPUSH").arg("team:queue").arg("a").arg("b"), - Ok(2u32), - ), - MockCmd::new( - redis::cmd("INFO"), - Ok("# Server\r\nredis_version:7.2.4\r\n"), - ), - MockCmd::new( - redis::cmd("LPOP").arg("team:queue").arg(2usize), - Ok(redis_test::redis_value!(["a", "b"])), - ), - MockCmd::new(redis::cmd("LPOP").arg("team:queue"), Ok("c")), - ]); - assert_eq!( - cache - .async_rpush("queue", vec!["a".into(), "b".into()]) - .await - .unwrap(), - 2 - ); - assert_eq!( - cache.async_rpush("queue", Vec::new()).await, - Err(Error::InvalidEntry(None)) - ); - assert_eq!( - cache.async_lpop("queue", Some(2)).await.unwrap(), - RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) - ); - assert_eq!( - cache.async_lpop("queue", None).await.unwrap(), - RedisLpopResult::Value(b"c".to_vec()) - ); -} - -/// Python `RedisCache.async_lpop` checks `redis_version` from `INFO` and, below major version 7, -/// pops a counted batch as `count` single-command `LPOP` pipelines, dropping `None` replies. -#[rstest] -#[tokio::test] -async fn counted_lpop_falls_back_to_single_pops_below_redis_7() { - let single_pop = || { - let mut pipeline = redis::pipe(); - pipeline.cmd("LPOP").arg("team:queue"); - pipeline - }; - let cache = team(vec![ - MockCmd::new( - redis::cmd("INFO"), - Ok("# Server\r\nredis_version:6.2.14\r\n"), - ), - MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("a")])), - MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("b")])), - MockCmd::with_values(single_pop(), Ok(vec![redis::Value::Nil])), - ]); - - assert_eq!( - cache.async_lpop("queue", Some(3)).await.unwrap(), - RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) - ); -} - -/// Python keeps `redis_version = "Unknown"` when `INFO` fails and then assumes -/// `DEFAULT_REDIS_MAJOR_VERSION` (7), so a counted pop is one `LPOP key count`. The version is read -/// once: the second pop sends no second `INFO`. -#[rstest] -#[tokio::test] -async fn counted_lpop_assumes_redis_7_when_info_fails() { - let cache = team(vec![ - MockCmd::new( - redis::cmd("INFO"), - Err::(redis::RedisError::from((redis::ErrorKind::Io, "down"))), - ), - MockCmd::new( - redis::cmd("LPOP").arg("team:queue").arg(2usize), - Ok(redis_test::redis_value!(["a", "b"])), - ), - MockCmd::new( - redis::cmd("LPOP").arg("team:queue").arg(1usize), - Ok(redis_test::redis_value!(["c"])), - ), - ]); - - assert_eq!( - cache.async_lpop("queue", Some(2)).await.unwrap(), - RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) - ); - assert_eq!( - cache.async_lpop("queue", Some(1)).await.unwrap(), - RedisLpopResult::Values(vec![b"c".to_vec()]) - ); -} - -fn push_and_trim(start: i64) -> redis::Pipeline { - let mut pipeline = redis::pipe(); - pipeline - .atomic() - .cmd("RPUSH") - .arg("ns:buf") - .arg("c") - .arg("d") - .cmd("LTRIM") - .arg("ns:buf") - .arg(start) - .arg(-1); - pipeline -} - -#[rstest] -#[case::keeps_newest_entries(3, -3)] -#[case::zero_keeps_everything(0, 0)] -#[tokio::test] -async fn rpush_and_trim_runs_push_and_trim_in_one_transaction( - #[case] max_len: usize, - #[case] start: i64, -) { - let cache = json_cache(vec![MockCmd::with_values( - push_and_trim(start), - Ok(vec![redis::Value::Array(vec![ - redis::Value::Int(4), - redis::Value::Okay, - ])]), - )]) - .with_namespace(Some("ns".into())); - - assert_eq!( - cache - .async_rpush_and_trim("buf", vec!["c".into(), "d".into()], max_len) - .await - .unwrap(), - 4 - ); -} - -#[rstest] -#[tokio::test] -async fn rpush_and_trim_raises_when_a_queued_command_fails() { - let wrong_type = redis::parse_redis_value( - b"-WRONGTYPE Operation against a key holding the wrong kind of value\r\n", - ) - .unwrap(); - let cache = json_cache(vec![MockCmd::with_values( - push_and_trim(-3), - Ok(vec![redis::Value::Array(vec![ - wrong_type, - redis::Value::Okay, - ])]), - )]) - .with_namespace(Some("ns".into())); - - assert!(matches!( - cache - .async_rpush_and_trim("buf", vec!["c".into(), "d".into()], 3) - .await, - Err(Error::Unavailable(Some(_))) - )); - assert_eq!( - cache.async_rpush_and_trim("buf", Vec::new(), 3).await, - Err(Error::InvalidEntry(None)) - ); -} - -#[rstest] -#[tokio::test] -async fn scripts_and_eval_namespace_their_keys() { - let eval = || { - MockCmd::new( - redis::cmd("EVAL") - .arg("return KEYS[1]") - .arg(1usize) - .arg("team:key"), - Ok("team:key"), - ) - }; - let cache = team(vec![eval(), eval()]); - assert_eq!( - cache - .async_eval("return KEYS[1]".into(), vec!["key".into()], Vec::new()) - .await - .unwrap(), - redis::Value::BulkString(b"team:key".to_vec()) - ); - assert_eq!( - cache - .async_register_script("return KEYS[1]".into()) - .invoke(vec!["key".into()], Vec::new()) - .await - .unwrap(), - redis::Value::BulkString(b"team:key".to_vec()) - ); -} - -#[rstest] -fn client_list_and_info_return_server_text() { - let cache = team(vec![ - MockCmd::new(redis::cmd("CLIENT").arg("LIST"), Ok("id=1")), - MockCmd::new(redis::cmd("INFO"), Ok("redis_version:7")), - ]); - assert_eq!(cache.client_list().unwrap(), "id=1"); - assert_eq!(cache.info().unwrap(), "redis_version:7"); -} - -#[rstest] -#[tokio::test] -async fn pipelines_preserve_operation_order() { - let mut rpush_pipeline = redis::pipe(); - rpush_pipeline - .cmd("RPUSH") - .arg("team:a") - .arg("one") - .cmd("RPUSH") - .arg("team:b") - .arg("two"); - let mut lpop_pipeline = redis::pipe(); - lpop_pipeline - .cmd("LPOP") - .arg("team:a") - .arg(2usize) - .cmd("LPOP") - .arg("team:b"); - let queue = team(vec![ - MockCmd::with_values( - rpush_pipeline, - Ok(vec![redis::Value::Int(1), redis::Value::Int(2)]), - ), - MockCmd::new( - redis::cmd("INFO"), - Ok("# Server\r\nredis_version:7.2.4\r\n"), - ), - MockCmd::with_values( - lpop_pipeline, - Ok(vec![redis_test::redis_value!(["one"]), redis::Value::Nil]), - ), - ]); - - assert_eq!( - queue - .async_rpush_pipeline(vec![ - RedisRpushOperation { - key: "a".into(), - values: vec![RedisArg::from("one")], - }, - RedisRpushOperation { - key: "b".into(), - values: vec![RedisArg::from("two")], - }, - ]) - .await - .unwrap(), - [1, 2] - ); - assert_eq!( - queue - .async_lpop_pipeline(vec![ - RedisLpopOperation { - key: "a".into(), - count: Some(2), - }, - RedisLpopOperation { - key: "b".into(), - count: None, - }, - ]) - .await - .unwrap(), - [ - RedisLpopResult::Values(vec![b"one".to_vec()]), - RedisLpopResult::Missing, - ] - ); -} - -/// Below major version 7 a counted `LPOP` is unsupported, so a pipeline that mixes counted and -/// plain pops runs each operation through `async_lpop`: `count` single-`LPOP` pipelines for the -/// counted ones, a bare `LPOP` for the rest. No `LPOP key count` reaches the connection. -#[rstest] -#[tokio::test] -async fn lpop_pipeline_pops_one_at_a_time_below_redis_7() { - let single_pop = |key: &str| { - let mut pipeline = redis::pipe(); - pipeline.cmd("LPOP").arg(key); - pipeline - }; - let cache = team(vec![ - MockCmd::new( - redis::cmd("INFO"), - Ok("# Server\r\nredis_version:6.2.14\r\n"), - ), - MockCmd::with_values( - single_pop("team:a"), - Ok(vec![redis_test::redis_value!("one")]), - ), - MockCmd::with_values( - single_pop("team:a"), - Ok(vec![redis_test::redis_value!("two")]), - ), - MockCmd::new(redis::cmd("LPOP").arg("team:b"), Ok(redis::Value::Nil)), - ]); - - assert_eq!( - cache - .async_lpop_pipeline(vec![ - RedisLpopOperation { - key: "a".into(), - count: Some(2), - }, - RedisLpopOperation { - key: "b".into(), - count: None, - }, - ]) - .await - .unwrap(), - [ - RedisLpopResult::Values(vec![b"one".to_vec(), b"two".to_vec()]), - RedisLpopResult::Missing, - ] - ); -} - -#[rstest] -#[tokio::test] -async fn increment_pipeline_expires_only_operations_with_a_ttl() { - let mut increment_pipeline = redis::pipe(); - increment_pipeline - .cmd("INCRBYFLOAT") - .arg("team:counter") - .arg(1.5f64) - .cmd("EXPIRE") - .arg("team:counter") - .arg(10u64) - .ignore() - .cmd("INCRBYFLOAT") - .arg("team:counter") - .arg(2.0f64); - let counters = team(vec![MockCmd::with_values( - increment_pipeline, - Ok(vec![ - redis::Value::BulkString(b"1.5".to_vec()), - redis::Value::Int(1), - redis::Value::BulkString(b"3.5".to_vec()), - ]), - )]); - assert_eq!( - counters - .async_increment_pipeline(vec![ - IncrementOperation { - key: "counter".into(), - amount: 1.5, - ttl: Some(Duration::from_secs(10)), - }, - IncrementOperation { - key: "counter".into(), - amount: 2.0, - ttl: None, - }, - ]) - .await - .unwrap(), - [1.5, 3.5] - ); -} - -const INCREMENT_SCRIPT: &str = concat!( - "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", - "if redis.call('TTL', KEYS[1]) == -1 then ", - "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" -); -const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!( - "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ", - "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ", - "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return count" -); -const SET_MAX_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if current == false or tonumber(current) < tonumber(ARGV[1]) then ", - "redis.call('SET', KEYS[1], ARGV[1]); ", - "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return ARGV[1]; end; return current" -); - -fn increment_script(amount: f64) -> MockCmd { - MockCmd::new( - redis::cmd("EVAL") - .arg(INCREMENT_SCRIPT) - .arg(1) - .arg("counter") - .arg(amount) - .arg(600), - Ok("4.5"), - ) -} - -#[rstest] -#[tokio::test] -async fn increments_keep_an_existing_ttl_in_one_atomic_script(context: ExactCacheContext) { - let cache = RedisCache::with_connection( - mock(vec![increment_script(2.5), increment_script(2.5)]), - None, - JsonCodec::::new(), - ); - - assert_eq!( - cache - .increment_cache("counter", 2.5, context.clone()) - .unwrap(), - 4.5 - ); - assert_eq!( - cache - .async_increment("counter", 2.5, context, false) - .await - .unwrap(), - 4.5 - ); -} - -#[rstest] -#[case::explicit_ttl(Some(Duration::from_secs(60)), 60u64)] -#[case::default_ttl(None, 600u64)] -#[tokio::test] -async fn refresh_ttl_increments_rearm_the_ttl_in_the_same_round_trip( - #[case] ttl: Option, - #[case] seconds: u64, -) { - let mut pipeline = redis::pipe(); - pipeline - .cmd("INCRBYFLOAT") - .arg("ns:spend:key:k") - .arg(1.5f64) - .cmd("EXPIRE") - .arg("ns:spend:key:k") - .arg(seconds); - let cache = json_cache(vec![MockCmd::with_values( - pipeline, - Ok(vec![ - redis::Value::BulkString(b"1.5".to_vec()), - redis::Value::Int(1), - ]), - )]) - .with_namespace(Some("ns".into())); - - assert_eq!( - cache - .async_increment("spend:key:k", 1.5, ExactCacheContext { ttl }, true) - .await - .unwrap(), - 1.5 - ); -} - -#[rstest] -#[tokio::test] -async fn counter_repairs_are_atomic_and_use_default_ttl() { - let floor = || { - MockCmd::new( - redis::cmd("EVAL") - .arg(INCREMENT_WITH_FLOOR_SCRIPT) - .arg(1) - .arg("team:counter") - .arg(-2i64) - .arg(30u64), - Ok(0i64), - ) - }; - let cache = team(vec![ - floor(), - floor(), - MockCmd::new( - redis::cmd("EVAL") - .arg(SET_MAX_SCRIPT) - .arg(1) - .arg("team:counter") - .arg(4.5f64) - .arg(600u64), - Ok("4.5"), - ), - ]); - - assert_eq!( - cache - .increment_with_floor("counter", -2, Duration::from_secs(30)) - .unwrap(), - 0 - ); - assert_eq!( - cache - .async_increment_with_floor("counter", -2, Duration::from_secs(30)) - .await - .unwrap(), - 0 - ); - assert_eq!( - cache.async_set_max("counter", 4.5, None).await.unwrap(), - 4.5 - ); -} - -const CLAIM_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ", - "elseif current ~= ARGV[1] then return 0; end; ", - "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ", - "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1" -); - -fn claim_eval(expected: &str, write: &str, refresh: bool, applied: i64) -> MockCmd { - MockCmd::new( - redis::cmd("EVAL") - .arg(CLAIM_SCRIPT) - .arg(1) - .arg("pin") - .arg(expected) - .arg(600) - .arg(write) - .arg(u8::from(refresh)), - Ok(applied), - ) -} - -#[rstest] -#[tokio::test] -async fn claims_match_eligible_values_written_by_another_encoder(context: ExactCacheContext) { - let python_payload = r#"{"model_id": "a", "deployment": "east"}"#; - let stored = json!({"deployment": "east", "model_id": "a"}); - let cache = json_cache(vec![ - MockCmd::new(redis::cmd("GET").arg("pin"), Ok(python_payload)), - claim_eval(python_payload, "", true, 1), - ]); - - assert_eq!( - cache - .async_claim_cache( - "pin", - json!({"model_id": "b"}), - vec![stored.clone()], - context - ) - .await - .unwrap(), - stored - ); -} - -#[rstest] -fn claims_retry_when_the_key_changes_and_replace_ineligible_winners(context: ExactCacheContext) { - let candidate = json!({"model_id": "b"}); - let payload = r#"{"model_id":"b"}"#; - let cache = json_cache(vec![ - MockCmd::new(redis::cmd("GET").arg("pin"), Ok(redis::Value::Nil)), - claim_eval("", payload, false, 0), - MockCmd::new(redis::cmd("GET").arg("pin"), Ok(r#"{"model_id":"gone"}"#)), - claim_eval(r#"{"model_id":"gone"}"#, payload, false, 1), - ]); - - assert_eq!( - cache - .claim_cache( - "pin", - candidate.clone(), - &[json!({"model_id": "a"})], - context - ) - .unwrap(), - candidate - ); -} - -#[rstest] -fn claims_without_eligible_values_keep_the_winner_without_refreshing_its_ttl( - context: ExactCacheContext, -) { - let stored = r#"{"model_id": "a"}"#; - let cache = json_cache(vec![ - MockCmd::new(redis::cmd("GET").arg("pin"), Ok(stored)), - claim_eval(stored, "", false, 1), - ]); - - assert_eq!( - cache - .claim_cache("pin", json!({"model_id": "b"}), &[], context) - .unwrap(), - json!({"model_id": "a"}) - ); -} - -#[rstest] -#[tokio::test] -async fn test_connection_reports_success_with_the_python_message() { - let cache = team(vec![MockCmd::new(redis::cmd("PING"), Ok("PONG"))]); - - let result = cache.test_connection().await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Success); - assert_eq!(result.message, "Redis connection test successful"); - assert_eq!(result.error, None); -} - -#[rstest] -#[case::unexpected_reply(Ok("NOPE"), "Redis ping returned False", None)] -#[case::connection_refused( - Err(redis::RedisError::from((redis::ErrorKind::Io, "connection refused"))), - "Redis connection failed:", - Some("connection refused") -)] -#[tokio::test] -async fn test_connection_failures_use_the_python_result_contract( - #[case] reply: redis::RedisResult<&'static str>, - #[case] message: &str, - #[case] cause: Option<&str>, -) { - let cache = json_cache(vec![MockCmd::new(redis::cmd("PING"), reply)]); - - let result = cache.test_connection().await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Failed); - assert!(result.message.starts_with(message), "{}", result.message); - assert_eq!(result.error.is_some(), cause.is_some()); - assert!(cause.is_none_or(|cause| { - result - .error - .as_deref() - .is_some_and(|error| error.contains(cause)) - })); -} - -#[rstest] -#[tokio::test] -async fn disconnect_keeps_a_caller_owned_connection_usable() { - let cache = team(vec![MockCmd::new(redis::cmd("PING"), Ok("PONG"))]); - cache.disconnect().await.unwrap(); - assert!(cache.ping().await.unwrap()); -} - -#[rstest] -#[tokio::test] -async fn disconnect_drains_an_idle_pool_without_connecting() { - let cache = RedisCache::new("redis://127.0.0.1:1", None, JsonCodec::::new()).unwrap(); - cache.disconnect().await.unwrap(); -} diff --git a/litellm-rust/crates/cache-redis/tests/cluster.rs b/litellm-rust/crates/cache-redis/tests/cluster.rs deleted file mode 100644 index e1ba01bcb04..00000000000 --- a/litellm-rust/crates/cache-redis/tests/cluster.rs +++ /dev/null @@ -1,563 +0,0 @@ -//! Tests against a real Redis Cluster. Set `LITELLM_TEST_REDIS_CLUSTER_NODES` to a comma -//! separated `host:port` list (for example `127.0.0.1:7000,127.0.0.1:7001`) to run them. - -mod support; - -use std::{collections::HashSet, time::Duration}; - -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, BoundedCounterCache, BulkDeleteCache, CacheConnectionStatus, - CacheScript, ClaimCache, ClientInfoCache, ConnectionCache, CounterCache, DeleteCache, - DisconnectCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec, - PingCache, QueueCache, RefreshTtlCache, ScanCache, ScriptCache, SetCache, TtlCache, - TtlPipelineCache, -}; -use litellm_cache_redis::{ - RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisNode, RedisRpushOperation, - RedisTopology, -}; -use redis::cluster_routing::Slot; -use rstest::{fixture, rstest}; -use serde_json::json; -use support::{JsonCache, cluster_cache, cluster_url}; - -type Counter = RedisCache>; - -#[fixture] -fn cache(#[default("cache")] label: &str) -> Option { - cluster_cache(label, Duration::from_secs(120), JsonCodec::new()) -} - -#[fixture] -fn counter(#[default("counter")] label: &str) -> Option { - cluster_cache(label, Duration::from_secs(60), JsonCodec::new()) -} - -#[fixture] -fn context() -> ExactCacheContext { - ExactCacheContext::default() -} - -fn multi_slot_keys(count: usize) -> Vec { - let keys: Vec = (0..count).map(|index| format!("key-{index}")).collect(); - let slots: HashSet = keys.iter().map(Slot::for_key).collect(); - assert!(slots.len() > 1, "keys must span multiple slots"); - keys -} - -fn seconds(seconds: u64) -> Option { - Some(Duration::from_secs(seconds)) -} - -#[rstest] -#[case::no_startup_nodes("redis://127.0.0.1:7000", Vec::new())] -#[case::unix_socket_url( - "redis+unix:///tmp/redis.sock", - vec![RedisNode { host: "127.0.0.1".into(), port: 7000 }] -)] -fn constructor_rejects_unusable_cluster_configs( - #[case] url: &str, - #[case] startup_nodes: Vec, -) { - let error = JsonCache::connect( - url, - &RedisTopology::Cluster { startup_nodes }, - None, - JsonCodec::new(), - ) - .err(); - assert!(matches!(error, Some(Error::Unavailable(_)))); -} - -#[rstest] -#[tokio::test] -async fn single_key_operations_round_trip_with_ttl_rounding( - #[with("single")] cache: Option, -) { - let Some(cache) = cache else { return }; - let context = ExactCacheContext { - ttl: Some(Duration::from_millis(1500)), - }; - let keys = multi_slot_keys(12); - for (index, key) in keys.iter().enumerate() { - cache - .set_cache(key, json!({ "index": index }), &context) - .unwrap(); - } - for (index, key) in keys.iter().enumerate() { - assert_eq!( - cache.get_cache(key, &context).unwrap(), - Some(json!({ "index": index })) - ); - } - assert_eq!(cache.async_get_ttl(&keys[0]).await.unwrap(), seconds(2)); - assert!( - cache - .async_refresh_ttl(&keys[0], seconds(40)) - .await - .unwrap() - ); - assert_eq!(cache.async_get_ttl(&keys[0]).await.unwrap(), seconds(40)); - cache.delete_cache(&keys[0]).unwrap(); - assert_eq!(cache.get_cache(&keys[0], &context).unwrap(), None); - assert!(!cache.async_refresh_ttl(&keys[0], None).await.unwrap()); - assert!(cache.sync_ping().unwrap()); - cache.async_flush_cache().await.unwrap(); -} - -#[rstest] -#[tokio::test] -async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries( - #[with("batch")] cache: Option, - context: ExactCacheContext, -) { - let Some(cache) = cache else { return }; - let keys = multi_slot_keys(40); - for (index, key) in keys.iter().enumerate() { - if index % 5 == 0 { - continue; - } - cache - .async_set_cache(key, json!(index), context.clone()) - .await - .unwrap(); - } - let mut raw = redis::cluster::ClusterClient::new(vec![cluster_url()]) - .unwrap() - .get_connection() - .unwrap(); - let malformed = format!("{}:{}", cache.namespace().unwrap(), keys[1]); - redis::cmd("SET") - .arg(&malformed) - .arg("not json") - .exec(&mut raw) - .unwrap(); - - let entries = cache - .async_batch_get_cache(keys.clone(), context.clone()) - .await - .unwrap(); - assert_eq!(entries.len(), keys.len()); - for (index, entry) in entries.iter().enumerate() { - let expected = if index == 1 { - BatchEntry::Invalid - } else if index % 5 == 0 { - BatchEntry::Miss - } else { - BatchEntry::Hit(json!(index)) - }; - assert_eq!(*entry, expected, "entry {index}"); - } - assert_eq!(cache.batch_get_cache(&keys, &context).unwrap(), entries); - - cache.delete_cache_keys(keys.clone()).await.unwrap(); - let entries = cache.async_batch_get_cache(keys, context).await.unwrap(); - assert!(entries.iter().all(|entry| *entry == BatchEntry::Miss)); -} - -#[rstest] -#[tokio::test] -async fn pipelines_group_by_slot_and_return_results_in_submission_order( - #[with("pipeline")] cache: Option, - counter: Option, - context: ExactCacheContext, -) { - let (Some(cache), Some(counter)) = (cache, counter) else { - return; - }; - let keys = multi_slot_keys(30); - cache - .async_set_cache_pipeline( - keys.iter() - .enumerate() - .map(|(index, key)| (key.clone(), json!(index))) - .collect(), - context.clone(), - ) - .await - .unwrap(); - let hits = cache - .async_batch_get_cache(keys.clone(), context.clone()) - .await - .unwrap(); - assert!( - hits.iter() - .enumerate() - .all(|(index, entry)| *entry == BatchEntry::Hit(json!(index))) - ); - - cache - .async_set_cache_pipeline_with_ttls( - keys.iter() - .enumerate() - .map(|(index, key)| (key.clone(), json!(index), seconds(index as u64 + 10))) - .collect(), - ) - .await - .unwrap(); - for (index, key) in keys.iter().enumerate() { - assert_eq!( - cache.async_get_ttl(key).await.unwrap(), - seconds(index as u64 + 10), - "{key}" - ); - } - - let queues: Vec = keys.iter().map(|key| format!("queue:{key}")).collect(); - let pushed = cache - .async_rpush_pipeline( - queues - .iter() - .enumerate() - .map(|(index, key)| RedisRpushOperation { - key: key.clone(), - values: (0..=index) - .map(|value| RedisArg::Integer(value as i64)) - .collect(), - }) - .collect(), - ) - .await - .unwrap(); - assert_eq!(pushed, (1..=keys.len()).collect::>()); - let popped = cache - .async_lpop_pipeline( - queues - .iter() - .enumerate() - .map(|(index, key)| RedisLpopOperation { - key: key.clone(), - count: (index % 2 == 0).then_some(2), - }) - .collect(), - ) - .await - .unwrap(); - for (index, result) in popped.into_iter().enumerate() { - match result { - RedisLpopResult::Value(value) => { - assert_eq!(index % 2, 1, "queue {index}"); - assert_eq!(value, b"0"); - } - RedisLpopResult::Values(values) => { - assert_eq!(index % 2, 0, "queue {index}"); - let expected: Vec> = (0..=index) - .take(2) - .map(|value| value.to_string().into_bytes()) - .collect(); - assert_eq!(values, expected); - } - other => panic!("queue {index}: {other:?}"), - } - } - - let counters: Vec = keys.iter().map(|key| format!("counter:{key}")).collect(); - let totals = counter - .async_increment_pipeline( - counters - .iter() - .enumerate() - .map(|(index, key)| IncrementOperation { - key: key.clone(), - amount: index as f64 + 0.5, - ttl: (index % 3 == 0).then_some(Duration::from_secs(30)), - }) - .collect(), - ) - .await - .unwrap(); - let expected: Vec = (0..keys.len()).map(|index| index as f64 + 0.5).collect(); - assert_eq!(totals, expected); - assert_eq!( - counter.async_get_ttl(&counters[0]).await.unwrap(), - seconds(30) - ); - assert_eq!(counter.async_get_ttl(&counters[1]).await.unwrap(), None); - counter.async_flush_cache().await.unwrap(); - cache.async_flush_cache().await.unwrap(); -} - -#[rstest] -#[tokio::test] -async fn rpush_and_trim_is_one_transaction_on_the_key_slot( - #[with("trim")] cache: Option, -) { - let Some(cache) = cache else { return }; - let values = |values: &[&str]| values.iter().map(|value| RedisArg::from(*value)).collect(); - assert_eq!( - cache - .async_rpush_and_trim("buf", values(&["a", "b"]), 3) - .await - .unwrap(), - 2 - ); - assert_eq!( - cache - .async_rpush_and_trim("buf", values(&["c", "d"]), 3) - .await - .unwrap(), - 4 - ); - assert_eq!( - cache.async_lpop("buf", Some(10)).await.unwrap(), - RedisLpopResult::Values(vec![b"b".to_vec(), b"c".to_vec(), b"d".to_vec()]) - ); - cache.async_flush_cache().await.unwrap(); -} - -#[rstest] -#[tokio::test] -async fn scan_and_scoped_flush_cover_every_primary( - #[with("flush")] cache: Option, - #[from(cache)] - #[with("other")] - other: Option, - context: ExactCacheContext, -) { - let (Some(cache), Some(other)) = (cache, other) else { - return; - }; - let keys = multi_slot_keys(60); - for key in &keys { - cache - .async_set_cache(key, json!(true), context.clone()) - .await - .unwrap(); - other - .async_set_cache(key, json!(true), context.clone()) - .await - .unwrap(); - } - let mut scanned = cache.async_scan_iter("key-", 1000).await.unwrap(); - scanned.sort(); - let mut expected: Vec = keys - .iter() - .map(|key| format!("{}:{key}", cache.namespace().unwrap())) - .collect(); - expected.sort(); - assert_eq!(scanned, expected); - assert_eq!(cache.async_scan_iter("key-", 7).await.unwrap().len(), 7); - - cache.flush_cache().unwrap(); - let flushed = cache - .async_batch_get_cache(keys.clone(), context.clone()) - .await - .unwrap(); - assert!(flushed.iter().all(|entry| *entry == BatchEntry::Miss)); - let kept = other.async_batch_get_cache(keys, context).await.unwrap(); - assert!( - kept.iter() - .all(|entry| *entry == BatchEntry::Hit(json!(true))) - ); - other.async_flush_cache().await.unwrap(); -} - -fn ping_calls_per_node(startup: &redis::Client) -> Vec<(String, u64)> { - let mut connection = startup.get_connection().unwrap(); - let nodes: String = redis::cmd("CLUSTER") - .arg("NODES") - .query(&mut connection) - .unwrap(); - let mut counts: Vec<(String, u64)> = nodes - .lines() - .map(|line| { - let address = line.split_whitespace().nth(1).unwrap(); - let address = address.split('@').next().unwrap(); - let mut node = redis::Client::open(format!("redis://{address}")) - .unwrap() - .get_connection() - .unwrap(); - let stats: String = redis::cmd("INFO") - .arg("commandstats") - .query(&mut node) - .unwrap(); - let calls = stats - .lines() - .find_map(|stat| stat.strip_prefix("cmdstat_ping:calls=")) - .and_then(|rest| rest.split(',').next()) - .map_or(0, |calls| calls.parse().unwrap()); - (address.to_string(), calls) - }) - .collect(); - counts.sort(); - counts -} - -#[rstest] -#[tokio::test] -async fn ping_reaches_every_node(#[with("ping")] cache: Option) { - let Some(cache) = cache else { return }; - let startup = redis::Client::open(cluster_url()).unwrap(); - let before = ping_calls_per_node(&startup); - assert!(before.len() >= 2, "{before:?}"); - assert!(cache.ping().await.unwrap()); - let after = ping_calls_per_node(&startup); - for ((node, calls_before), (_, calls_after)) in before.iter().zip(&after) { - assert!(calls_after > calls_before, "{node} was not pinged"); - } - assert!(cache.sync_ping().unwrap()); - let result = cache.test_connection().await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Success); - assert_eq!(result.message, "Redis Cluster connection test successful"); -} - -#[rstest] -#[tokio::test] -async fn disconnect_closes_idle_connections_and_reconnects_on_demand( - #[with("disconnect")] cache: Option, -) { - let Some(cache) = cache else { return }; - assert!(cache.ping().await.unwrap()); - cache.disconnect().await.unwrap(); - assert!(cache.ping().await.unwrap()); -} - -#[rstest] -#[case::keep_existing_ttl(false)] -#[case::refresh_ttl(true)] -#[tokio::test] -async fn increments_refresh_the_ttl_only_when_asked( - counter: Option, - #[case] refresh_ttl: bool, -) { - let Some(counter) = counter else { return }; - let context = ExactCacheContext { ttl: seconds(60) }; - counter - .async_set_cache("spend", 0.0, ExactCacheContext { ttl: seconds(600) }) - .await - .unwrap(); - assert_eq!( - counter - .async_increment("spend", 1.5, context.clone(), refresh_ttl) - .await - .unwrap(), - 1.5 - ); - assert_eq!( - counter - .async_increment("spend", 2.0, context, refresh_ttl) - .await - .unwrap(), - 3.5 - ); - let ttl = counter.async_get_ttl("spend").await.unwrap().unwrap(); - assert_eq!(ttl <= Duration::from_secs(60), refresh_ttl, "{ttl:?}"); - counter.async_flush_cache().await.unwrap(); -} - -#[rstest] -#[tokio::test] -async fn counters_claims_scripts_and_sets_work_on_the_cluster( - counter: Option, - #[with("claim")] cache: Option, - context: ExactCacheContext, -) { - let (Some(counter), Some(cache)) = (counter, cache) else { - return; - }; - assert_eq!( - counter - .increment_cache("spend", 1.5, context.clone()) - .unwrap(), - 1.5 - ); - assert_eq!( - counter - .async_increment("spend", 2.0, context.clone(), false) - .await - .unwrap(), - 3.5 - ); - assert_eq!( - counter - .increment_with_floor("budget", -3, Duration::from_secs(30)) - .unwrap(), - 0 - ); - assert_eq!( - counter - .async_increment_with_floor("budget", 7, Duration::from_secs(30)) - .await - .unwrap(), - 7 - ); - assert_eq!(counter.async_set_max("peak", 4.0, None).await.unwrap(), 4.0); - assert_eq!(counter.async_set_max("peak", 2.0, None).await.unwrap(), 4.0); - counter.flush_cache().unwrap(); - - let owner = json!("owner-a"); - let rival = json!("owner-b"); - assert_eq!( - cache - .claim_cache("lock", owner.clone(), &[], context.clone()) - .unwrap(), - owner - ); - assert_eq!( - cache - .async_claim_cache("lock", rival.clone(), vec![owner.clone()], context.clone()) - .await - .unwrap(), - owner - ); - assert_eq!( - cache - .claim_cache("lock", rival.clone(), &[], context.clone()) - .unwrap(), - owner - ); - assert_eq!( - cache - .async_claim_cache("lock", rival.clone(), vec![rival.clone()], context.clone()) - .await - .unwrap(), - rival - ); - - let script = cache - .async_register_script("return redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])".into()); - let reply = script - .invoke( - vec!["scripted".into()], - vec![RedisArg::Bytes(b"payload".to_vec()), RedisArg::Integer(5)], - ) - .await - .unwrap(); - assert_eq!(reply, redis::Value::Okay); - assert_eq!(cache.async_get_ttl("scripted").await.unwrap(), seconds(5)); - let evaluated = cache - .async_eval( - "return redis.call('GET', KEYS[1])".into(), - vec!["scripted".into()], - Vec::new(), - ) - .await - .unwrap(); - assert_eq!(evaluated, redis::Value::BulkString(b"payload".to_vec())); - - assert_eq!( - cache - .async_set_cache_sadd( - "members", - vec![ - RedisArg::Bytes(b"a".to_vec()), - RedisArg::Bytes(b"b".to_vec()) - ], - seconds(9), - ) - .await - .unwrap(), - 2 - ); - assert_eq!(cache.async_get_ttl("members").await.unwrap(), seconds(9)); - - let result = cache.test_connection().await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Success); - assert!(cache.ping().await.unwrap()); - let info = cache.info().unwrap(); - assert!(info.matches("redis_version").count() > 1, "{info}"); - assert!(cache.client_list().unwrap().contains("id=")); - cache.async_flush_cache().await.unwrap(); - assert_eq!(cache.async_get_ttl("members").await.unwrap(), None); - assert_eq!(cache.get_cache("lock", &context).unwrap(), None); -} diff --git a/litellm-rust/crates/cache-redis/tests/contract.rs b/litellm-rust/crates/cache-redis/tests/contract.rs deleted file mode 100644 index 85d043b90b2..00000000000 --- a/litellm-rust/crates/cache-redis/tests/contract.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! The shared cache contracts, run against the in-process fake connection and, when -//! `LITELLM_TEST_REDIS_CLUSTER_NODES` is set, against a live Redis Cluster. - -mod support; - -use std::time::Duration; - -use litellm_cache::{ExactCacheContext, JsonCodec}; -use litellm_cache_testing as contract; -use rstest::rstest; -use serde_json::json; -use support::{JsonCache, cluster_cache, fake_cache}; - -const PREFIX: &str = "contract:"; - -#[derive(Clone, Copy, Debug)] -enum Contract { - HitAndMiss, - SyncAsyncEquivalence, - OverwriteReplaces, - PipelineWritesEveryEntry, - BatchPreservesOrder, - DeleteRemovesKey, - FlushClears, - CounterAccumulates, -} - -#[derive(Clone, Copy, Debug)] -enum Server { - Fake, - Cluster, -} - -async fn check(contract: Contract, cache: &JsonCache) -where - C: redis::ConnectionLike + Send + 'static, -{ - let context = ExactCacheContext::default(); - match contract { - Contract::HitAndMiss => { - contract::hit_and_miss(cache, context, PREFIX, json!({"answer": 42})).await - } - Contract::SyncAsyncEquivalence => { - contract::sync_async_equivalence(cache, context, PREFIX, json!("first"), json!([2])) - .await - } - Contract::OverwriteReplaces => { - contract::overwrite_replaces(cache, context, PREFIX, json!(1), json!({"b": 2})).await - } - Contract::PipelineWritesEveryEntry => { - contract::pipeline_writes_every_entry( - cache, - context, - PREFIX, - vec![json!("a"), json!(2), json!({"c": true})], - ) - .await - } - Contract::BatchPreservesOrder => { - contract::batch_preserves_order(cache, context, PREFIX, json!("first"), json!(2)).await - } - Contract::DeleteRemovesKey => { - contract::delete_removes_key(cache, context, PREFIX, json!("value")).await - } - Contract::FlushClears => { - contract::flush_clears(cache, context, PREFIX, json!("value")).await - } - Contract::CounterAccumulates => contract::counter_accumulates(cache, context, PREFIX).await, - } -} - -#[rstest] -#[case::hit_and_miss(Contract::HitAndMiss)] -#[case::sync_async_equivalence(Contract::SyncAsyncEquivalence)] -#[case::overwrite_replaces(Contract::OverwriteReplaces)] -#[case::pipeline_writes_every_entry(Contract::PipelineWritesEveryEntry)] -#[case::batch_preserves_order(Contract::BatchPreservesOrder)] -#[case::delete_removes_key(Contract::DeleteRemovesKey)] -#[case::flush_clears(Contract::FlushClears)] -#[case::counter_accumulates(Contract::CounterAccumulates)] -#[tokio::test] -async fn redis_satisfies_the_cache_contract( - #[case] contract: Contract, - #[values(Server::Fake, Server::Cluster)] server: Server, -) { - match server { - Server::Fake => check(contract, &fake_cache("contract")).await, - Server::Cluster => { - let label = format!("{contract:?}"); - if let Some(cache) = cluster_cache(&label, Duration::from_secs(120), JsonCodec::new()) { - check(contract, &cache).await; - } - } - } -} diff --git a/litellm-rust/crates/cache-redis/tests/support/mod.rs b/litellm-rust/crates/cache-redis/tests/support/mod.rs deleted file mode 100644 index b04e1c29f3b..00000000000 --- a/litellm-rust/crates/cache-redis/tests/support/mod.rs +++ /dev/null @@ -1,231 +0,0 @@ -#![allow(dead_code)] - -use std::{ - collections::BTreeMap, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use litellm_cache::{CacheCodec, Error, JsonCodec}; -use litellm_cache_redis::{RedisCache, RedisNode, RedisTopology}; - -/// Encodes a byte behind a tag, so a value written with another tag decodes as invalid. -pub struct TaggedByteCodec(pub u8); - -impl CacheCodec for TaggedByteCodec { - type Value = u8; - - fn encode(&self, value: &u8) -> Result, Error> { - if *value > 127 { - return Err(Error::InvalidEntry(None)); - } - Ok(vec![self.0, *value]) - } - - fn decode(&self, bytes: &[u8]) -> Result { - match bytes { - [tag, value] if *tag == self.0 => Ok(*value), - _ => Err(Error::InvalidEntry(None)), - } - } -} - -/// A stateful in-process stand-in for a Redis server that understands the string commands the -/// shared contracts exercise, so they run without a live server. TTLs are accepted and ignored. -#[derive(Default)] -pub struct FakeRedis { - strings: BTreeMap, Vec>, -} - -impl FakeRedis { - fn run(&mut self, command: Vec>) -> redis::RedisResult { - let name = String::from_utf8_lossy(&command[0]).to_ascii_uppercase(); - let args = &command[1..]; - Ok(match name.as_str() { - "PING" => redis::Value::SimpleString("PONG".into()), - "SET" | "SETEX" => { - let value = if name == "SET" { &args[1] } else { &args[2] }; - self.strings.insert(args[0].clone(), value.clone()); - redis::Value::Okay - } - "GET" => self.get(&args[0]), - "MGET" => redis::Value::Array(args.iter().map(|key| self.get(key)).collect()), - "DEL" => { - let removed = args - .iter() - .filter(|key| self.strings.remove(*key).is_some()) - .count(); - redis::Value::Int(removed as i64) - } - "SCAN" => { - let pattern = &args[2]; - let keys = self - .strings - .keys() - .filter(|key| glob(pattern, key)) - .map(|key| redis::Value::BulkString(key.clone())) - .collect(); - redis::Value::Array(vec![ - redis::Value::BulkString(b"0".to_vec()), - redis::Value::Array(keys), - ]) - } - "EVAL" if args[0].windows(11).any(|window| window == b"INCRBYFLOAT") => { - self.increment_by_float(&args[2], &args[3]) - } - "INCRBYFLOAT" => self.increment_by_float(&args[0], &args[1]), - _ => { - return Err(redis::RedisError::from(( - redis::ErrorKind::Client, - "unsupported command", - name, - ))); - } - }) - } - - fn get(&self, key: &[u8]) -> redis::Value { - self.strings.get(key).map_or(redis::Value::Nil, |value| { - redis::Value::BulkString(value.clone()) - }) - } - - fn increment_by_float(&mut self, key: &[u8], amount: &[u8]) -> redis::Value { - let current = self - .strings - .get(key) - .map_or(0.0, |value| parse_float(value)); - let total = format!("{}", current + parse_float(amount)); - self.strings - .insert(key.to_vec(), total.clone().into_bytes()); - redis::Value::BulkString(total.into_bytes()) - } -} - -fn parse_float(bytes: &[u8]) -> f64 { - std::str::from_utf8(bytes).unwrap().parse().unwrap() -} - -/// Redis `MATCH` globbing for `*`, `?` and backslash escapes. -fn glob(pattern: &[u8], key: &[u8]) -> bool { - match pattern.split_first() { - None => key.is_empty(), - Some((b'*', rest)) => (0..=key.len()).any(|skip| glob(rest, &key[skip..])), - Some((b'?', rest)) => !key.is_empty() && glob(rest, &key[1..]), - Some((b'\\', [escaped, rest @ ..])) => { - key.first() == Some(escaped) && glob(rest, &key[1..]) - } - Some((literal, rest)) => key.first() == Some(literal) && glob(rest, &key[1..]), - } -} - -/// Splits RESP request bytes into the commands they carry. -fn commands(mut bytes: &[u8]) -> Vec>> { - fn line<'a>(bytes: &mut &'a [u8]) -> &'a [u8] { - let end = bytes - .windows(2) - .position(|window| window == b"\r\n") - .unwrap(); - let (line, rest) = bytes.split_at(end); - *bytes = &rest[2..]; - line - } - fn length(line: &[u8]) -> usize { - std::str::from_utf8(&line[1..]).unwrap().parse().unwrap() - } - let mut commands = Vec::new(); - while !bytes.is_empty() { - let count = length(line(&mut bytes)); - let command = (0..count) - .map(|_| { - let size = length(line(&mut bytes)); - let (argument, rest) = bytes.split_at(size); - bytes = &rest[2..]; - argument.to_vec() - }) - .collect(); - commands.push(command); - } - commands -} - -impl redis::ConnectionLike for FakeRedis { - fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult { - let command = commands(cmd).into_iter().next().unwrap(); - self.run(command) - } - - fn req_packed_commands( - &mut self, - cmd: &[u8], - offset: usize, - count: usize, - ) -> redis::RedisResult> { - let replies = commands(cmd) - .into_iter() - .map(|command| self.run(command)) - .collect::>>()?; - Ok(replies.into_iter().skip(offset).take(count).collect()) - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } -} - -pub type JsonCache = RedisCache, C>; - -pub fn fake_cache(namespace: &str) -> JsonCache { - RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new()) - .with_namespace(Some(namespace.into())) -} - -/// Startup nodes from `LITELLM_TEST_REDIS_CLUSTER_NODES` (`host:port,host:port`); tests that -/// need a live cluster skip when it is unset. -pub fn cluster_topology() -> Option { - let nodes = std::env::var("LITELLM_TEST_REDIS_CLUSTER_NODES").ok()?; - let startup_nodes = nodes - .split(',') - .map(|node| { - let (host, port) = node.trim().rsplit_once(':').expect("host:port"); - RedisNode { - host: host.to_string(), - port: port.parse().expect("port"), - } - }) - .collect(); - Some(RedisTopology::Cluster { startup_nodes }) -} - -pub fn cluster_url() -> String { - std::env::var("LITELLM_TEST_REDIS_CLUSTER_URL") - .unwrap_or_else(|_| "redis://127.0.0.1:7000".into()) -} - -pub fn unique_namespace(label: &str) -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - format!("cluster-test:{label}:{nanos}") -} - -pub fn cluster_cache( - label: &str, - default_ttl: Duration, - codec: S, -) -> Option> { - let topology = cluster_topology()?; - Some( - RedisCache::connect(&cluster_url(), &topology, Some(default_ttl), codec) - .expect("cluster connection") - .with_namespace(Some(unique_namespace(label))), - ) -} diff --git a/litellm-rust/crates/cache-response/Cargo.toml b/litellm-rust/crates/cache-response/Cargo.toml index 8f4570edc6f..46ef6ab92a8 100644 --- a/litellm-rust/crates/cache-response/Cargo.toml +++ b/litellm-rust/crates/cache-response/Cargo.toml @@ -16,9 +16,6 @@ sha2.workspace = true litellm-cache-gcs.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-memory.workspace = true -litellm-cache-redis.workspace = true -redis.workspace = true -redis-test.workspace = true rstest.workspace = true tokio.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/cache-response/tests/connection.rs b/litellm-rust/crates/cache-response/tests/connection.rs index bc24cf56846..f64057f6f1c 100644 --- a/litellm-rust/crates/cache-response/tests/connection.rs +++ b/litellm-rust/crates/cache-response/tests/connection.rs @@ -2,79 +2,11 @@ mod support; use std::{sync::Arc, time::Duration}; -use litellm_cache::CacheConnectionStatus; use litellm_cache_memory::InMemoryCache; -use litellm_cache_response::{ - CacheEntry, ConnectionProbe, ExactResponseCache, ResponseCache, ResponseCacheRequest, -}; -use redis_test::MockCmd; +use litellm_cache_response::{CacheEntry, ExactResponseCache, ResponseCache, ResponseCacheRequest}; use rstest::rstest; use serde_json::json; -use support::{keyed, memory, redis, request}; - -#[rstest] -#[case::reachable( - Ok("PONG"), - CacheConnectionStatus::Success, - "Redis connection test successful", - false -)] -#[case::unexpected_reply( - Ok("NOPE"), - CacheConnectionStatus::Failed, - "Redis ping returned False", - false -)] -#[case::connection_refused( - Err(redis::RedisError::from((redis::ErrorKind::Io, "connection refused"))), - CacheConnectionStatus::Failed, - "Redis connection failed:", - true -)] -#[tokio::test] -async fn connection_backends_are_reachable_as_a_probe( - #[case] reply: redis::RedisResult<&'static str>, - #[case] status: CacheConnectionStatus, - #[case] message: &str, - #[case] has_error: bool, -) { - let probe: Arc = - Arc::new(redis(vec![MockCmd::new(redis::cmd("PING"), reply)], None)); - - let result = probe.test_connection().await.unwrap(); - assert_eq!(result.status, status); - assert!(result.message.starts_with(message), "{}", result.message); - assert_eq!(result.error.is_some(), has_error); -} - -#[rstest] -#[tokio::test] -async fn one_service_serves_both_the_exact_cache_and_its_probe(request: ResponseCacheRequest) { - let service = Arc::new(redis( - vec![ - MockCmd::new(redis::cmd("PING"), Ok("PONG")), - MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(br#"{"timestamp":100.0,"response":{"ok":true}}"#.to_vec()), - ), - ], - Some("tenant"), - )); - let probe: Arc = service.clone(); - let exact: Arc = service; - - assert_eq!( - probe.test_connection().await.unwrap().status, - CacheConnectionStatus::Success - ); - assert_eq!( - exact - .async_lookup(&request, Duration::from_secs(100)) - .await - .unwrap(), - Some(json!({"ok": true})) - ); -} +use support::{keyed, memory, request}; /// The in-memory backend has no `test_connection`, as in Python, and still serves every response /// operation. diff --git a/litellm-rust/crates/cache-response/tests/response.rs b/litellm-rust/crates/cache-response/tests/response.rs index 70ae259d33a..659ba065cf2 100644 --- a/litellm-rust/crates/cache-response/tests/response.rs +++ b/litellm-rust/crates/cache-response/tests/response.rs @@ -17,10 +17,9 @@ use litellm_cache_response::{ CacheControls, CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheRequest, WriteBuffer, cache_key, }; -use redis_test::MockCmd; use rstest::{fixture, rstest}; use serde_json::{Value, json}; -use support::{keyed, memory, redis, request}; +use support::{keyed, memory, request}; type Memory = Arc>>; @@ -331,79 +330,6 @@ async fn directives_skip_io_and_keep_reads_and_writes_independent( ); } -#[rstest] -#[case::python_sync_literal( - br#"{'timestamp': 100.0, 'response': '{"ok": true, "text": "cached"}'}"#.as_slice() -)] -#[case::python_async_json(br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice())] -#[tokio::test] -async fn redis_consumer_reads_python_sync_and_async_envelopes( - request: ResponseCacheRequest, - #[case] stored: &[u8], - #[values(false, true)] asynchronous: bool, -) { - let cache = redis( - vec![MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(stored.to_vec()), - )], - Some("tenant"), - ); - let now = Duration::from_secs(101); - let found = if asynchronous { - cache.async_lookup(&request, now).await.unwrap() - } else { - cache.lookup(&request, now).unwrap() - }; - assert_eq!(found, Some(json!({"ok": true, "text": "cached"}))); -} - -#[rstest] -#[case::object( - json!({"ok": true, "text": "cached"}), - br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice() -)] -#[case::array(json!([1, 2]), br#"{"timestamp":100.0,"response":"[1,2]"}"#.as_slice())] -#[tokio::test] -async fn redis_consumer_writes_python_compatible_json( - request: ResponseCacheRequest, - #[case] response: Value, - #[case] wire: &[u8], -) { - let cache = redis( - vec![MockCmd::new( - redis::cmd("SETEX").arg("tenant:key").arg(600).arg(wire), - Ok("OK"), - )], - Some("tenant"), - ); - cache - .async_store(&request, response, Duration::from_secs(100)) - .await - .unwrap(); -} - -#[rstest] -#[tokio::test] -async fn invalid_entries_are_misses_and_disabled_reads_do_not_touch_redis( - mut request: ResponseCacheRequest, -) { - let cache = redis( - vec![MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(b"invalid".to_vec()), - )], - None, - ); - request.controls.no_cache = true; - assert_eq!(cache.lookup(&request, Duration::ZERO).unwrap(), None); - request.controls.no_cache = false; - assert_eq!( - cache.async_lookup(&request, Duration::ZERO).await.unwrap(), - None - ); -} - #[rstest] #[tokio::test] async fn captured_service_keeps_the_selected_backend_for_background_writes( @@ -541,27 +467,6 @@ async fn batch_lookup_reports_partial_hits_and_batch_store_populates_misses( assert_eq!(memory.lookup(&requests[2], now).unwrap(), None); } -#[rstest] -#[tokio::test] -async fn batch_lookup_with_no_readable_request_skips_the_backend( - #[values(false, true)] asynchronous: bool, -) { - let cache = redis(Vec::new(), None); - let mut request = keyed("key"); - request.controls.no_cache = true; - let requests = [request.clone(), request]; - let partial = if asynchronous { - cache - .async_lookup_batch(&requests, Duration::ZERO) - .await - .unwrap() - } else { - cache.lookup_batch(&requests, Duration::ZERO).unwrap() - }; - assert_eq!(partial.values, vec![None, None]); - assert_eq!(partial.missing_indices, vec![0, 1]); -} - #[rstest] #[tokio::test] async fn deferred_entries_keep_the_time_they_were_produced( diff --git a/litellm-rust/crates/cache-response/tests/support/mod.rs b/litellm-rust/crates/cache-response/tests/support/mod.rs index b992a937b70..f84da303659 100644 --- a/litellm-rust/crates/cache-response/tests/support/mod.rs +++ b/litellm-rust/crates/cache-response/tests/support/mod.rs @@ -1,15 +1,9 @@ use std::{sync::Arc, time::Duration}; use litellm_cache_memory::InMemoryCache; -use litellm_cache_redis::RedisCache; -use litellm_cache_response::{ - CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, -}; -use redis_test::{MockCmd, MockRedisConnection}; +use litellm_cache_response::{CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheRequest}; use rstest::fixture; -pub type MockedRedis = RedisCache; - #[fixture] pub fn memory() -> Arc>> { Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new( @@ -29,12 +23,3 @@ pub fn keyed(key: &str) -> ResponseCacheRequest { ..Default::default() }) } - -/// A Redis response cache that must receive exactly `commands`, in order. -pub fn redis(commands: Vec, namespace: Option<&str>) -> ResponseCache { - let connection = MockRedisConnection::new(commands).assert_all_commands_consumed(); - ResponseCache::new(Arc::new( - RedisCache::with_connection(connection, None, ResponseCacheCodec) - .with_namespace(namespace.map(str::to_owned)), - )) -} diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs index 659390a5659..02707c24881 100644 --- a/litellm-rust/crates/cache-s3/src/auth.rs +++ b/litellm-rust/crates/cache-s3/src/auth.rs @@ -36,9 +36,12 @@ impl ProvideCredentials for S3Credentials { self.config.secret_access_key.clone(), ) { return Ok(AwsCredentials::new( - access_key_id, - secret_access_key, - self.config.session_token.clone(), + access_key_id.expose(), + secret_access_key.expose(), + self.config + .session_token + .as_ref() + .map(|value| value.expose().to_owned()), None, "litellm-s3-cache", )); diff --git a/litellm-rust/crates/cache-s3/tests/auth.rs b/litellm-rust/crates/cache-s3/tests/auth.rs index 59c5aa2e091..5aa251ac749 100644 --- a/litellm-rust/crates/cache-s3/tests/auth.rs +++ b/litellm-rust/crates/cache-s3/tests/auth.rs @@ -5,9 +5,9 @@ use rstest::rstest; fn explicit(session_token: Option<&str>) -> AwsAuthConfig { AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - session_token: session_token.map(str::to_string), + access_key_id: Some("key".into()), + secret_access_key: Some("secret".into()), + session_token: session_token.map(Into::into), region_name: Some("us-east-1".to_string()), ..Default::default() } diff --git a/litellm-rust/crates/cache-s3/tests/support/mod.rs b/litellm-rust/crates/cache-s3/tests/support/mod.rs index 7378dde21bf..311d2402e72 100644 --- a/litellm-rust/crates/cache-s3/tests/support/mod.rs +++ b/litellm-rust/crates/cache-s3/tests/support/mod.rs @@ -23,8 +23,8 @@ pub fn config(endpoint: &str) -> S3CacheConfig { url: endpoint.to_string(), }), auth: AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), + access_key_id: Some("key".into()), + secret_access_key: Some("secret".into()), region_name: Some("us-east-1".to_string()), ..Default::default() }, diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml deleted file mode 100644 index 5b4e99de382..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "litellm-cache-valkey-semantic" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -litellm-cache.workspace = true -litellm-cache-redis.workspace = true -redis = { workspace = true, features = ["tls-rustls"] } -sha2.workspace = true -thiserror.workspace = true -uuid = { version = "1", features = ["v4"] } - -[dev-dependencies] -litellm-error.workspace = true -litellm-cache-testing.workspace = true -redis-test.workspace = true -rstest.workspace = true -serde_json.workspace = true -tokio.workspace = true diff --git a/litellm-rust/crates/cache-valkey-semantic/src/cache.rs b/litellm-rust/crates/cache-valkey-semantic/src/cache.rs deleted file mode 100644 index 947c0edc479..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/src/cache.rs +++ /dev/null @@ -1,245 +0,0 @@ -use std::{ - sync::{Arc, Mutex}, - time::Duration, -}; - -use litellm_cache::{ - BaseCache, CacheCodec, Error, SemanticCacheContext, - semantic::{Embedder, SemanticCache, SemanticLookup, prompt_from_context}, -}; -use litellm_cache_redis::{RedisTopology, connection::Connections}; - -use crate::{ - ValkeySemanticConfig, - index::IndexState, - search::{embedding_bytes, scope_tag, search_document, write_document}, -}; - -/// `ValkeySemanticCache`: a semantic cache on valkey-search's TAG + VECTOR index. Values go -/// through the injected codec, so the response layer decides what a cached entry is. -pub struct ValkeySemanticCache { - connections: Arc>, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - index_dimension: Arc>>, -} - -impl ValkeySemanticCache { - pub fn new( - url: &str, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - ) -> Result { - Ok(Self { - connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), - embedder, - codec, - config, - index_dimension: Arc::new(Mutex::new(None)), - }) - } -} - -impl ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn with_connection( - connection: C, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - ) -> Self { - Self { - connections: Arc::new(Connections::fixed(connection)), - embedder, - codec, - config, - index_dimension: Arc::new(Mutex::new(None)), - } - } - - pub fn similarity_threshold(&self) -> f64 { - self.config.similarity_threshold - } - - pub fn index_name(&self) -> &str { - &self.config.index_name - } - - fn index_state(&self) -> IndexState { - IndexState { - name: self.config.index_name.clone(), - prefix: format!("{}:", self.config.index_name), - dimension: Arc::clone(&self.index_dimension), - similarity_threshold: self.config.similarity_threshold, - } - } - - fn decode(&self, lookup: SemanticLookup>) -> Result, Error> { - Ok(SemanticLookup { - value: lookup - .value - .map(|bytes| self.codec.decode(&bytes)) - .transpose()?, - similarity: lookup.similarity, - }) - } -} - -impl ValkeySemanticCache -where - E: Embedder, - S: CacheCodec + Clone, - C: redis::ConnectionLike + Send + 'static, -{ - /// The same index and connections behind a different embedder. - pub fn with_embedder(&self, embedder: E2) -> ValkeySemanticCache { - ValkeySemanticCache { - connections: Arc::clone(&self.connections), - embedder, - codec: self.codec.clone(), - config: self.config.clone(), - index_dimension: Arc::clone(&self.index_dimension), - } - } -} - -impl BaseCache for ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Value = S::Value; - type Context = SemanticCacheContext; - - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &Self::Context, - ) -> Result<(), Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(()); - }; - let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let response = self.codec.encode(&value)?; - let index = self.index_state(); - self.connections.execute(|connection| { - write_document( - connection, - &index, - &scope_tag(key), - &prompt, - response, - embedding_bytes(&embedding), - self.get_ttl(context), - ) - }) - } - - fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { - self.get_cache_with_similarity(key, context) - .map(|lookup| lookup.value) - } - - async fn async_set_cache( - &self, - key: &str, - value: Self::Value, - context: Self::Context, - ) -> Result<(), Error> { - let Some(prompt) = prompt_from_context(&context) else { - return Ok(()); - }; - let embedding = self - .embedder - .async_embed(&prompt, context.metadata.as_ref()) - .await?; - let response = self.codec.encode(&value)?; - let index = self.index_state(); - let scope = scope_tag(key); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - write_document( - connection, - &index, - &scope, - &prompt, - response, - embedding_bytes(&embedding), - context.ttl, - ) - }) - .await - } - - async fn async_get_cache( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - self.async_get_cache_with_similarity(key, context) - .await - .map(|lookup| lookup.value) - } -} - -/// Python stamps a similarity of `0.0` when there is no prompt or no document in the key's -/// scope, and the closest document's similarity even when it misses the threshold. -impl SemanticCache for ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn get_cache_with_similarity( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let index = self.index_state(); - let lookup = self.connections.execute(|connection| { - search_document( - connection, - &index, - &scope_tag(key), - embedding_bytes(&embedding), - ) - })?; - self.decode(lookup) - } - - async fn async_get_cache_with_similarity( - &self, - key: &str, - context: &Self::Context, - ) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let embedding = self - .embedder - .async_embed(&prompt, context.metadata.as_ref()) - .await?; - let index = self.index_state(); - let scope = scope_tag(key); - let lookup = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - search_document(connection, &index, &scope, embedding_bytes(&embedding)) - }) - .await?; - self.decode(lookup) - } -} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/config.rs b/litellm-rust/crates/cache-valkey-semantic/src/config.rs deleted file mode 100644 index c417652c27b..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/src/config.rs +++ /dev/null @@ -1,8 +0,0 @@ -/// `ValkeySemanticCache.DEFAULT_VALKEY_INDEX_NAME`. -pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; - -#[derive(Clone, Debug, PartialEq)] -pub struct ValkeySemanticConfig { - pub similarity_threshold: f64, - pub index_name: String, -} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/index.rs b/litellm-rust/crates/cache-valkey-semantic/src/index.rs deleted file mode 100644 index 3dfff32ff7c..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/src/index.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use litellm_cache::Error; -use litellm_cache_redis::connection::ConnectionRef; - -use crate::search::value_text; - -/// The valkey-search index one cache writes to, with the dimension it was last ensured for. -#[derive(Clone)] -pub(crate) struct IndexState { - pub(crate) name: String, - pub(crate) prefix: String, - pub(crate) dimension: Arc>>, - pub(crate) similarity_threshold: f64, -} - -/// `_ensure_index_sync` / `_ensure_index_async`: create the TAG + HNSW index once per dimension, -/// and accept an existing index unless it reports a different dimension. -pub(crate) fn ensure_index( - connection: &mut ConnectionRef<'_>, - index: &IndexState, - dimension: usize, -) -> Result<(), Error> { - if index - .dimension - .lock()? - .is_some_and(|existing| existing == dimension) - { - return Ok(()); - } - let create = redis::cmd("FT.CREATE") - .arg(&index.name) - .arg("ON") - .arg("HASH") - .arg("PREFIX") - .arg(1) - .arg(&index.prefix) - .arg("SCHEMA") - .arg("litellm_cache_key") - .arg("TAG") - .arg("embedding") - .arg("VECTOR") - .arg("HNSW") - .arg(6) - .arg("TYPE") - .arg("FLOAT32") - .arg("DIM") - .arg(dimension) - .arg("DISTANCE_METRIC") - .arg("COSINE") - .query::(connection) - .map(|_| ()); - if let Err(error) = create { - if !index_exists(&error) { - return Err(Error::unavailable(error)); - } - let info = redis::cmd("FT.INFO") - .arg(&index.name) - .query::(connection) - .map_err(Error::unavailable)?; - if let Some(existing) = - index_dimension_from_info(&info).filter(|existing| *existing != dimension) - { - return Err(Error::unavailable(DimensionMismatch { - existing, - dimension, - })); - } - } - *index.dimension.lock()? = Some(dimension); - Ok(()) -} - -fn index_exists(error: &redis::RedisError) -> bool { - error.code().is_some() - && error - .detail() - .is_some_and(|detail| detail.to_ascii_lowercase().contains("already exists")) -} - -#[derive(Debug, thiserror::Error)] -#[error("index exists with dimension {existing}, not {dimension}")] -struct DimensionMismatch { - existing: usize, - dimension: usize, -} - -/// `_extract_index_dim`: flatten each attribute one level and read the value after -/// `dimensions`. -fn index_dimension_from_info(value: &redis::Value) -> Option { - let redis::Value::Array(values) = value else { - return None; - }; - let attributes = values.windows(2).find_map(|pair| { - (value_text(&pair[0]).as_deref() == Some("attributes")).then_some(&pair[1]) - })?; - let redis::Value::Array(fields) = attributes else { - return None; - }; - fields.iter().find_map(|field| { - let redis::Value::Array(values) = field else { - return None; - }; - let values = values - .iter() - .flat_map(|value| match value { - redis::Value::Array(values) => values.as_slice(), - _ => std::slice::from_ref(value), - }) - .collect::>(); - values.windows(2).find_map(|pair| { - (value_text(pair[0]).as_deref() == Some("dimensions")) - .then(|| value_text(pair[1]).and_then(|value| value.parse().ok())) - .flatten() - }) - }) -} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs deleted file mode 100644 index f2f62bc95bd..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod cache; -mod config; -mod index; -mod search; - -pub use cache::ValkeySemanticCache; -pub use config::{DEFAULT_INDEX_NAME, ValkeySemanticConfig}; diff --git a/litellm-rust/crates/cache-valkey-semantic/src/search.rs b/litellm-rust/crates/cache-valkey-semantic/src/search.rs deleted file mode 100644 index c5e4e748b58..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/src/search.rs +++ /dev/null @@ -1,161 +0,0 @@ -use std::time::Duration; - -use litellm_cache::{Error, semantic::SemanticLookup}; -use litellm_cache_redis::connection::ConnectionRef; -use sha2::{Digest, Sha256}; -use uuid::Uuid; - -use crate::index::{IndexState, ensure_index}; - -/// `_scope_tag`: valkey-search TAG fields cannot match arbitrary keys verbatim, so scopes are -/// the key's lowercase SHA-256. -pub(crate) fn scope_tag(key: &str) -> String { - let digest = Sha256::digest(key.as_bytes()); - digest.iter().map(|byte| format!("{byte:02x}")).collect() -} - -pub(crate) fn embedding_bytes(embedding: &[f32]) -> Vec { - embedding - .iter() - .flat_map(|value| value.to_le_bytes()) - .collect() -} - -/// `HSET` a fresh `:` document, then `EXPIRE` it when a TTL is set. -pub(crate) fn write_document( - connection: &mut ConnectionRef<'_>, - index: &IndexState, - scope: &str, - prompt: &str, - response: Vec, - vector: Vec, - ttl: Option, -) -> Result<(), Error> { - ensure_index(connection, index, vector.len() / size_of::())?; - let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4()); - let mut pipeline = redis::pipe(); - pipeline - .cmd("HSET") - .arg(&document) - .arg("litellm_cache_key") - .arg(scope) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(response) - .arg("embedding") - .arg(vector) - .ignore(); - if let Some(ttl) = ttl { - pipeline - .cmd("EXPIRE") - .arg(&document) - .arg(ttl.as_secs()) - .ignore(); - } - pipeline.query::<()>(connection).map_err(Error::unavailable) -} - -/// The KNN-1 search within `scope`: the closest document's similarity, and its stored response -/// when that similarity reaches the threshold. No document reads as a similarity of `0.0`. -pub(crate) fn search_document( - connection: &mut ConnectionRef<'_>, - index: &IndexState, - scope: &str, - vector: Vec, -) -> Result>, Error> { - ensure_index(connection, index, vector.len() / size_of::())?; - let query = - format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"); - let response = redis::cmd("FT.SEARCH") - .arg(&index.name) - .arg(query) - .arg("PARAMS") - .arg(2) - .arg("vec") - .arg(vector) - .arg("RETURN") - .arg(2) - .arg("response") - .arg("vector_distance") - .arg("DIALECT") - .arg(2) - .query::(connection) - .map_err(Error::unavailable)?; - let Some(fields) = search_fields(response)? else { - return Ok(SemanticLookup::miss(Some(0.0))); - }; - let field = |name: &str| { - fields - .iter() - .find_map(|(field, value)| (field == name).then(|| value.clone())) - .ok_or(Error::InvalidEntry(None)) - }; - let response = field("response")?; - let similarity = 1.0 - parse_f64(&field("vector_distance")?)?; - Ok(SemanticLookup { - value: (similarity >= index.similarity_threshold).then_some(response), - similarity: Some(similarity), - }) -} - -type SearchFields = Vec<(String, Vec)>; - -fn search_fields(value: redis::Value) -> Result, Error> { - let redis::Value::Array(values) = value else { - return Err(Error::InvalidEntry(None)); - }; - let total = parse_i64(values.first().ok_or(Error::InvalidEntry(None))?)?; - if total <= 0 || values.len() < 3 { - return Ok(None); - } - let redis::Value::Array(fields) = &values[2] else { - return Err(Error::InvalidEntry(None)); - }; - let (pairs, remainder) = fields.as_chunks::<2>(); - if !remainder.is_empty() { - return Err(Error::InvalidEntry(None)); - } - pairs - .iter() - .map(|pair| { - Ok(( - value_text(&pair[0]).ok_or(Error::InvalidEntry(None))?, - value_bytes(&pair[1])?, - )) - }) - .collect::, Error>>() - .map(Some) -} - -fn parse_i64(value: &redis::Value) -> Result { - value_text(value) - .ok_or(Error::InvalidEntry(None))? - .parse() - .map_err(Error::invalid_entry) -} - -fn parse_f64(value: &[u8]) -> Result { - std::str::from_utf8(value) - .map_err(Error::invalid_entry)? - .parse() - .map_err(Error::invalid_entry) -} - -pub(crate) fn value_text(value: &redis::Value) -> Option { - match value { - redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), - redis::Value::SimpleString(value) => Some(value.clone()), - redis::Value::Int(value) => Some(value.to_string()), - _ => None, - } -} - -fn value_bytes(value: &redis::Value) -> Result, Error> { - match value { - redis::Value::BulkString(bytes) => Ok(bytes.clone()), - redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()), - redis::Value::Int(value) => Ok(value.to_string().into_bytes()), - _ => Err(Error::InvalidEntry(None)), - } -} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs deleted file mode 100644 index 10e32c0183b..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs +++ /dev/null @@ -1,430 +0,0 @@ -mod support; - -use std::{ - sync::{Arc, Mutex}, - time::Duration, -}; - -use litellm_cache::{ - BaseCache, Error, JsonCodec, SemanticCacheContext, - semantic::{PreparedEmbedding, SemanticCache, SemanticLookup}, -}; -use litellm_cache_valkey_semantic::{ - DEFAULT_INDEX_NAME, ValkeySemanticCache, ValkeySemanticConfig, -}; -use redis_test::MockRedisConnection; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use support::{EmbedCalls, FakeEmbedder, RecordingConnection}; - -type Requests = Arc>>>; -type RecordingCache = ValkeySemanticCache, RecordingConnection>; - -const KEY_SCOPE: &str = "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"; - -struct Recording { - cache: RecordingCache, - requests: Requests, - calls: EmbedCalls, -} - -impl Recording { - fn text(&self) -> String { - self.requests - .lock() - .unwrap() - .iter() - .map(|request| String::from_utf8_lossy(request).into_owned()) - .collect::>() - .join("\n") - } -} - -fn config() -> ValkeySemanticConfig { - ValkeySemanticConfig { - similarity_threshold: 0.8, - index_name: "test".into(), - } -} - -fn recording(replies: impl IntoIterator>) -> Recording { - let connection = RecordingConnection::new(replies); - let requests = connection.requests(); - let embedder = FakeEmbedder::new(&[]); - let calls = Arc::clone(&embedder.calls); - Recording { - cache: ValkeySemanticCache::with_connection( - connection, - embedder, - JsonCodec::new(), - config(), - ), - requests, - calls, - } -} - -#[fixture] -fn entry() -> Value { - json!({"timestamp": 1.0, "response": {"answer": "ok"}}) -} - -#[fixture] -fn context() -> SemanticCacheContext { - SemanticCacheContext { - messages: Some(json!([{"role": "user", "content": "hello"}])), - metadata: Some(json!({"source": "test"})), - ..Default::default() - } -} - -fn ok() -> redis::RedisResult { - Ok(redis::Value::SimpleString("OK".into())) -} - -fn already_exists() -> redis::RedisResult { - Err(support::server_error("Index already exists")) -} - -fn bulk(value: &[u8]) -> redis::Value { - redis::Value::BulkString(value.to_vec()) -} - -fn search_hit(response: &[u8], distance: &str) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::Int(1), - bulk(b"test:document"), - redis::Value::Array(vec![ - bulk(b"response"), - bulk(response), - bulk(b"vector_distance"), - bulk(distance.as_bytes()), - ]), - ]) -} - -fn encoded(value: &Value) -> Vec { - serde_json::to_vec(value).unwrap() -} - -/// `FT.INFO` with the vector field's dimension nested one level down, as valkey-search reports. -fn nested_dimension_info(dimension: i64) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::SimpleString("attributes".into()), - redis::Value::Array(vec![redis::Value::Array(vec![ - redis::Value::SimpleString("embedding".into()), - redis::Value::Array(vec![ - redis::Value::SimpleString("dimensions".into()), - redis::Value::Int(dimension), - ]), - ])]), - ]) -} - -/// `FT.INFO` with `dimensions` as a sibling string of the identifier. -fn flat_dimension_info(dimension: &str) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::SimpleString("attributes".into()), - redis::Value::Array(vec![redis::Value::Array(vec![ - redis::Value::SimpleString("identifier".into()), - redis::Value::SimpleString("embedding".into()), - redis::Value::Array(vec![ - redis::Value::SimpleString("dimensions".into()), - redis::Value::SimpleString(dimension.into()), - ]), - ])]), - ]) -} - -#[rstest] -#[case::string_content(json!([{"content": "hello"}]), None, Some("hello"))] -#[case::text_parts(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] -#[case::non_object_parts_are_skipped(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("hello"))] -#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] -#[case::responses_string_input(json!([]), Some(json!(" hello ")), Some("hello"))] -#[case::responses_item_input(json!([]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] -#[case::blank_input(json!([]), Some(json!(" ")), None)] -fn prompt_shapes_follow_redis_semantic_extraction( - #[case] messages: Value, - #[case] input: Option, - #[case] expected: Option<&str>, -) { - let recording = recording([ok(), Ok(redis::Value::Array(vec![redis::Value::Int(0)]))]); - let context = SemanticCacheContext { - messages: Some(messages), - input, - ..Default::default() - }; - - assert_eq!(recording.cache.get_cache("key", &context).unwrap(), None); - - let prompts = recording - .calls - .lock() - .unwrap() - .iter() - .map(|(prompt, _)| prompt.clone()) - .collect::>(); - assert_eq!( - prompts, - expected.into_iter().map(str::to_owned).collect::>() - ); - assert_eq!( - recording.requests.lock().unwrap().is_empty(), - expected.is_none() - ); -} - -#[rstest] -#[case::key("key", KEY_SCOPE)] -#[case::empty_key("", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")] -fn documents_are_scoped_by_the_keys_sha256( - #[case] key: &str, - #[case] scope: &str, - entry: Value, - context: SemanticCacheContext, -) { - let recording = recording([ok()]); - - recording.cache.set_cache(key, entry, &context).unwrap(); - - let text = recording.text(); - assert!(text.contains(&format!("test:{scope}:"))); - assert!(text.contains(&format!("litellm_cache_key\r\n$64\r\n{scope}\r\n"))); -} - -#[rstest] -#[case::no_ttl(None, None)] -#[case::whole_seconds(Some(Duration::from_secs(5)), Some("5"))] -#[case::fractional_seconds_truncate(Some(Duration::from_millis(1900)), Some("1"))] -fn set_writes_hset_and_expires_only_with_a_ttl( - #[case] ttl: Option, - #[case] expire: Option<&str>, - entry: Value, - context: SemanticCacheContext, -) { - let recording = recording([ok()]); - - recording - .cache - .set_cache( - "key", - entry, - &SemanticCacheContext { - ttl, - ..context.clone() - }, - ) - .unwrap(); - - let text = recording.text(); - assert!(text.contains("FT.CREATE")); - assert!(text.contains("HSET")); - let expire_seconds = text - .split_once("EXPIRE\r\n") - .and_then(|(_, rest)| rest.split("\r\n").nth(3)); - assert_eq!(expire_seconds, expire); - assert_eq!( - *recording.calls.lock().unwrap(), - vec![("hello".to_owned(), context.metadata)] - ); -} - -#[rstest] -fn second_set_skips_create_after_dimension_is_cached(entry: Value, context: SemanticCacheContext) { - let recording = recording([ok()]); - - recording - .cache - .set_cache("key", entry.clone(), &context) - .unwrap(); - recording.cache.set_cache("key", entry, &context).unwrap(); - - let text = recording.text(); - assert_eq!(text.matches("FT.CREATE").count(), 1); - assert_eq!(text.matches("HSET").count(), 2); -} - -#[rstest] -#[case::nested_matching(nested_dimension_info(3), Ok(()))] -#[case::nested_mismatch( - nested_dimension_info(2), - Err("cache is unavailable: index exists with dimension 2, not 3") -)] -#[case::flat_matching(flat_dimension_info("3"), Ok(()))] -#[case::flat_mismatch( - flat_dimension_info("2"), - Err("cache is unavailable: index exists with dimension 2, not 3") -)] -#[case::unreported_dimension_is_accepted(redis::Value::Array(vec![]), Ok(()))] -fn existing_index_dimension_must_match_embedding( - #[case] info: redis::Value, - #[case] expected: Result<(), &str>, - entry: Value, - context: SemanticCacheContext, -) { - let recording = recording([already_exists(), Ok(info)]); - - let result = recording.cache.set_cache("key", entry, &context); - assert_eq!( - result.map_err(|error| litellm_error::Report::new(&error).to_string()), - expected.map_err(str::to_owned) - ); -} - -#[rstest] -#[case::create_failure(Err(redis::RedisError::from((redis::ErrorKind::Io, "boom"))))] -fn index_creation_failures_are_unavailable( - #[case] reply: redis::RedisResult, - entry: Value, - context: SemanticCacheContext, -) { - let recording = recording([reply]); - - assert!(matches!( - recording.cache.set_cache("key", entry, &context), - Err(Error::Unavailable(Some(_))) - )); -} - -#[rstest] -#[case::within_threshold(search_hit(&encoded(&entry()), "0.1"), Ok(Some(entry())))] -#[case::at_threshold(search_hit(&encoded(&entry()), "0.2"), Ok(Some(entry())))] -#[case::beyond_threshold(search_hit(&encoded(&entry()), "0.5"), Ok(None))] -#[case::zero_documents(redis::Value::Array(vec![redis::Value::Int(0)]), Ok(None))] -#[case::missing_response( - redis::Value::Array(vec![ - redis::Value::Int(1), - bulk(b"document"), - redis::Value::Array(vec![bulk(b"vector_distance"), bulk(b"0.1")]), - ]), - Err(Error::InvalidEntry(None)) -)] -#[case::unparsable_distance(search_hit(b"not-json", "abc"), Err(Error::InvalidEntry(None)))] -#[case::undecodable_response(search_hit(b"not-json", "0.1"), Err(Error::InvalidEntry(None)))] -fn get_applies_threshold_and_decodes_entry( - #[case] reply: redis::Value, - #[case] expected: Result, Error>, - context: SemanticCacheContext, -) { - let recording = recording([ok(), Ok(reply)]); - - let kind = |result: Result, Error>| { - result.map_err(|error| std::mem::discriminant(&error)) - }; - assert_eq!( - kind(recording.cache.get_cache("key", &context)), - kind(expected) - ); -} - -#[rstest] -#[case::hit(context(), Some(search_hit(&encoded(&entry()), "0.1")), Some(entry()), Some(1.0 - 0.1))] -#[case::below_threshold(context(), Some(search_hit(&encoded(&entry()), "0.5")), None, Some(1.0 - 0.5))] -#[case::no_results(context(), Some(redis::Value::Array(vec![redis::Value::Int(0)])), None, Some(0.0))] -#[case::no_prompt(SemanticCacheContext::default(), None, None, Some(0.0))] -#[tokio::test] -async fn lookup_reports_python_semantic_similarity( - #[case] context: SemanticCacheContext, - #[case] reply: Option, - #[case] value: Option, - #[case] similarity: Option, - #[values(false, true)] use_async: bool, -) { - let searched = reply.is_some(); - let recording = recording(reply.map_or_else(Vec::new, |reply| vec![ok(), Ok(reply)])); - - let lookup = if use_async { - recording - .cache - .async_get_cache_with_similarity("key", &context) - .await - } else { - recording.cache.get_cache_with_similarity("key", &context) - }; - - assert_eq!(lookup, Ok(SemanticLookup { value, similarity })); - assert_eq!(recording.text().contains("FT.SEARCH"), searched); -} - -#[rstest] -#[tokio::test] -async fn missing_prompt_does_not_touch_valkey(entry: Value) { - let cache = ValkeySemanticCache::with_connection( - MockRedisConnection::new([]).assert_all_commands_consumed(), - FakeEmbedder::new(&[]), - JsonCodec::new(), - config(), - ); - let context = SemanticCacheContext::default(); - - cache.set_cache("key", entry.clone(), &context).unwrap(); - assert_eq!(cache.get_cache("key", &context).unwrap(), None); - cache - .async_set_cache("key", entry, context.clone()) - .await - .unwrap(); - assert_eq!(cache.async_get_cache("key", &context).await.unwrap(), None); - assert_eq!(cache.get_ttl(&context), None); -} - -#[rstest] -fn with_embedder_shares_index_state_and_connections(entry: Value, context: SemanticCacheContext) { - let recording = recording([ok(), ok(), Ok(search_hit(&encoded(&entry), "0.1"))]); - recording - .cache - .set_cache("key", entry.clone(), &context) - .unwrap(); - - let prepared = recording - .cache - .with_embedder(PreparedEmbedding(vec![0.1, 0.2, 0.3])); - - assert_eq!(prepared.get_cache("key", &context).unwrap(), Some(entry)); - assert_eq!(recording.text().matches("FT.CREATE").count(), 1); -} - -#[rstest] -fn accessors_report_the_config() { - let recording = recording([]); - - assert_eq!(recording.cache.index_name(), "test"); - assert_eq!(recording.cache.similarity_threshold(), 0.8); - assert_eq!(DEFAULT_INDEX_NAME, "litellm_semantic_cache_index"); -} - -#[rstest] -#[tokio::test] -async fn async_set_and_get_use_shared_document_helpers( - entry: Value, - context: SemanticCacheContext, -) { - let recording = recording([ok(), ok(), ok(), Ok(search_hit(&encoded(&entry), "0.1"))]); - let context = SemanticCacheContext { - ttl: Some(Duration::from_millis(1900)), - ..context - }; - - recording - .cache - .async_set_cache("key", entry.clone(), context.clone()) - .await - .unwrap(); - assert_eq!( - recording - .cache - .async_get_cache("key", &context) - .await - .unwrap(), - Some(entry) - ); - - let text = recording.text(); - assert!(text.contains("FT.CREATE")); - assert!(text.contains("HSET")); - assert!(text.contains("EXPIRE")); - assert_eq!( - *recording.calls.lock().unwrap(), - vec![("hello".to_owned(), context.metadata.clone()); 2] - ); -} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs b/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs deleted file mode 100644 index 71081fca92b..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! `overwrite_replaces` does not apply: like Python, every write is a new `:` -//! document, so a second write with the same prompt adds a tie instead of replacing the first. - -mod support; - -use litellm_cache::{JsonCodec, SemanticCacheContext, semantic::PreparedEmbedding}; -use litellm_cache_testing as contract; -use litellm_cache_valkey_semantic::{ - DEFAULT_INDEX_NAME, ValkeySemanticCache, ValkeySemanticConfig, -}; -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; -use support::FakeSearch; - -type Cache = ValkeySemanticCache, FakeSearch>; - -const PREFIX: &str = "contract:"; - -#[fixture] -fn cache() -> Cache { - ValkeySemanticCache::with_connection( - FakeSearch::default(), - PreparedEmbedding(vec![0.6, 0.8]), - JsonCodec::new(), - ValkeySemanticConfig { - similarity_threshold: 0.9, - index_name: DEFAULT_INDEX_NAME.into(), - }, - ) -} - -#[fixture] -fn context() -> SemanticCacheContext { - SemanticCacheContext { - messages: Some(json!([{"role": "user", "content": "contract prompt"}])), - ..Default::default() - } -} - -#[rstest] -#[tokio::test] -async fn hit_and_miss(cache: Cache, context: SemanticCacheContext) { - contract::hit_and_miss(&cache, context, PREFIX, json!({"answer": 42})).await; -} - -#[rstest] -#[tokio::test] -async fn sync_async_equivalence(cache: Cache, context: SemanticCacheContext) { - contract::sync_async_equivalence(&cache, context, PREFIX, json!("first"), json!([2])).await; -} - -#[rstest] -#[tokio::test] -async fn pipeline_writes_every_entry(cache: Cache, context: SemanticCacheContext) { - contract::pipeline_writes_every_entry( - &cache, - context, - PREFIX, - vec![json!("a"), json!(2), json!({"c": true})], - ) - .await; -} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs deleted file mode 100644 index 2af05943c98..00000000000 --- a/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs +++ /dev/null @@ -1,352 +0,0 @@ -#![allow(dead_code)] - -use std::{ - collections::{BTreeMap, HashMap, VecDeque}, - sync::{Arc, Mutex}, -}; - -use litellm_cache::{Error, semantic::Embedder}; -use serde_json::Value; - -pub type EmbedCalls = Arc)>>>; - -/// Embeds known prompts to fixed vectors, anything else to `[0.1, 0.2, 0.3]`, and records every -/// prompt with its metadata. -pub struct FakeEmbedder { - vectors: HashMap>, - pub calls: EmbedCalls, -} - -impl FakeEmbedder { - pub fn new(vectors: &[(&str, &[f32])]) -> Self { - Self { - vectors: vectors - .iter() - .map(|(prompt, vector)| ((*prompt).to_owned(), vector.to_vec())) - .collect(), - calls: EmbedCalls::default(), - } - } -} - -impl Embedder for FakeEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - self.calls - .lock() - .unwrap() - .push((prompt.to_owned(), metadata.cloned())); - Ok(self - .vectors - .get(prompt) - .cloned() - .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) - } - - async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - self.embed(prompt, metadata) - } -} - -struct FakeIndex { - prefix: Vec, - dims: usize, - vector_field: String, -} - -#[derive(Default)] -struct SearchState { - indexes: HashMap, - hashes: BTreeMap, BTreeMap>>, -} - -/// An in-memory valkey-search speaking the `FT.*`, `HSET` and `EXPIRE` subset the semantic cache -/// sends, with exact cosine KNN over the hashes under an index prefix. -#[derive(Clone, Default)] -pub struct FakeSearch { - state: Arc>, -} - -impl FakeSearch { - fn run(&self, args: Vec>) -> redis::RedisResult { - let mut state = self.state.lock().unwrap(); - let text = |index: usize| String::from_utf8_lossy(&args[index]).into_owned(); - match text(0).to_uppercase().as_str() { - "FT.CREATE" => { - let name = text(1); - if state.indexes.contains_key(&name) { - return Err(server_error("Index already exists")); - } - let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); - let prefix = args[position("PREFIX").unwrap() + 2].clone(); - let dims = text(position("DIM").unwrap() + 1).parse().unwrap(); - let vector_field = text(position("VECTOR").unwrap() - 1); - state.indexes.insert( - name, - FakeIndex { - prefix, - dims, - vector_field, - }, - ); - Ok(redis::Value::Okay) - } - "FT.INFO" => { - let index = state - .indexes - .get(&text(1)) - .ok_or_else(|| server_error("Unknown index name"))?; - Ok(index_info(index)) - } - "FT.DROPINDEX" => { - state.indexes.remove(&text(1)); - Ok(redis::Value::Okay) - } - "HSET" => { - let hash = state.hashes.entry(args[1].clone()).or_default(); - for pair in args[2..].chunks(2) { - hash.insert( - String::from_utf8_lossy(&pair[0]).into_owned(), - pair[1].clone(), - ); - } - Ok(redis::Value::Int(((args.len() - 2) / 2) as i64)) - } - "EXPIRE" => Ok(redis::Value::Int(i64::from( - state.hashes.contains_key(&args[1]), - ))), - "FT.SEARCH" => { - let index = state - .indexes - .get(&text(1)) - .ok_or_else(|| server_error("no such index"))?; - let query = text(2); - let tag = query_tag(&query); - let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); - let vector = floats(&args[params + 3]); - let best = state - .hashes - .iter() - .filter(|(key, _)| key.starts_with(&index.prefix)) - .filter(|(_, fields)| { - fields.get("litellm_cache_key").map(Vec::as_slice) == Some(tag.as_bytes()) - }) - .filter_map(|(key, fields)| { - let stored = floats(fields.get(&index.vector_field)?); - (stored.len() == index.dims) - .then(|| (key, fields, 1.0 - cosine(&vector, &stored))) - }) - .min_by(|left, right| left.2.total_cmp(&right.2)); - let Some((key, fields, distance)) = best else { - return Ok(redis::Value::Array(vec![redis::Value::Int(0)])); - }; - let mut reply = fields - .iter() - .filter(|(name, _)| **name != index.vector_field) - .flat_map(|(name, value)| [bulk(name.as_bytes()), bulk(value)]) - .collect::>(); - reply.extend([ - bulk(b"vector_distance"), - bulk(distance.to_string().as_bytes()), - ]); - Ok(redis::Value::Array(vec![ - redis::Value::Int(1), - bulk(key), - redis::Value::Array(reply), - ])) - } - "PING" => Ok(redis::Value::SimpleString("PONG".into())), - _ => Err(server_error("unsupported command")), - } - } -} - -impl redis::ConnectionLike for FakeSearch { - fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { - let mut commands = parse_commands(command); - self.run(commands.remove(0)) - } - - fn req_packed_commands( - &mut self, - commands: &[u8], - offset: usize, - count: usize, - ) -> redis::RedisResult> { - let replies = parse_commands(commands) - .into_iter() - .map(|args| self.run(args)) - .collect::>>()?; - Ok(replies.into_iter().skip(offset).take(count).collect()) - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } -} - -pub fn server_error(reply: &str) -> redis::RedisError { - redis::parse_redis_value(format!("-{reply}\r\n").as_bytes()) - .expect("error reply") - .extract_error() - .expect_err("server error") -} - -fn bulk(bytes: &[u8]) -> redis::Value { - redis::Value::BulkString(bytes.to_vec()) -} - -fn index_info(index: &FakeIndex) -> redis::Value { - redis::Value::Array(vec![ - bulk(b"index_name"), - bulk(b"fake"), - bulk(b"attributes"), - redis::Value::Array(vec![ - redis::Value::Array(vec![ - bulk(b"identifier"), - bulk(b"litellm_cache_key"), - bulk(b"type"), - bulk(b"TAG"), - ]), - redis::Value::Array(vec![ - bulk(b"identifier"), - bulk(index.vector_field.as_bytes()), - bulk(b"type"), - bulk(b"VECTOR"), - bulk(b"index"), - redis::Value::Array(vec![ - bulk(b"dimensions"), - redis::Value::Int(index.dims as i64), - ]), - ]), - ]), - ]) -} - -/// The tag inside `@litellm_cache_key:{...}`, with query escapes removed. -fn query_tag(query: &str) -> String { - let start = query.find("@litellm_cache_key:{").unwrap() + "@litellm_cache_key:{".len(); - let mut tag = String::new(); - let mut characters = query[start..].chars(); - while let Some(character) = characters.next() { - match character { - '\\' => tag.extend(characters.next()), - '}' => break, - character => tag.push(character), - } - } - tag -} - -fn floats(bytes: &[u8]) -> Vec { - bytes - .as_chunks::<4>() - .0 - .iter() - .map(|chunk| f32::from_le_bytes(*chunk)) - .collect() -} - -fn cosine(left: &[f32], right: &[f32]) -> f64 { - let dot = left - .iter() - .zip(right) - .map(|(left, right)| f64::from(*left) * f64::from(*right)) - .sum::(); - let norm = |vector: &[f32]| { - vector - .iter() - .map(|value| f64::from(*value).powi(2)) - .sum::() - .sqrt() - }; - dot / (norm(left) * norm(right)) -} - -/// Splits a packed RESP request into each command's arguments. -fn parse_commands(mut bytes: &[u8]) -> Vec>> { - let line = |bytes: &mut &[u8]| { - let end = bytes - .windows(2) - .position(|window| window == b"\r\n") - .unwrap(); - let text = String::from_utf8(bytes[1..end].to_vec()).unwrap(); - *bytes = &bytes[end + 2..]; - text.parse::().unwrap() - }; - let mut commands = Vec::new(); - while !bytes.is_empty() { - let count = line(&mut bytes); - let mut args = Vec::with_capacity(count); - for _ in 0..count { - let length = line(&mut bytes); - args.push(bytes[..length].to_vec()); - bytes = &bytes[length + 2..]; - } - commands.push(args); - } - commands -} - -/// Records every packed request and answers from a script, `OK` once the script runs out. -pub struct RecordingConnection { - requests: Arc>>>, - replies: Mutex>>, -} - -impl RecordingConnection { - pub fn new(replies: impl IntoIterator>) -> Self { - Self { - requests: Arc::default(), - replies: Mutex::new(replies.into_iter().collect()), - } - } - - pub fn requests(&self) -> Arc>>> { - Arc::clone(&self.requests) - } - - fn reply(&self) -> redis::RedisResult { - self.replies - .lock() - .unwrap() - .pop_front() - .unwrap_or_else(|| Ok(redis::Value::SimpleString("OK".into()))) - } -} - -impl redis::ConnectionLike for RecordingConnection { - fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { - self.requests.lock().unwrap().push(command.to_vec()); - self.reply() - } - - fn req_packed_commands( - &mut self, - command: &[u8], - _offset: usize, - count: usize, - ) -> redis::RedisResult> { - self.requests.lock().unwrap().push(command.to_vec()); - (0..count).map(|_| self.reply()).collect() - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } -} diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index f65516265c5..ab887bbc014 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -33,6 +33,17 @@ "api_key", "additional_args" ], + "record_response": [ + "logger", + "status", + "headers", + "body" + ], + "record_api_window": [ + "logger", + "start", + "end" + ], "defers_async_logging": [ "logger" ], @@ -91,6 +102,13 @@ "logger", "head" ], + "stamp_stream_head": [ + "head", + "logger", + "kwargs", + "start_time", + "end_time" + ], "stream_success": [ "logger", "url_route", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 9431b9d1326..1bb84184e4c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -57,6 +57,8 @@ pub struct LegacyLogging { asynchronous: bool, internal: bool, cache_key: Option, + /// When the provider request went out, until its window is recorded (Messages only). + api_started: Option, } fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { @@ -87,6 +89,60 @@ impl LegacyLogging { asynchronous, internal: false, cache_key: None, + api_started: None, + } + } + + /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as + /// its own argument, so the logger never sees the caller's raw metadata, only the validated + /// request metadata merged in through `litellm_params`. Python updates the logger once, before + /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the + /// logged metadata is not rebuilt from what the call wrote into it since. + fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult> { + let kwargs = self.call.kwargs().bind(py).copy()?; + if self.operation != LoggingOperation::Messages { + return Ok(kwargs); + } + let consumed: &[&str] = if refresh { + &["metadata", "litellm_metadata"] + } else { + &["metadata"] + }; + for name in consumed { + if kwargs.contains(name)? { + kwargs.del_item(name)?; + } + } + Ok(kwargs) + } + + /// Python's Messages handler hands the logger its request params, validated `metadata` + /// included, as `litellm_params`, beside the `aanthropic_messages` call flag. + fn logged_request_params( + &self, + context: &RequestContext, + ) -> Option> { + if self.operation != LoggingOperation::Messages { + return None; + } + let mut params = context + .optional_params + .as_object() + .cloned() + .unwrap_or_default(); + params.insert( + "aanthropic_messages".to_owned(), + Value::Bool(self.asynchronous), + ); + Some(params) + } + + /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync + /// client is untimed. + fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> { + match self.api_started.take() { + Some(start) => self.logger()?.record_api_window(py, start, end), + None => Ok(()), } } @@ -111,6 +167,16 @@ impl LegacyLogging { } } + /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key. + fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> { + match self.operation { + LoggingOperation::Messages => Some(""), + LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => { + context.api_key.as_ref().map(|api_key| api_key.expose()) + } + } + } + fn stream_billing(&self) -> Option { match self.operation { LoggingOperation::Messages => Some(PassThroughStream { @@ -377,7 +443,13 @@ impl LegacyLogging { context: &RequestContext, ) -> PyResult>> { let logger = self.logger()?; - logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?; + logger.update_from_kwargs( + py, + &self.logging_kwargs(py, false)?, + &wire, + context, + self.logged_request_params(context).as_ref(), + )?; let body = to_py(py, &wire.body)? .into_bound(py) .cast_into::()?; @@ -400,11 +472,14 @@ impl LegacyLogging { self.logger()?.pre_call( py, self.input_description(), - context.api_key.as_ref().map(|api_key| api_key.expose()), + self.logged_api_key(context), &body, &headers, &wire.url, )?; + if self.operation == LoggingOperation::Messages && self.asynchronous { + self.api_started = Some(epoch_seconds()); + } let headers = headers .iter() .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) @@ -455,7 +530,7 @@ impl LegacyLogging { py, ( &logger, - self.call.kwargs(), + self.logging_kwargs(py, true)?, &facts.provider.model, logger.getattr("optional_params")?, params, @@ -483,6 +558,16 @@ impl LegacyLogging { py: Python<'_>, raw: &RawResponse, ) -> PyResult> { + if self.operation == LoggingOperation::Messages { + // Python's Messages handler never calls `post_call` at receipt: it stores the raw + // response, and logging replays it through `AnthropicConfig.transform_response`. + self.close_api_window(py, epoch_seconds())?; + if let Some(head) = &raw.head { + self.logger()? + .record_response(py, head.status, &head.headers, &raw.body)?; + } + return Ok(HookStep::Ready(())); + } let api_key = self .request .as_ref() @@ -522,6 +607,9 @@ impl LegacyLogging { ) -> PyResult> { self.end = Some(datetime(py, timing.end_time)?); self.error = Some(error.clone_ref(py).into_value(py)); + if self.logger.is_some() { + self.close_api_window(py, timing.end_time)?; + } if self.stream.is_some() { return self.stream_failure(py); } @@ -543,6 +631,20 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } + self.close_api_window(py, epoch_seconds())?; + if self.operation == LoggingOperation::Messages { + // Python's `@client` stamps the call metadata onto the stream it hands back. + Streaming::StampHead.call( + py, + ( + head.bind(py), + self.logger()?.object(py), + self.call.kwargs(), + &self.start, + datetime(py, epoch_seconds())?, + ), + )?; + } Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), @@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update let step = logging .before_provider_request(py, Box::new(wire), context) .unwrap(); - let raw = RawResponse { - body: "raw response".into(), - }; + let raw = RawResponse::new("raw response".into()); assert!(matches!( logging .on_event( @@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response py, &locals, c" -assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls assert logger.calls[0][1] is head -chunks = logger.calls[1][1] +assert logger.calls[1][1] is head +chunks = logger.calls[2][1] assert len(chunks) == 2 assert chunks[0] is first assert chunks[1] is last diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index f185f21a9c3..1951c9ebfb4 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -5,6 +5,7 @@ use litellm_host::interceptors::{RequestContext, WireRequest}; use litellm_host_python::to_py; use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; +use serde_json::Value; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; @@ -15,9 +16,10 @@ pub trait LegacyCallbacks { fn update_from_kwargs( &self, py: Python<'_>, - kwargs: &Py, + kwargs: &Bound<'_, PyDict>, wire: &WireRequest, context: &RequestContext, + request_params: Option<&serde_json::Map>, ) -> PyResult<()>; /// `Logging.pre_call`. @@ -31,6 +33,20 @@ pub trait LegacyCallbacks { url: &str, ) -> PyResult<()>; + /// Records the provider's raw HTTP response on the logger, as Python's Messages handler + /// does with `httpx_response` and `response_headers`. + fn record_response( + &self, + py: Python<'_>, + status: u16, + headers: &[(String, String)], + body: &str, + ) -> PyResult<()>; + + /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post` + /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`. + fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>; + /// `Logging.post_call`. fn post_call( &self, @@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger { fn update_from_kwargs( &self, py: Python<'_>, - kwargs: &Py, + kwargs: &Bound<'_, PyDict>, wire: &WireRequest, context: &RequestContext, + request_params: Option<&serde_json::Map>, ) -> PyResult<()> { let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect(); - let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?; + let redacted_kwargs = redact(py, kwargs, &secret_fields)?; let optional_params = redact( py, &to_py(py, &context.optional_params)? @@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger { &secret_fields, )?; let params = PyDict::new(py); - params.set_item( - "litellm_call_id", - kwargs.bind(py).get_item("litellm_call_id")?, - )?; + params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?; params.set_item("api_base", &wire.url)?; for name in ["logger_fn", "litellm_request_debug"] { - if let Some(value) = kwargs.bind(py).get_item(name)? { + if let Some(value) = kwargs.get_item(name)? { params.set_item(name, value)?; } } for name in custom_pricing_fields(py)? { - if let Some(value) = kwargs.bind(py).get_item(&name)? + if let Some(value) = kwargs.get_item(&name)? && !value.is_none() { params.set_item(name, value)?; } } + for (name, value) in request_params.into_iter().flatten() { + params.set_item(name, to_py(py, value)?)?; + } Logging::Update.call( py, ( @@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } + fn record_response( + &self, + py: Python<'_>, + status: u16, + headers: &[(String, String)], + body: &str, + ) -> PyResult<()> { + Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?; + Ok(()) + } + + fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> { + Logging::RecordApiWindow.call(py, (self.object(py), start, end))?; + Ok(()) + } + fn post_call( &self, py: Python<'_>, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 6f1ee9dfd78..6d910d35f53 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -44,6 +44,10 @@ pub(crate) enum Logging { PreCall, #[strum(serialize = "post_call")] PostCall, + #[strum(serialize = "record_response")] + RecordResponse, + #[strum(serialize = "record_api_window")] + RecordApiWindow, #[strum(serialize = "defers_async_logging")] DefersAsync, #[strum(serialize = "defer_success")] @@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks { pub(crate) enum Streaming { #[strum(serialize = "stream_opened")] Opened, + #[strum(serialize = "stamp_stream_head")] + StampHead, #[strum(serialize = "stream_success")] Success, #[strum(serialize = "stream_failure")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 093ce60e510..8acfa7a898b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -65,6 +65,12 @@ FAKES = { 'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call( original_response, api_key, additional_args ), + 'record_response': lambda logger, status, headers, body: logger.model_call_details.update( + response_headers=dict(headers) + ), + 'record_api_window': lambda logger, start, end: logger.model_call_details.update( + llm_api_duration_ms=(end - start) * 1000 + ), 'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)), 'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending), 'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls( @@ -85,6 +91,7 @@ FAKES = { ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), 'stream_opened': lambda logger, head: logger.record('stream_opened', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index 12cec683748..70f6016780c 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -151,6 +151,8 @@ pub enum Uplift { Residency(Residency), /// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier` NonGlobalEndpoint, + /// OpenAI FedRAMP endpoint (gov.api.openai.com), from `fedramp_uplift_multiplier` + FedrampEndpoint, /// Anthropic `speed: fast`, from `provider_specific_entry.fast` Speed(Speed), /// Anthropic and Bedrock geo, from `provider_specific_entry.`, lowercased as on main diff --git a/litellm-rust/crates/host-http/tests/serve.rs b/litellm-rust/crates/host-http/tests/serve.rs index 2ba39ad4cd2..fd1a8940b33 100644 --- a/litellm-rust/crates/host-http/tests/serve.rs +++ b/litellm-rust/crates/host-http/tests/serve.rs @@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(Bytes::from(wire.url))) }, @@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once( move |_, _, route_hooks, _observations| async move { if event { route_hooks - .after_provider_response(RawResponse { - body: "response".into(), - }) + .after_provider_response(RawResponse::new("response".into())) .await?; } else { route_hooks @@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(json!({"url": wire.url}))) }, @@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting( None, |_, _, route_hooks, _observations| async move { route_hooks - .after_provider_response(RawResponse { body: "raw".into() }) + .after_provider_response(RawResponse::new("raw".into())) .await?; Err(TestError::Provider) }, diff --git a/litellm-rust/crates/host-native/tests/driver.rs b/litellm-rust/crates/host-native/tests/driver.rs index c41c1e5a43b..44c30234328 100644 --- a/litellm-rust/crates/host-native/tests/driver.rs +++ b/litellm-rust/crates/host-native/tests/driver.rs @@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine { ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(wire.url)) }, @@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re "input", Some(sender), |_, _, interceptors, observers| async move { - let raw = RawResponse { - body: "provider response".into(), - }; + let raw = RawResponse::new("provider response".into()); observers.unwrap().emit(CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, )); diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs index 003fce74309..15e864c0718 100644 --- a/litellm-rust/crates/host-python/tests/support/mod.rs +++ b/litellm-rust/crates/host-python/tests/support/mod.rs @@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory { .before_provider_request(wire(), context()) .await?; host.interceptors - .after_provider_response(RawResponse { body: "raw".into() }) + .after_provider_response(RawResponse::new("raw".into())) .await?; Ok(format!("{projected}|{signed}|{}", wire.url)) }) diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs index 39af7c410e0..9398ea4bad6 100644 --- a/litellm-rust/crates/host/src/interceptors.rs +++ b/litellm-rust/crates/host/src/interceptors.rs @@ -28,6 +28,28 @@ pub struct RequestContext { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RawResponse { pub body: String, + /// The provider's HTTP status and headers, when the route forwards them to logging. + pub head: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResponseHead { + pub status: u16, + pub headers: Vec<(String, String)>, +} + +impl RawResponse { + pub fn new(body: String) -> Self { + Self { body, head: None } + } + + #[must_use] + pub fn with_head(self, head: ResponseHead) -> Self { + Self { + head: Some(head), + ..self + } + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -186,7 +208,7 @@ mod tests { .await?; Interceptors::after_provider_response( &channel.interceptors, - RawResponse { body: "raw".into() }, + RawResponse::new("raw".into()), ) .await?; Ok((sent, ())) diff --git a/litellm-rust/crates/host/tests/observation.rs b/litellm-rust/crates/host/tests/observation.rs index 22872047d81..fba85b63bea 100644 --- a/litellm-rust/crates/host/tests/observation.rs +++ b/litellm-rust/crates/host/tests/observation.rs @@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile #[rstest] fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) { - let mut raw = RawResponse { - body: "provider response".into(), - }; + let mut raw = RawResponse::new("provider response".into()); let event: CallEvent<(), (), &RawResponse> = CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw }); let expected = raw.clone(); diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 29c8b988c0b..7078d4aba2a 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -58,8 +58,20 @@ pub(crate) enum Repr { Network(#[source] reqwest::Error), #[error("upstream request timed out")] Timeout(#[source] reqwest::Error), + /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says + /// the response head had already arrived. + #[error("upstream request timed out")] + ReadTimeout { + #[source] + elapsed: Elapsed, + body: bool, + }, #[error("upstream request failed with status {status}: {body}")] - Status { status: u16, body: UpstreamBody }, + Status { + status: u16, + body: UpstreamBody, + headers: Vec<(String, String)>, + }, #[error("invalid WebSocket URL")] WebSocketUrl(#[source] url::ParseError), @@ -136,13 +148,59 @@ impl Repr { impl Error { pub fn upstream(status: u16, body: String) -> Self { + Self::upstream_with_headers(status, body, Vec::new()) + } + + /// An upstream failure that keeps the provider's response headers (`retry-after`, + /// `request-id`) for the caller. + pub fn upstream_with_headers( + status: u16, + body: String, + headers: Vec<(String, String)>, + ) -> Self { Repr::Status { status, body: UpstreamBody(body), + headers, } .into() } + /// The provider's response headers on an upstream failure; empty for every other error. + pub fn upstream_headers(&self) -> &[(String, String)] { + match &*self.0 { + Repr::Status { headers, .. } => headers, + _ => &[], + } + } + + /// A timeout reached before the connection was established, as opposed to while reading. + pub fn is_connect_timeout(&self) -> bool { + matches!(&*self.0, Repr::Timeout(error) if error.is_connect()) + } + + /// A read the caller timed itself (httpx's per-read timeout) ran out. + pub fn read_timeout(elapsed: Elapsed) -> Self { + Repr::ReadTimeout { + elapsed, + body: false, + } + .into() + } + + /// A read of the response body the caller timed itself ran out, after the head arrived. + pub fn body_read_timeout(elapsed: Elapsed) -> Self { + Repr::ReadTimeout { + elapsed, + body: true, + } + .into() + } + + pub fn is_body_read_timeout(&self) -> bool { + matches!(&*self.0, Repr::ReadTimeout { body: true, .. }) + } + pub fn computed_header(name: impl Into) -> Self { Repr::ComputedHeader(name.into()).into() } @@ -169,6 +227,7 @@ impl Error { match &*self.0 { Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::Handshake(_) | Repr::HandshakeTimeout(_) @@ -227,6 +286,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::Handshake(_) | Repr::HandshakeTimeout(_) @@ -259,6 +319,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -295,6 +356,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -333,6 +395,7 @@ impl Error { | Repr::Signature(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -356,7 +419,10 @@ impl Error { pub fn is_timeout(&self) -> bool { match &*self.0 { - Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true, + Repr::Timeout(_) + | Repr::ReadTimeout { .. } + | Repr::HandshakeTimeout(_) + | Repr::DownloadTimeout(_) => true, Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -392,7 +458,7 @@ impl Error { pub fn upstream_response(&self) -> Option<(u16, &str)> { match &*self.0 { - Repr::Status { status, body } => Some((*status, &body.0)), + Repr::Status { status, body, .. } => Some((*status, &body.0)), Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -408,6 +474,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) | Repr::WebSocketSchemeRewrite(_) @@ -431,7 +498,7 @@ impl Error { pub fn status(&self) -> Option { match &*self.0 { Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status), - Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408), + Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408), Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -482,6 +549,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index f948e8fffaf..cdcc3e911b9 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -108,7 +108,7 @@ pub(super) async fn execute( if !status.is_success() { return Err(litellm_http::Error::upstream(status.as_u16(), text).into()); } - let raw = RawResponse { body: text.clone() }; + let raw = RawResponse::new(text.clone()); if let Some(observers) = observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index ac52cb65a0d..0c5be52550d 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -24,6 +24,7 @@ litellm-tracing.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } +tokio.workspace = true tracing.workspace = true [dev-dependencies] @@ -34,5 +35,4 @@ litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-inference-testing.workspace = true rstest.workspace = true -tokio.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index a57eca1c6a9..e5ece2837d8 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -35,6 +35,14 @@ impl MessagesProvider { pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig { self.config } + + /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped. + pub(crate) fn accepts_service_tier(self) -> bool { + matches!( + self.provider, + LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle + ) + } } /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude @@ -119,3 +127,69 @@ mod tests { ); } } + +/// Python's `_merge_messages_header_sources`: later sources win per header name, except +/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and +/// sorted into one lowercase `anthropic-beta` entry. +pub fn merge_messages_header_sources( + sources: impl IntoIterator>, +) -> Map { + let mut merged = Map::new(); + let mut betas = std::collections::BTreeSet::new(); + for (name, value) in sources.into_iter().flatten() { + if !name.eq_ignore_ascii_case("anthropic-beta") { + merged.insert(name, value); + continue; + } + let values: Vec<&str> = match &value { + Value::String(text) => vec![text.as_str()], + Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(), + _ => Vec::new(), + }; + betas.extend( + values + .iter() + .flat_map(|value| value.split(',')) + .map(str::trim) + .filter(|beta| !beta.is_empty()) + .map(str::to_owned), + ); + } + if !betas.is_empty() { + merged.insert( + "anthropic-beta".to_owned(), + Value::String(betas.into_iter().collect::>().join(",")), + ); + } + merged +} + +#[cfg(test)] +mod merge_tests { + use serde_json::{Map, Value, json}; + + use super::merge_messages_header_sources; + + fn map(value: Value) -> Map { + value.as_object().cloned().unwrap() + } + + #[test] + fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() { + let merged = merge_messages_header_sources([ + map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})), + map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})), + map(json!({"anthropic-beta": 5})), + ]); + assert_eq!( + Value::Object(merged), + json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"}) + ); + } + + #[test] + fn no_beta_values_leave_no_beta_header() { + let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]); + assert!(merged.is_empty()); + } +} diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 2e647688a1a..e0a7aa5adb8 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -10,9 +10,9 @@ use litellm_core_utils::{ get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, }; use litellm_host::interceptors::{ - Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest, + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, + ResponseHead, ResultSource, WireRequest, }; -use litellm_http::request::with_default_headers; use litellm_llms::base_llm::{ auth::{Authenticated, ValidatedEnvironment, resolve_auth}, messages::{ @@ -21,14 +21,16 @@ use litellm_llms::base_llm::{ transformation::BaseMessagesConfig, }, }; -use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; +use litellm_llms_types::formats::messages::{ + MessagesOptionalParams, MessagesRequest, MessagesResponse, +}; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; use serde_json::Value; use super::{ Error, MessagesCall, MessagesCallResponse, MessagesRoute, - common_utils::{MessagesProvider, string_headers}, + common_utils::{MessagesProvider, merge_messages_header_sources, string_headers}, constants::MESSAGES_TIMEOUT_SECS, route::Messages, types::invalid_request, @@ -126,14 +128,18 @@ async fn provider_call( MessagesRequest { model, ..body }, shaping.settings.reasoning_auto_summary, )?; + let transform_context = + MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); + let filtered = requested_optional_params(sanitized, provider, &transform_context)?; let trimmed = - without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?; let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); - let forwarded = string_headers(Some( - extra_headers.into_iter().flatten().chain(scoped).collect(), - ))?; + let forwarded = string_headers(Some(merge_messages_header_sources([ + extra_headers.unwrap_or_default(), + scoped, + ])))?; let validated = config.validate_environment( forwarded, api_key.as_deref(), @@ -144,16 +150,13 @@ async fn provider_call( )?; let body = config.request_body( - config.transform_anthropic_messages_request( - trimmed, - &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), - )?, + config.transform_anthropic_messages_request(trimmed, &transform_context)?, &validated.headers, shaping.settings.drop_params, )?; let environment = ValidatedEnvironment { headers: config.request_headers( - with_default_headers(validated.headers, config.default_headers()), + with_python_default_headers(validated.headers, config.default_headers()), &body, ), auth: validated.auth, @@ -184,7 +187,7 @@ async fn provider_call( .before_provider_request( WireRequest { url, - headers: authenticated.headers, + headers: joined_header_values(authenticated.headers), body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), }, request_context, @@ -213,6 +216,64 @@ async fn provider_call( }) } +/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in +/// case into one lowercase header whose values are joined with `", "`. +fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> { + let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len()); + let mut positions: std::collections::HashMap = + std::collections::HashMap::with_capacity(headers.len()); + for (name, value) in headers { + match positions.entry(name.to_ascii_lowercase()) { + std::collections::hash_map::Entry::Occupied(position) => { + let present = &mut joined[*position.get()].1; + present.push_str(", "); + present.push_str(&value); + } + std::collections::hash_map::Entry::Vacant(position) => { + position.insert(joined.len()); + joined.push((name, value)); + } + } + } + joined +} + +/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is +/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it. +fn with_python_default_headers( + headers: Vec<(String, String)>, + defaults: &[(&str, &str)], +) -> Vec<(String, String)> { + let missing: Vec<(String, String)> = defaults + .iter() + .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name)) + .map(|(name, value)| ((*name).to_string(), (*value).to_string())) + .collect(); + headers.into_iter().chain(missing).collect() +} + +/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier +/// and sampling params before `additional_drop_params` and the environment check run. +fn requested_optional_params( + request: MessagesRequest, + provider: MessagesProvider, + context: &MessagesTransformContext, +) -> Result { + let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params( + request, context, + )?; + if provider.accepts_service_tier() { + return Ok(request); + } + Ok(MessagesRequest { + params: MessagesOptionalParams { + service_tier: None, + ..request.params + }, + ..request + }) +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], @@ -245,6 +306,7 @@ async fn call_provider( } = request; let provider_name = provider.as_str(); log_request_body(provider_name, stream, &wire.body); + let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); let response = send( http, Authenticated { @@ -254,10 +316,11 @@ async fn call_provider( &wire.url, &wire.body, timeout, + stream, ) .await?; if !response.status().is_success() { - return Err(provider_error(response).await); + return Err(provider_error(response, timeout).await); } let config = provider.config(); if stream { @@ -265,11 +328,24 @@ async fn call_provider( response, config.stream_decoder(), provider_name, + timeout, )); } + let head = ResponseHead { + status: response.status().as_u16(), + headers: litellm_http::request::header_pairs(response.headers()), + }; + let status = head.status; let text = response.text().await.map_err(litellm_http::Error::from)?; log_response_body(&text); - context.response_received(&text).await?; + context + .raw_response_received(RawResponse::new(text.clone()).with_head(head)) + .await?; + if serde_json::from_str::(&text).is_err() { + // Python raises `AnthropicError(status_code=, message=)` for a + // body it cannot parse, and the public mapping classifies it by that status. + return Err(litellm_http::Error::upstream(status, text).into()); + } decode_response(config, &identity.model, &text) .map(|message| MessagesCallResponse::Complete(Box::new(message))) } @@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error { )) } +/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete +/// response keeps the client's overall deadline; a stream is bounded until its head arrives and +/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off. async fn send( http: &litellm_http::Client, authenticated: Authenticated, url: &str, body: &Value, - timeout: Option, + timeout: Duration, + stream: bool, ) -> Result { - let request = outbound_request( - authenticated, - url, - body, - Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))), - )?; - Ok(litellm_inference::outbound::send(request, http).await?) + let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?; + if !stream { + return Ok(litellm_inference::outbound::send(request, http).await?); + } + tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http)) + .await + .map_err(litellm_http::Error::read_timeout)? + .map_err(Error::from) } -async fn provider_error(response: reqwest::Response) -> Error { +/// The stream's body, each read bounded by `timeout` as httpx bounds each read. +fn timed_chunks( + response: reqwest::Response, + timeout: Duration, +) -> impl futures_util::Stream> + Send + 'static { + futures_util::stream::try_unfold(response, move |mut response| async move { + let chunk = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::body_read_timeout)? + .map_err(litellm_http::Error::from)?; + Ok(chunk.map(|chunk| (chunk, response))) + }) +} + +/// The provider's error, its body read under `timeout`: a stream has no overall deadline once +/// its head arrives, so an error body that stalls would otherwise hold the call forever. +async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error { let status = response.status().as_u16(); - match response.text().await { - Ok(text) => { + let headers = litellm_http::request::header_pairs(response.headers()); + match tokio::time::timeout(timeout, response.text()).await { + Ok(Ok(text)) => { log_error_body(status, &text); - litellm_http::Error::upstream(status, text).into() + litellm_http::Error::upstream_with_headers(status, text, headers).into() } - Err(error) => litellm_http::Error::from(error).into(), + Ok(Err(error)) => litellm_http::Error::from(error).into(), + Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(), } } @@ -329,22 +428,22 @@ fn streaming_response( response: reqwest::Response, decoder: Option, provider: &'static str, + timeout: Duration, ) -> MessagesCallResponse { let headers = response .headers() .iter() .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) .collect(); + let chunks = timed_chunks(response, timeout) + .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); let chunks = match decoder { - None => futures_util::stream::try_unfold(response, move |mut response| async move { - let chunk = response.chunk().await.map_err(litellm_http::Error::from)?; - Ok(chunk.map(|chunk| { - log_chunk(provider, "provider_response", &chunk); - (chunk, response) - })) - }) - .boxed(), - Some(decode) => decoded_chunks(response, decode, provider), + None => chunks.map_err(Error::from).boxed(), + Some(decode) => decoded_chunks( + chunks.map_err(std::io::Error::other).boxed(), + decode, + provider, + ), }; MessagesCallResponse::Stream { head: super::route::MessagesStreamHead { headers }, @@ -353,15 +452,10 @@ fn streaming_response( } fn decoded_chunks( - response: reqwest::Response, + bytes: ByteStream, decode: StreamDecoder, provider: &'static str, ) -> BoxStream<'static, Result> { - let bytes: ByteStream = response - .bytes_stream() - .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)) - .map_err(|error| std::io::Error::other(litellm_http::Error::from(error))) - .boxed(); futures_util::stream::try_unfold(decode(bytes), move |mut events| async move { let Some(event) = events.try_next().await? else { return Ok(None); @@ -964,9 +1058,12 @@ mod tests { .send() .await .unwrap(); - let MessagesCallResponse::Stream { mut chunks, .. } = - streaming_response(response, Some(streaming::event_stream), "test") - else { + let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response( + response, + Some(streaming::event_stream), + "test", + Duration::from_secs(5), + ) else { panic!("a streaming response returns chunks"); }; diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index bfefb9c63b0..745442d0314 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use crate::common_utils::messages_provider; +pub use common_utils::merge_messages_header_sources; pub use litellm_inference::Error; pub use types::{ MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs index e53bdb8d252..216aa96fbe6 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs @@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M #[case::deepseek("deepseek")] #[case::minimax("minimax")] #[tokio::test] -async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) { +/// Python adds a default unless that exact lowercase name is present, then folds the casings +/// into one header, so a capitalised caller header is sent beside the default. +async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults( + call: MessagesCall, + #[case] provider: &str, +) { let upstream = upstream([message_response()]).await; run_message(MessagesCall { @@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas .await; let request = only_request(&upstream).await; - assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]); + assert_eq!( + request.header_values("anthropic-version"), + ["2024-01-01, 2023-06-01"] + ); assert_eq!( request.header_values("content-type"), - ["application/json; charset=utf-8"] + ["application/json; charset=utf-8, application/json"] ); } diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index 7aff1286858..76813fe0a34 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case] } #[rstest] -#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))] -#[case::not_a_message(json_response(json!({"unexpected": true})))] #[tokio::test] -async fn an_unreadable_success_body_is_an_invalid_response( - call: MessagesCall, - #[case] response: ResponseTemplate, -) { - let upstream = upstream([response]).await; +async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await; + + let error = run(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect_err("an unparseable body fails"); + + assert_eq!(error.upstream_response(), Some((200, "not json"))); +} + +#[rstest] +#[tokio::test] +async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) { + let upstream = upstream([json_response(json!({"unexpected": true}))]).await; let error = run(MessagesCall { api_key: Some("sk".into()), diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs index 4a8a061f764..99ba08495aa 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs @@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream( .unwrap(); } +/// Answers with an error head whose promised body never arrives. +async fn stalling_error_upstream() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let connection = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let _ = socket.read(&mut request).await; + socket + .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n") + .await + .unwrap(); + let _ = socket.read_to_end(&mut Vec::new()).await; + }); + (base, connection) +} + +#[rstest] +#[tokio::test] +async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { + let (base, connection) = stalling_error_upstream().await; + let host = RecordingStreamHost::new( + MessagesCall { + timeout: Some(Duration::from_millis(300)), + ..streaming(call, base) + }, + usize::MAX, + ); + + let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host)) + .await + .expect("the stalled error body gives up within the timeout") + .expect_err("a stalled error body fails the call"); + + assert!(error.dispatched(), "{error:?}"); + assert_eq!(error.upstream_response(), None); + assert!(host.seen.into_inner().unwrap().is_empty()); + tokio::time::timeout(Duration::from_secs(5), connection) + .await + .expect("timing out closes the upstream connection") + .unwrap(); +} + #[rstest] #[tokio::test] async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) { diff --git a/litellm-rust/crates/inference-ocr/src/handler.rs b/litellm-rust/crates/inference-ocr/src/handler.rs index c126a192153..be23af60c69 100644 --- a/litellm-rust/crates/inference-ocr/src/handler.rs +++ b/litellm-rust/crates/inference-ocr/src/handler.rs @@ -91,9 +91,7 @@ impl> CallHooks for OcrCallHooks<'_, H> { } fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> { - let raw = RawResponse { - body: String::from_utf8_lossy(body).into_owned(), - }; + let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned()); if let Some(observers) = self.observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference-ocr/tests/resources.rs b/litellm-rust/crates/inference-ocr/tests/resources.rs index d618647e89b..3132a837eaf 100644 --- a/litellm-rust/crates/inference-ocr/tests/resources.rs +++ b/litellm-rust/crates/inference-ocr/tests/resources.rs @@ -7,7 +7,7 @@ use std::sync::{ use litellm_auth::AuthServices; use litellm_auth_gcp::{ - CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource, + CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth, }; use litellm_http::{HttpSettings, Resolution}; use litellm_inference::resources::CoreResources; @@ -20,28 +20,28 @@ use support::{ReceivedRequest, json_response, upstream}; struct TokenSource(String); -impl VertexTokenSource for TokenSource { - fn project_id(&self) -> VertexAuthFuture<'_, String> { +impl GcpTokenSource for TokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { Box::pin(async { Ok(self.0.clone()) }) } - fn token(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async { Ok(self.0.clone()) }) + fn token(&self) -> GcpAuthFuture<'_, litellm_secret_value::SecretValue> { + Box::pin(async { Ok(litellm_secret_value::SecretValue::new(self.0.clone())) }) } } #[derive(Default)] struct Loader(AtomicUsize); -impl VertexProviderLoader for Loader { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc> { +impl GcpProviderLoader for Loader { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc> { Box::pin(async move { self.0.fetch_add(1, Ordering::SeqCst); let identity = match source { CredentialSource::Trusted(secret) => secret.expose().to_string(), other => panic!("unexpected credential source: {other:?}"), }; - Ok(Arc::new(TokenSource(identity)) as Arc) + Ok(Arc::new(TokenSource(identity)) as Arc) }) } } diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index 19092a63a14..f81a51ad192 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -114,7 +114,7 @@ pub(super) async fn execute( }); } let body = response.text().await.map_err(litellm_http::Error::from)?; - let raw = RawResponse { body: body.clone() }; + let raw = RawResponse::new(body.clone()); if let Some(observers) = observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index 963aa7b90c5..ef2635d24a1 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -42,9 +42,11 @@ impl<'a, I: Interceptors> CallContext<'a, I> { } pub async fn response_received(&self, body: &str) -> Result<(), Error> { - let raw = RawResponse { - body: body.to_owned(), - }; + self.raw_response_received(RawResponse::new(body.to_owned())) + .await + } + + pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> { if let Some(observers) = &self.observers { observers.emit(CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index a8e6fb4e1b3..841e0c53799 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -146,6 +146,34 @@ impl Error { } } + /// The provider's response headers on an upstream failure; empty for every other error. + pub fn upstream_headers(&self) -> &[(String, String)] { + match self { + Self::Llm(error) => error.upstream_headers(), + Self::Http(error) => error.upstream_headers(), + _ => &[], + } + } + + /// A read of a streamed body timed out after the response head arrived. + pub fn body_read_timeout(&self) -> bool { + match self { + Self::Llm(error) => error.is_body_read_timeout(), + Self::Http(error) => error.is_body_read_timeout(), + _ => false, + } + } + + /// `Some(connect)` when the call timed out, where `connect` says the connection was never + /// established; `None` for every other error. + pub fn timeout(&self) -> Option { + match self { + Self::Llm(error) => error.is_timeout(), + Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()), + _ => None, + } + } + pub fn upstream_response(&self) -> Option<(u16, &str)> { match self { Self::Llm(error) => error.upstream_response(), diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index 9dbfa068025..d30d21e74e7 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -264,7 +264,12 @@ pub enum ThinkingDisplay { #[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct EnabledThinking { - #[serde(default, skip_serializing_if = "Option::is_none")] + /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::present" + )] pub budget_tokens: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub display: Option>, diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs index 9b0c5d31b3a..276a30a6cb4 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs @@ -10,14 +10,29 @@ pub struct MessagesResponse { pub content: Vec, pub stop_reason: Option, pub stop_sequence: Option, - #[serde(skip_serializing_if = "Option::is_none")] + /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps. + #[serde( + default, + deserialize_with = "present_value", + skip_serializing_if = "Option::is_none" + )] pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + deserialize_with = "present_value", + skip_serializing_if = "Option::is_none" + )] pub container: Option, #[serde(flatten)] pub extra: Map, } +fn present_value<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + >::deserialize(deserializer).map(Some) +} + #[cfg(test)] mod tests { use super::*; @@ -73,4 +88,21 @@ mod tests { assert_eq!(body.get("usage").cloned(), usage); assert_eq!(body.get("container").cloned(), container); } + + #[test] + fn explicit_null_usage_and_container_survive_a_round_trip() { + let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": null, "stop_sequence": null, + "usage": null, "container": null}); + let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses"); + assert_eq!(serde_json::to_value(parsed).expect("serializable"), body); + } + + #[test] + fn absent_usage_and_container_stay_absent() { + let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": null, "stop_sequence": null}); + let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses"); + assert_eq!(serde_json::to_value(parsed).expect("serializable"), body); + } } diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 58e8dd7a2e7..0545389afbc 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -7,6 +7,16 @@ pub enum Recognized { Unrecognized(Value), } +/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay +/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`. +pub fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + Recognized: serde::Deserialize<'de>, +{ + as serde::Deserialize<'de>>::deserialize(deserializer).map(Some) +} + impl Recognized { pub fn known(&self) -> Option<&T> { match self { diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index aec5d9ebc27..13653f438f7 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -154,11 +154,42 @@ pub fn complete_anthropic_url( ) -> String { let api_base = resolve_anthropic_api_base(api_base, env_lookup); - let api_base = api_base.trim_end_matches('/'); - if api_base.ends_with(MESSAGES_PATH) { + ensure_api_route(&api_base, MESSAGES_PATH) +} + +/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds. +fn url_path(url: &str) -> &str { + let before_fragment = url.split('#').next().unwrap_or_default(); + let before_query = before_fragment.split('?').next().unwrap_or_default(); + match before_query.split_once("://") { + Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]), + None => before_query, + } +} + +/// Python's `_strip_trailing_slashes_keeping_the_scheme`. +fn strip_trailing_slashes(api_base: &str) -> String { + let stripped = api_base.trim_end_matches('/'); + if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') { return api_base.to_string(); } - format!("{api_base}{MESSAGES_PATH}") + match stripped.split_once('?') { + Some((before, query)) if url_path(stripped).ends_with('/') => { + format!("{}?{query}", before.trim_end_matches('/')) + } + _ => stripped.to_string(), + } +} + +/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as +/// it is, query string included; anything else gets the route appended to the whole string. +fn ensure_api_route(api_base: &str, route: &str) -> String { + let base = strip_trailing_slashes(api_base); + let route = route.trim_start_matches('/'); + if url_path(&base).ends_with(&format!("/{route}")) { + return base; + } + format!("{}/{route}", strip_trailing_slashes(&base)) } pub fn existing_betas(headers: &[(String, String)]) -> BetaSet { @@ -1922,4 +1953,24 @@ mod tests { expected ); } + + #[rstest] + #[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")] + #[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")] + #[case::route_present( + "https://api.example.com/v1/messages/", + "https://api.example.com/v1/messages" + )] + #[case::route_and_query( + "https://api.example.com/v1/messages?beta=true", + "https://api.example.com/v1/messages?beta=true" + )] + #[case::query_without_route( + "https://api.example.com/?x=1", + "https://api.example.com?x=1/v1/messages" + )] + #[case::scheme_only("https://", "https:///v1/messages")] + fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) { + assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index df1ae13fd28..ae3488a21f9 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -2,8 +2,8 @@ use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig, - ThinkingDisplay, + AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, + ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, @@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti fn effort_for_budget( budgets: &ThinkingBudgets, - budget_tokens: u64, + budget_tokens: i64, capabilities: &MessagesModelCapabilities, ) -> EffortLevel { - if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh { + let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier); + if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh { return EffortLevel::Xhigh; } - if budget_tokens >= budgets.high { + if reaches(budgets.high) { return EffortLevel::High; } - if budget_tokens >= budgets.medium { + if reaches(budgets.medium) { return EffortLevel::Medium; } EffortLevel::Low @@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) - } } +/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string +/// as a budget, and nothing else. +fn numeric_budget_tokens(budget: &Recognized) -> Option { + let value = match budget { + Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)), + Recognized::Unrecognized(value) => value, + }; + match value { + Value::Number(number) => number + .as_i64() + .or_else(|| number.as_f64().map(|float| float.trunc() as i64)), + Value::String(text) => python_int(text), + Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None, + } +} + +/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits. +fn python_int(text: &str) -> Option { + let trimmed = text.trim(); + let digits = trimmed.trim_start_matches(['+', '-']); + let well_formed = trimmed.len() - digits.len() <= 1 + && !digits.is_empty() + && !digits.starts_with('_') + && !digits.ends_with('_') + && !digits.contains("__") + && digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_'); + if !well_formed { + return None; + } + let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect(); + cleaned.parse::().ok().or_else(|| { + Some(if cleaned.starts_with('-') { + i64::MIN + } else { + i64::MAX + }) + }) +} + +fn python_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +fn kept_display( + display: Option<&Recognized>, + context: &ThinkingContext, +) -> Option> { + match display? { + Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => { + display.cloned() + } + Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => { + display.cloned() + } + Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None, + } +} + +/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive +/// keys always, and any other key unless `drop_params` is set. +fn carried_thinking_keys( + extra: &serde_json::Map, + context: &ThinkingContext, +) -> serde_json::Map { + extra + .iter() + .filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`. fn translate_legacy_thinking_for_adaptive_model( request: MessagesRequest, context: &ThinkingContext, -) -> MessagesRequest { +) -> Result { let capabilities = &context.capabilities; if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking { - return request; + return Ok(request); } let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else { - return request; + return Ok(request); }; - let budget = enabled - .budget_tokens - .as_ref() - .and_then(Recognized::known) - .copied() - .unwrap_or(0); - let level = effort_for_budget(&context.budgets, budget, capabilities); - MessagesRequest { + let budget = match &enabled.budget_tokens { + None => Some(0), + Some(budget) => numeric_budget_tokens(budget), + }; + if budget.is_none() && !context.drop_params { + let actual = match &enabled.budget_tokens { + Some(Recognized::Unrecognized(value)) => python_type_name(value), + Some(Recognized::Known(_)) | None => "int", + }; + return Err(Error::InvalidRequest( + crate::ErrorDetail::NonNumericThinkingBudget { actual }, + )); + } + let thinking = ThinkingConfig::Adaptive(AdaptiveThinking { + display: kept_display(enabled.display.as_ref(), context), + extra: carried_thinking_keys(&enabled.extra, context), + }); + let output_config = match budget { + Some(budget) => with_default_effort( + request.params.output_config, + effort_for_budget(&context.budgets, budget, capabilities), + ), + None => request.params.output_config, + }; + Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))), - output_config: with_default_effort(request.params.output_config, level), + thinking: Some(Recognized::Known(thinking)), + output_config, ..request.params }, ..request - } + }) } fn translate_adaptive_effort_for_non_adaptive_model( @@ -337,7 +435,7 @@ pub fn translate_thinking( ) -> Result { let request = translate_reasoning_effort(request, context)?; let request = drop_disabled_thinking(request, context); - let request = translate_legacy_thinking_for_adaptive_model(request, context); + let request = translate_legacy_thinking_for_adaptive_model(request, context)?; let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?; Ok(drop_incompatible_temperature_for_thinking(request, context)) } @@ -363,6 +461,7 @@ mod tests { ThinkingContext { capabilities, budgets: ThinkingBudgets::default(), + ..ThinkingContext::default() } } @@ -1239,10 +1338,102 @@ mod tests { let context = ThinkingContext { capabilities, budgets: overridden_budgets(overrides), + ..ThinkingContext::default() }; assert_eq!( translate_thinking(request(input), &context), Ok(request(expected)) ); } + + use serde_json::json; + + fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext { + ThinkingContext { + drop_params, + keeps_display_updates, + ..context(opus_4_7()) + } + } + + #[rstest] + #[case::float_budget(json!(8192.0), "xhigh")] + #[case::float_below_a_tier(json!(1023.9), "low")] + #[case::string_budget(json!(" 16_000 "), "xhigh")] + #[case::negative_budget(json!(-5), "low")] + fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) { + let translated = translate_thinking( + request( + json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}), + ), + &legacy_context(false, false), + ); + assert_eq!( + translated, + Ok(request(json!({ + "max_tokens": 64000, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": effort}, + }))) + ); + } + + #[rstest] + #[case::text(json!("lots"), "str")] + #[case::list(json!([1]), "list")] + #[case::null(Value::Null, "NoneType")] + #[case::boolean(json!(true), "bool")] + fn a_non_numeric_legacy_budget_is_rejected_unless_dropped( + #[case] budget: Value, + #[case] actual: &'static str, + ) { + let fields = + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}}); + assert_eq!( + translate_thinking(request(fields.clone()), &legacy_context(false, false)), + Err(Error::InvalidRequest( + crate::ErrorDetail::NonNumericThinkingBudget { actual } + )) + ); + assert_eq!( + translate_thinking(request(fields), &legacy_context(true, false)), + Ok(request( + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}) + )) + ); + } + + #[rstest] + #[case::summarized("summarized", false, Some("summarized"))] + #[case::updates_kept("updates", true, Some("updates"))] + #[case::updates_dropped("updates", false, None)] + #[case::unknown("verbose", true, None)] + fn legacy_display_survives_only_where_python_keeps_it( + #[case] display: &str, + #[case] keeps_display_updates: bool, + #[case] kept: Option<&str>, + ) { + let translated = translate_thinking( + request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})), + &legacy_context(false, keeps_display_updates), + ) + .unwrap(); + let thinking = serde_json::to_value(translated.params.thinking).unwrap(); + assert_eq!(thinking.get("display").and_then(Value::as_str), kept); + } + + #[rstest] + #[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))] + #[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))] + fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) { + let translated = translate_thinking( + request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})), + &legacy_context(drop_params, false), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(translated.params.thinking).unwrap(), + expected + ); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index fb6d7259bf9..18789e5b2be 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - transform_messages_request(request, context) + transform_messages_request(request, &context.keeping_display_updates()) } fn secret_names(&self) -> &'static [&'static str] { @@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet { .then_some(AnthropicBeta::PerTurnControl20260701), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), + uses_thinking_block_binding(params) + .then_some(AnthropicBeta::ThinkingBindingControls20260801), messages_carry_tool_changes(&request.messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), @@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool { display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates) } +/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`. +fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool { + match params.thinking.as_ref() { + Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Unrecognized(Value::Object(thinking))) => { + thinking.contains_key("block_binding") + } + Some(Recognized::Unrecognized(_)) | None => false, + } +} + fn is_tool_reference_change(block: &ContentBlock) -> bool { matches!( &block.block_type, @@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error }) } -fn drop_unsupported_params( +/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`. +pub fn drop_unsupported_params( request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { @@ -838,6 +860,8 @@ mod tests { low: 2000, ..ThinkingBudgets::default() }, + drop_params: true, + keeps_display_updates: false, }, drop_params: true, } diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index a2ba2cee0d3..d13cfa81a94 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { .capabilities .supports_mid_conversation_system, )), - context, + &context.keeping_display_updates(), ) } diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index e59a63813b0..b514e975aa4 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -110,6 +110,11 @@ impl ThinkingBudgets { pub struct ThinkingContext { pub capabilities: MessagesModelCapabilities, pub budgets: ThinkingBudgets, + /// `drop_params` as Python's thinking translation reads it. + pub drop_params: bool, + /// The provider accepts `display: "updates"` on thinking translated from the legacy shape + /// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`). + pub keeps_display_updates: bool, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct MessagesTransformContext { @@ -130,8 +135,22 @@ impl MessagesTransformContext { thinking: ThinkingContext { capabilities, budgets: ThinkingBudgets::from_lookup(env), + drop_params, + keeps_display_updates: false, }, drop_params, } } + + /// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`. + #[must_use] + pub fn keeping_display_updates(&self) -> Self { + Self { + thinking: ThinkingContext { + keeps_display_updates: true, + ..self.thinking + }, + ..*self + } + } } diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index b3d1421083a..2844fbf3e01 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -92,6 +92,24 @@ impl Error { } } + pub fn upstream_headers(&self) -> &[(String, String)] { + match self { + Self::Http(error) => error.upstream_headers(), + _ => &[], + } + } + + pub fn is_body_read_timeout(&self) -> bool { + matches!(self, Self::Http(error) if error.is_body_read_timeout()) + } + + pub fn is_timeout(&self) -> Option { + match self { + Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()), + _ => None, + } + } + pub fn upstream_response(&self) -> Option<(u16, &str)> { match self { Self::Http(error) => error.upstream_response(), @@ -158,6 +176,10 @@ pub enum ErrorDetail { value: String, hint: String, }, + #[error( + "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" + )] + NonNumericThinkingBudget { actual: &'static str }, #[error( "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params" )] diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index e5e3a056fab..038617c2e10 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - let request = transform_messages_request(request, context)?; + let request = transform_messages_request(request, &context.keeping_display_updates())?; let request = strip_cache_control_scope(normalize_system_role_messages( request, context diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs index 4c3fc38a01a..7acfc781431 100644 --- a/litellm-rust/crates/model-catalog/src/lower.rs +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -104,6 +104,7 @@ impl From<&EntryWire> for Adjustments { (Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift), (Uplift::Residency(Residency::Eu), wire.residency_uplift_eu), (Uplift::Residency(Residency::Us), wire.residency_uplift_us), + (Uplift::FedrampEndpoint, wire.fedramp_uplift), ] .into_iter() .filter_map(|(uplift, factor)| Some((uplift, factor?))), @@ -746,7 +747,8 @@ mod tests { fn uplifts_are_adjustments_not_prices() { let wire = wire(json!({ "regional_endpoint_uplift_multiplier": 1.1, - "regional_processing_uplift_multiplier_eu": "1.2" + "regional_processing_uplift_multiplier_eu": "1.2", + "fedramp_uplift_multiplier": 1.1 })); let adjustments = Adjustments::from(&wire); @@ -764,6 +766,12 @@ mod tests { Some(1.2) ); assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None); + assert_eq!( + adjustments + .get(&Uplift::FedrampEndpoint) + .map(Multiplier::get), + Some(1.1) + ); } #[rstest] diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 141702bf857..afbbfec59e6 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -96,6 +96,7 @@ pub struct ModelInfoFields { pub output_cost_per_token_ultrafast: Option, pub regional_processing_uplift_multiplier_eu: Option, pub regional_processing_uplift_multiplier_us: Option, + pub fedramp_uplift_multiplier: Option, pub regional_endpoint_uplift_multiplier: Option, pub output_cost_per_audio_token: Option, pub output_cost_per_character: Option, diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs index 66f9933a3e3..6fba985a8bd 100644 --- a/litellm-rust/crates/model-catalog/src/schema.rs +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -29,6 +29,8 @@ struct PriceBlocks { regional_processing_uplift_multiplier_eu: Option, /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%). regional_processing_uplift_multiplier_us: Option, + /// Multiplier applied to all token costs when served from the OpenAI FedRAMP endpoint (e.g. 1.10 = +10%). + fedramp_uplift_multiplier: Option, /// USD cost per web search query, keyed by search context size. search_context_cost_per_query: Option, /// Context-length or result-count tiered rates; each tier's costs apply within its range. diff --git a/litellm-rust/crates/model-catalog/src/vocabulary.rs b/litellm-rust/crates/model-catalog/src/vocabulary.rs index 2b7ac8bd1ab..6bf2cc55224 100644 --- a/litellm-rust/crates/model-catalog/src/vocabulary.rs +++ b/litellm-rust/crates/model-catalog/src/vocabulary.rs @@ -93,6 +93,12 @@ pub enum Endpoint { Realtime, #[serde(rename = "/v1/realtime/transcription_sessions")] RealtimeTranscriptionSessions, + #[serde(rename = "/v1/realtime/translations")] + RealtimeTranslations, + #[serde(rename = "/v1/realtime/translations/calls")] + RealtimeTranslationCalls, + #[serde(rename = "/v1/realtime/translations/client_secrets")] + RealtimeTranslationClientSecrets, #[serde(rename = "/v1/videos")] Videos, #[serde(rename = "/v1/ocr")] diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 307f25f0bda..401634f8c62 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -93,6 +93,10 @@ attributes! { search_free_requests: SearchFreeRequests, /// URL of the provider pricing/model page this entry was taken from. source: String, + /// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists. + successor_model: String, + /// URL of the provider page or feed `deprecation_date` was read from. + deprecation_source: String, /// Audio container formats the model can return. supported_audio_formats: Vec, /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. @@ -303,6 +307,10 @@ mod tests { #[case::known_mode("mode", json!("chat"), true)] #[case::unknown_mode("mode", json!("chatt"), false)] #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)] + #[case::successor_model("successor_model", json!("gpt-6-luna"), true)] + #[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)] + #[case::deprecation_source("deprecation_source", json!("https://developers.openai.com/api/docs/deprecations"), true)] + #[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)] #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)] #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)] #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] diff --git a/litellm-rust/crates/model-catalog/src/wire/entry.rs b/litellm-rust/crates/model-catalog/src/wire/entry.rs index 9c155c6496a..d0bdb394720 100644 --- a/litellm-rust/crates/model-catalog/src/wire/entry.rs +++ b/litellm-rust/crates/model-catalog/src/wire/entry.rs @@ -24,6 +24,7 @@ pub struct EntryWire { pub regional_endpoint_uplift: Option, pub residency_uplift_eu: Option, pub residency_uplift_us: Option, + pub fedramp_uplift: Option, pub unknown: Fields, pub problems: Vec, } @@ -57,6 +58,7 @@ impl EntryWire { "regional_processing_uplift_multiplier_us" => { self.residency_uplift_us = reader.value(path(), value) } + "fedramp_uplift_multiplier" => self.fedramp_uplift = reader.value(path(), value), _ => self.read_open_key(reader, key, value), } } diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index e3ef45f7876..ba892199dcf 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -25,12 +25,11 @@ litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true +async-trait.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true litellm-error.workspace = true -litellm-cache-memory.workspace = true -litellm-cache-redis.workspace = true litellm-cache-response.workspace = true serde.workspace = true litellm-auth.workspace = true @@ -49,6 +48,7 @@ litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "has litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true +litellm-router-core.workspace = true litellm-host-python.workspace = true litellm-model-catalog.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } @@ -56,7 +56,6 @@ pyo3.workspace = true prost.workspace = true pyo3-async-runtimes.workspace = true reqwest.workspace = true -redis = { workspace = true, features = ["tls-rustls"] } serde_json.workspace = true strum.workspace = true veil.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md index 41c82187373..e79562b8762 100644 --- a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md @@ -1,7 +1,7 @@ # Cache boundary -This folder owns how Rust inference reaches the selected cache: global cache selection, route admission, delegation to a Python cache and the experimental V2 native handles. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates +This folder owns how Rust inference reaches the selected Python cache. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates -`mod.rs` exposes the cache boundary to routes and module registration. Adapter directories remain private. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache, and owns route admission and inference protocol composition for both adapters +`mod.rs` exposes the cache boundary to routes. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache and owns route admission and inference protocol composition -`python/` delegates operations to the selected Python cache without discovering configuration. `native/` owns the experimental V2 native cache handles. Neither adapter depends on shared selection or the other adapter. Shared composition depends on the adapters, and routes use only the parent module's exports +`python/` delegates operations to the selected Python cache without discovering configuration. Routes use only the parent module's exports diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index 87dba3efcb1..4e3847951d8 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -1,32 +1,15 @@ -mod native; mod python; mod selection; -pub(crate) use native::NativeCacheHandle; pub(crate) use python::{CacheCall, PythonCache}; -pub(crate) use selection::{Cached, Selection, configure, configured_native}; +pub(crate) use selection::{Cached, Selection, configure, ensure_cache_disabled}; -use litellm_cache::Error; -use pyo3::{ - exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, - prelude::*, -}; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; -/// A cache call the bridge rejects itself, before or after the cache ran. #[derive(Debug, thiserror::Error)] pub(crate) enum HandleError { - #[error("recreate the v2 cache after fork")] - Forked, - #[error("cache limits must be positive")] - Limits, - #[error("namespace and a positive cache limit are required")] - RedisLimits, - #[error("cache durations must be finite and positive")] - Duration(#[source] Option), - #[error("unsupported v2 cache control: {0}")] - Control(String), - #[error("native inference needs a v2 cache, the catalog keeps Python caches on Python")] - NotNative, + #[error("this inference route does not support configured caches")] + Unsupported, #[error("cache operation without configured cache")] Unconfigured, #[error("cache arguments unavailable")] @@ -39,66 +22,10 @@ impl From for PyErr { fn from(error: HandleError) -> Self { let message = litellm_error::Report::new(&error).to_string(); match error { - HandleError::Limits - | HandleError::RedisLimits - | HandleError::Duration(_) - | HandleError::Control(_) => PyValueError::new_err(message), - HandleError::Forked - | HandleError::NotNative + HandleError::Unsupported | HandleError::Unconfigured | HandleError::MissingArguments | HandleError::NothingPending => PyRuntimeError::new_err(message), } } } - -fn cache_error(error: Error) -> PyErr { - let message = litellm_error::Report::new(&error).to_string(); - match error { - Error::InvalidEntry(_) => PyValueError::new_err(message), - Error::UnsupportedOperation => PyNotImplementedError::new_err(message), - Error::Unavailable(_) | Error::UnscopedFlush | Error::MissingPrompt => { - PyRuntimeError::new_err(message) - } - } -} - -#[cfg(test)] -mod tests { - use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, - }; - use rstest::rstest; - - use super::{Error, cache_error}; - - fn refused() -> std::io::Error { - std::io::Error::other("connection refused") - } - - #[rstest] - #[case::unavailable( - Error::unavailable(refused()), - false, - "cache is unavailable: connection refused" - )] - #[case::invalid_entry( - Error::invalid_entry(refused()), - true, - "invalid cache entry: connection refused" - )] - fn python_sees_the_class_and_the_whole_chain( - #[case] error: Error, - #[case] caller_fault: bool, - #[case] message: &str, - ) { - Python::initialize(); - Python::attach(|py| { - let failure = cache_error(error); - assert_eq!(failure.is_instance_of::(py), caller_fault); - assert_eq!(failure.is_instance_of::(py), !caller_fault); - assert_eq!(failure.value(py).to_string(), message); - }); - } -} diff --git a/litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md deleted file mode 100644 index 09a34d188cd..00000000000 --- a/litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md +++ /dev/null @@ -1,9 +0,0 @@ -# Native cache bindings - -This directory exposes the experimental V2 native cache handles (`NativeCacheHandle`) to Python. Cache algorithms and storage protocols remain in their cache crates - -Accept the cache object selected by the parent module. Do not read global `litellm.cache`, decide route admission, or select the Python cache adapter here - -Keep Python-facing cache classes and method signatures stable when reorganizing modules. Native internals stay private to this directory unless the shared cache boundary or Python module registration needs them - -Verify changes with the V2 cache tests using a freshly built extension. Test cache behavior, not module paths or file structure diff --git a/litellm-rust/crates/python-bridge/src/cache/native/mod.rs b/litellm-rust/crates/python-bridge/src/cache/native/mod.rs deleted file mode 100644 index da1460f90ed..00000000000 --- a/litellm-rust/crates/python-bridge/src/cache/native/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(super) mod v2; - -pub(crate) use v2::NativeCacheHandle; diff --git a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs b/litellm-rust/crates/python-bridge/src/cache/native/v2.rs deleted file mode 100644 index c54cb641843..00000000000 --- a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs +++ /dev/null @@ -1,336 +0,0 @@ -use crate::cache::{HandleError, cache_error}; -use std::{ - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use litellm_cache::{DeleteCache, DisconnectCache, PingCache}; -use litellm_host_python::{from_py, release_gil, to_py}; -use serde_json::Value; - -use litellm_cache_memory::InMemoryCache; -use litellm_cache_redis::{RedisCache, RedisTopology}; -use litellm_cache_response::{ - CacheEntry, CacheKeyInput, ExactResponseCache, ResponseCache, ResponseCacheCodec, - ResponseCacheConfig, ResponseCacheRequest, ResponseCacheService, -}; -use pyo3::{prelude::*, types::PyDict}; - -#[pyclass( - frozen, - name = "NativeCacheHandle", - module = "litellm.rust_bridge._native" -)] -pub(crate) struct NativeCacheHandle { - service: Arc, - backend: Arc, - storage: Storage, - pid: u32, -} - -#[derive(Clone)] -enum Storage { - Memory(Arc>), - Redis(Arc>), -} - -impl NativeCacheHandle { - fn check_process(&self) -> PyResult<()> { - if self.pid != std::process::id() { - return Err(HandleError::Forked.into()); - } - Ok(()) - } -} - -fn request(key: String, ttl: Option) -> PyResult { - let mut request: ResponseCacheRequest = ResponseCacheRequest::new(CacheKeyInput { - preset: Some(key), - ..Default::default() - }); - request.context.ttl = ttl.map(duration).transpose()?; - Ok(request) -} - -#[pymethods] -impl NativeCacheHandle { - #[staticmethod] - #[pyo3(signature = (*, ttl=600.0, capacity=200, max_entry_bytes=4194304))] - fn memory(ttl: f64, capacity: usize, max_entry_bytes: usize) -> PyResult { - let ttl = duration(ttl)?; - if capacity == 0 || max_entry_bytes == 0 { - return Err(HandleError::Limits.into()); - } - let storage = Arc::new(InMemoryCache::new(Some(capacity), Some(ttl))); - let backend = Arc::new(ResponseCache::new(storage.clone()).with_config( - ResponseCacheConfig { - namespace: "sdk".into(), - max_entry_bytes, - }, - )); - Ok(Self { - service: backend.clone(), - backend, - storage: Storage::Memory(storage), - pid: std::process::id(), - }) - } - - #[staticmethod] - #[pyo3(signature = (url, *, namespace, ttl=600.0, max_entry_bytes=4194304))] - fn redis( - py: Python<'_>, - url: &str, - namespace: String, - ttl: f64, - max_entry_bytes: usize, - ) -> PyResult { - let ttl = duration(ttl)?; - if namespace.is_empty() || max_entry_bytes == 0 { - return Err(HandleError::RedisLimits.into()); - } - let storage = Arc::new( - release_gil(py, || { - RedisCache::connect( - url, - &RedisTopology::Standalone, - Some(ttl), - ResponseCacheCodec, - ) - }) - .map_err(cache_error)? - .with_namespace(Some(namespace.clone())), - ); - let backend = Arc::new(ResponseCache::new(storage.clone()).with_config( - ResponseCacheConfig { - namespace, - max_entry_bytes, - }, - )); - Ok(Self { - service: backend.clone(), - backend, - storage: Storage::Redis(storage), - pid: std::process::id(), - }) - } - fn get(&self, py: Python<'_>, key: String) -> PyResult> { - self.check_process()?; - let request = request(key, None)?; - let value = - release_gil(py, || self.backend.lookup(&request, now())).map_err(cache_error)?; - to_py(py, &value) - } - - #[pyo3(signature = (key, value, *, ttl=None))] - fn set( - &self, - py: Python<'_>, - key: String, - value: &Bound<'_, PyAny>, - ttl: Option, - ) -> PyResult<()> { - self.check_process()?; - let request = request(key, ttl)?; - let value: Value = from_py(value)?; - release_gil(py, || self.backend.store(&request, value, now())).map_err(cache_error) - } - - fn async_get<'py>(&self, py: Python<'py>, key: String) -> PyResult> { - self.check_process()?; - let request = request(key, None)?; - let backend = self.backend.clone(); - crate::execution::run_async( - py, - async move { backend.async_lookup(&request, now()).await }, - cache_error, - ) - } - - #[pyo3(signature = (key, value, *, ttl=None))] - fn async_set<'py>( - &self, - py: Python<'py>, - key: String, - value: &Bound<'_, PyAny>, - ttl: Option, - ) -> PyResult> { - self.check_process()?; - let request = request(key, ttl)?; - let value: Value = from_py(value)?; - let backend = self.backend.clone(); - crate::execution::run_async( - py, - async move { backend.async_store(&request, value, now()).await }, - cache_error, - ) - } - - #[pyo3(signature = (entries, *, ttl=None))] - fn async_set_many<'py>( - &self, - py: Python<'py>, - entries: &Bound<'_, PyAny>, - ttl: Option, - ) -> PyResult> { - self.check_process()?; - let entries: Vec<(String, Value)> = from_py(entries)?; - let entries = entries - .into_iter() - .map(|(key, value)| Ok((request(key, ttl)?, value))) - .collect::>>()?; - let backend = self.backend.clone(); - crate::execution::run_async( - py, - async move { backend.async_store_batch(entries, now()).await }, - cache_error, - ) - } - - fn flush(&self, py: Python<'_>) -> PyResult> { - self.check_process()?; - let backend = self.backend.clone(); - crate::execution::run_sync(py, async move { backend.async_flush().await }, cache_error) - } - - fn async_flush<'py>(&self, py: Python<'py>) -> PyResult> { - self.check_process()?; - let backend = self.backend.clone(); - crate::execution::run_async(py, async move { backend.async_flush().await }, cache_error) - } - - fn ping<'py>(&self, py: Python<'py>) -> PyResult> { - self.check_process()?; - let storage = self.storage.clone(); - crate::execution::run_async( - py, - async move { - match storage { - Storage::Memory(_) => Ok(true), - Storage::Redis(cache) => cache.ping().await, - } - }, - cache_error, - ) - } - - fn disconnect<'py>(&self, py: Python<'py>) -> PyResult> { - self.check_process()?; - let storage = self.storage.clone(); - crate::execution::run_async( - py, - async move { - match storage { - Storage::Memory(cache) => cache.disconnect().await, - Storage::Redis(cache) => cache.disconnect().await, - } - }, - cache_error, - ) - } - - fn delete<'py>(&self, py: Python<'py>, keys: Vec) -> PyResult> { - self.check_process()?; - let storage = self.storage.clone(); - crate::execution::run_async( - py, - async move { - for key in keys { - match &storage { - Storage::Memory(cache) => cache.async_delete_cache(&key).await?, - Storage::Redis(cache) => cache.async_delete_cache(&key).await?, - } - } - Ok(()) - }, - cache_error, - ) - } -} - -fn duration(seconds: f64) -> Result { - match Duration::try_from_secs_f64(seconds) { - Ok(duration) if !duration.is_zero() => Ok(duration), - Ok(_) => Err(HandleError::Duration(None)), - Err(error) => Err(HandleError::Duration(Some(error))), - } -} - -pub(in crate::cache) fn native_handle<'py>( - configured: &Bound<'py, PyAny>, -) -> PyResult>> { - Ok(configured - .getattr_opt("cache")? - .map(|backend| backend.getattr_opt("native_handle")) - .transpose()? - .flatten() - .filter(|handle| handle.is_instance_of::())) -} - -pub(in crate::cache) fn configured( - configured: &Bound<'_, PyAny>, - kwargs: &Bound<'_, PyDict>, -) -> PyResult<( - Option>, - litellm_cache_response::CacheOptions, -)> { - let handle = native_handle(configured)?.ok_or(HandleError::NotNative)?; - let cache = handle.extract::>()?; - cache.check_process()?; - let controls = kwargs.get_item("cache")?.filter(|value| !value.is_none()); - let controls = controls - .as_ref() - .map(|value| value.cast::()) - .transpose()?; - if let Some(controls) = controls { - for name in controls.keys() { - let name = name.extract::()?; - if !matches!( - name.as_str(), - "no-cache" | "no-store" | "ttl" | "s-maxage" | "s-max-age" | "use-cache" - ) { - return Err(HandleError::Control(name).into()); - } - } - } - let boolean = |name: &str| -> PyResult { - controls - .map(|values| values.get_item(name)) - .transpose()? - .flatten() - .map(|value| value.extract()) - .transpose() - .map(|value| value.unwrap_or(false)) - }; - let seconds = |name: &str| -> PyResult> { - controls - .map(|values| values.get_item(name)) - .transpose()? - .flatten() - .map(|value| Ok(duration(value.extract()?)?)) - .transpose() - }; - Ok(( - Some(cache.service.clone()), - litellm_cache_response::CacheOptions { - policy: litellm_cache_response::CachePolicy { - caching: kwargs - .get_item("caching")? - .filter(|value| !value.is_none()) - .map(|value| value.extract()) - .transpose()?, - no_cache: boolean("no-cache")?, - no_store: boolean("no-store")?, - ttl: seconds("ttl")?, - max_age: seconds("s-max-age")?.or(seconds("s-maxage")?), - }, - scope: litellm_cache_response::CacheScope::Shared, - }, - )) -} - -fn now() -> Duration { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() -} diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index 00fc7cd822f..e96f079e5da 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -1,14 +1,10 @@ -use super::{native, python}; -use litellm_cache_response::{ - CacheOptions, CachePolicy, CacheScope, ResponseCacheService, ScopedCache, -}; +use super::{HandleError, python}; +use litellm_cache_response::{CacheOptions, CachePolicy, CacheScope, ScopedCache}; use litellm_host::{ machine::{HostServices, MachineFault}, protocol::Protocol, }; use pyo3::{prelude::*, types::PyDict}; -use std::sync::Arc; - const CACHE_HOST_MODULE: &str = "litellm.rust_bridge.host.cache"; pub(crate) struct Cached

(std::marker::PhantomData

); @@ -24,7 +20,6 @@ impl Protocol for Cached

{ enum Backend { Disabled, - Native(Arc), Python { namespace: String }, } @@ -43,7 +38,6 @@ impl Selection { { let service = match self.backend { Backend::Disabled => None, - Backend::Native(service) => Some(service), Backend::Python { namespace } => Some(python::service(services, namespace)), }; ( @@ -76,37 +70,21 @@ fn selected_cache<'py>( Ok(Some(configured)) } -pub(crate) fn configured_native( +pub(crate) fn ensure_cache_disabled( py: Python<'_>, kwargs: &Bound<'_, PyDict>, call_type: &str, -) -> PyResult<( - Option>, - litellm_cache_response::CacheOptions, -)> { +) -> PyResult<()> { let Some(configured) = selected_cache(py, kwargs, call_type)? else { - return Ok(( - None, - litellm_cache_response::CacheOptions::new(litellm_cache_response::CacheScope::Shared), - )); + return Ok(()); }; - native_configuration(&configured, kwargs) -} - -fn native_configuration( - configured: &Bound<'_, PyAny>, - kwargs: &Bound<'_, PyDict>, -) -> PyResult<(Option>, CacheOptions)> { - if !configured + if configured .call_method("should_use_cache", (), Some(kwargs))? .extract::()? { - return Ok(( - None, - litellm_cache_response::CacheOptions::new(litellm_cache_response::CacheScope::Shared), - )); + return Err(HandleError::Unsupported.into()); } - native::v2::configured(configured, kwargs) + Ok(()) } pub(crate) fn configure( @@ -122,13 +100,6 @@ pub(crate) fn configure( options: CacheOptions::new(CacheScope::Shared), }); }; - if native::v2::native_handle(&cache)?.is_some() { - let (native, options) = native_configuration(&cache, arguments)?; - return Ok(Selection { - backend: native.map_or(Backend::Disabled, Backend::Native), - options, - }); - } let enabled = cache .call_method("should_use_cache", (), Some(arguments))? .extract::()?; diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index b42c08b7e78..426cd861729 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -23,6 +23,17 @@ pyo3::create_exception!( ); const MESSAGES_REQUEST_ERROR: &str = "messages_request_error"; +/// The rejection is a parameter the model does not support, which Python raises as +/// `UnsupportedParamsError`. +const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param"; +/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or +/// `Body` for a streamed body read that timed out after the head arrived. +const MESSAGES_TIMEOUT: &str = "messages_timeout"; +/// No credential reached the provider, which Python raises as `AuthenticationError`. +const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential"; +/// Python refuses the request only once its logger knows the resolved model and provider (the +/// provider config's request transform), so the failure logs them. +const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection"; pub(crate) fn upstream_error( py: Python<'_>, @@ -78,17 +89,57 @@ pub(crate) fn route_failure( return Ok(raised); } if let Some((status, body)) = error.upstream_response() { - return public(upstream_error(py, status, body, &HeaderMap::new())?); + let headers = error + .upstream_headers() + .iter() + .filter_map(|(name, value)| { + Some(( + reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?, + reqwest::header::HeaderValue::from_str(value).ok()?, + )) + }) + .collect::(); + return public(upstream_error(py, status, body, &headers)?); } let native = match (marker, error.rejected_request()) { (RequestMarker::Messages, Some(rejection)) => { let native = PyValueError::new_err(Report::new(rejection).to_string()); - native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?; + let value = native.value(py); + value.setattr(MESSAGES_REQUEST_ERROR, true)?; + if matches!( + rejection.downcast_ref::(), + Some( + litellm_llms::ErrorDetail::UnsupportedParameter { .. } + | litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. } + ) + ) { + value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?; + } + if matches!( + rejection.downcast_ref::(), + Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }) + ) { + value.setattr(MESSAGES_LATE_REJECTION, true)?; + } native } - (RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => { - by_fault(error.is_request(), &error) + (RequestMarker::Messages, None) => { + let native = by_fault(error.is_request(), &error); + let value = native.value(py); + if let Some(connect) = error.timeout() { + let phase = match (connect, error.body_read_timeout()) { + (true, _) => "Connection", + (false, true) => "Body", + (false, false) => "Read", + }; + value.setattr(MESSAGES_TIMEOUT, phase)?; + } + if error.is_missing_credential() { + value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?; + } + native } + (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error), }; public(native) } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index fdcb969a423..f4a4f7c2b26 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,6 +12,7 @@ mod marshal; mod model_registry; mod preflight; mod python_settings; +mod router; mod routes; mod secrets; mod tokenizer; @@ -36,6 +37,8 @@ mod _native { #[pymodule_export] use crate::model_registry::bundled_model_cost_map; #[pymodule_export] + use crate::router::{RouterCall, RouterCoreHandle}; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -65,10 +68,6 @@ mod _native { fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); let dict = module.dict(); - dict.set_item( - "NativeCacheHandle", - py.get_type::(), - )?; dict.set_item( "_SecretManagerRuntime", py.get_type::(), @@ -92,7 +91,6 @@ mod tests { Python::initialize(); Python::attach(|py| { let mut expected = vec![ - "NativeCacheHandle", "RustBridgeDeclined", "RustUpstreamError", "ForkedAfterNativeRuntimeStarted", @@ -120,6 +118,8 @@ mod tests { "bundled_model_cost_map", "TokenCounter", "Tokenizer", + "RouterCoreHandle", + "RouterCall", "gil_stats", "process_state_started", "reserve_process_for_forking", diff --git a/litellm-rust/crates/python-bridge/src/router/config.rs b/litellm-rust/crates/python-bridge/src/router/config.rs new file mode 100644 index 00000000000..0b46f2208c5 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/config.rs @@ -0,0 +1,408 @@ +//! The documents `RouterCoreHandle` reads: the router's static and dynamic config, a call's +//! request and an `apply` change set. A config key router-core does not read declines the +//! handle, so Python keeps the router on its own core. + +use std::collections::BTreeMap; + +use litellm_router_core::select::PassThrough; +use litellm_router_core::{ + ChangeSet, ConfigError, DeploymentSpec, OutcomeCallbacks, RouterConfig, RouterDynamicConfig, + RouterRequest, + config::{Fallbacks, RetryPolicy}, + host::UncountedRetries, + mock::MockFailure, + registry::IndexOverrides, +}; +use serde::{Deserialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +/// The `RouterConfig` keys router-core reads. +const STATIC_KEYS: &[&str] = &[ + "model_list", + "routing_strategy", + "routing_groups", + "routing_group_members", + "plugins", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_policy", + "model_group_retry_policy", + "retry_after", + "timeout", + "request_timeout", + "drop_params", + "enable_pre_call_checks", + "routing_plugins", + "callback_filters", + "enable_health_check_routing", + "health_check_staleness_threshold", + "background_health_check_model_groups", + "enable_weighted_failover", + "fallback_on_unsupported_params", + "provider_default_deployment_ids", +]; + +/// The `RouterDynamicConfig` keys router-core reads. +const DYNAMIC_KEYS: &[&str] = &[ + "version", + "model_alias_map", + "model_group_alias", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "allowed_fails_is_default", + "fallback_keys", + "default_deployment", + "default_deployment_raises", +]; + +/// Router arguments that do not change routing, accepted in either config and dropped. +const IGNORED_KEYS: &[&str] = &[ + "set_verbose", + "debug_level", + "alerting_config", + "client_ttl", +]; + +/// Fallback lists Python treats as unset when empty (`context_window_fallbacks or None`). +const EMPTY_MEANS_UNSET: &[&str] = &["context_window_fallbacks", "content_policy_fallbacks"]; + +/// Which document a failure is about. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub(crate) enum Document { + #[strum(serialize = "router config")] + Static, + #[strum(serialize = "dynamic router config")] + Dynamic, + #[strum(serialize = "router request")] + Request, + #[strum(serialize = "change set")] + Changes, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("the {document} must be a JSON object")] + NotObject { document: Document }, + #[error("router-core does not read `{key}` in the {document}")] + UnknownKey { document: Document, key: String }, + #[error("the {document} does not have the shape router-core reads")] + Shape { + document: Document, + #[source] + source: serde_json::Error, + }, + #[error("the request names no model")] + MissingModel, + #[error("router-core rejected the router config")] + Core(#[from] ConfigError), +} + +impl Error { + /// Whether router-core cannot serve this config, so Python keeps its own core. + pub(crate) fn declined(&self) -> bool { + match self { + Self::UnknownKey { .. } | Self::Core(_) => true, + Self::Shape { document, .. } => { + matches!(document, Document::Static | Document::Dynamic) + } + Self::NotObject { .. } | Self::MissingModel => false, + } + } +} + +fn object(document: Document, value: Value) -> Result, Error> { + match value { + Value::Object(object) => Ok(object), + _ => Err(Error::NotObject { document }), + } +} + +fn shaped(document: Document, object: Map) -> Result { + serde_json::from_value(Value::Object(object)) + .map_err(|source| Error::Shape { document, source }) +} + +/// Drop unset and ignored keys, then reject any key outside `known`. +fn routing_keys( + document: Document, + value: Value, + known: &[&str], +) -> Result, Error> { + let mut object = object(document, value)?; + object.retain(|key, value| { + let empty_unset = EMPTY_MEANS_UNSET.contains(&key.as_str()) + && value.as_array().is_some_and(Vec::is_empty); + !(value.is_null() || empty_unset || IGNORED_KEYS.contains(&key.as_str())) + }); + if let Some(key) = object.keys().find(|key| !known.contains(&key.as_str())) { + return Err(Error::UnknownKey { + document, + key: key.clone(), + }); + } + Ok(object) +} + +pub(crate) fn parse_router_config(value: Value) -> Result { + let object = routing_keys(Document::Static, value, STATIC_KEYS)?; + shaped(Document::Static, object) +} + +pub(crate) fn parse_dynamic_config(value: Value) -> Result { + let object = routing_keys(Document::Dynamic, value, DYNAMIC_KEYS)?; + shaped(Document::Dynamic, object) +} + +/// A call's request as Python spells it. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestSpec { + model: Option, + team_id: Option, + listing_team_id: Option, + proxy_admin: Option, + allowed_models: Option>, + admin_model: Option, + specific_deployment: Option, + num_retries: Option, + uncounted_retries: Option, + disable_fallbacks: Option, + router_weights: Option, + pinned_deployment_id: Option, + routing_strategy: Option, + filter_tags: Option, + /// `listed` or `healthy`: which pass-through pick the request makes. + pass_through: Option, + target_order: Option, + excluded_ids: Option>, + retry_skipped_ids: Option>, + pre_routes: Option, + has_image: Option, + endpoint: Option, + web_search: Option, + route_handler: Option, + decisions_media: Option, + wildcard_request: Option, + unrouted: Option, + passthrough_on_no_deployment: Option, + skip_weighted_failover: Option, + same_group_fallbacks_only: Option, + model_group_retry_policy: Option>, + retry_policy_raises: Option>, + #[serde(default, deserialize_with = "present")] + fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + context_window_fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + content_policy_fallbacks: Option>, + max_fallbacks: Option, + mock_failure: Option, + cooldown_time: Option, + health_check_probe: Option, + outcome_callbacks: Option, + body: Option, +} + +/// A key the request passed, even as `None`: `Some(None)`. +fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +/// Read a request; `model` replaces the request's own when given. +pub(crate) fn parse_request(value: Value, model: Option<&str>) -> Result { + let spec: RequestSpec = shaped(Document::Request, object(Document::Request, value)?)?; + let model = model + .map(str::to_owned) + .or(spec.model) + .ok_or(Error::MissingModel)?; + Ok(RouterRequest { + model, + body: spec.body.unwrap_or_default(), + team_id: spec.team_id, + listing_team_id: spec.listing_team_id, + proxy_admin: spec.proxy_admin.unwrap_or(false), + allowed_models: spec.allowed_models.unwrap_or_default(), + admin_model: spec.admin_model, + specific_deployment: spec.specific_deployment.unwrap_or(false), + num_retries: spec.num_retries, + uncounted_retries: spec.uncounted_retries, + disable_fallbacks: spec.disable_fallbacks.unwrap_or(false), + router_weights: spec.router_weights, + pinned_deployment_id: spec.pinned_deployment_id, + routing_strategy: spec.routing_strategy, + filter_tags: spec.filter_tags.unwrap_or(false), + pass_through: match spec.pass_through.as_deref() { + None => None, + Some("healthy") => Some(PassThrough::Healthy), + Some(_) => Some(PassThrough::Listed), + }, + target_order: spec.target_order, + excluded_ids: spec.excluded_ids.unwrap_or_default(), + retry_skipped_ids: spec.retry_skipped_ids.unwrap_or_default(), + pre_routes: spec.pre_routes.unwrap_or(false), + has_image: spec.has_image.unwrap_or(false), + endpoint: spec.endpoint, + web_search: spec.web_search.unwrap_or(false), + route_handler: spec.route_handler, + decisions_media: spec.decisions_media.unwrap_or(false), + wildcard_request: spec.wildcard_request, + unrouted: spec.unrouted.unwrap_or(false), + passthrough_on_no_deployment: spec.passthrough_on_no_deployment.unwrap_or(false), + skip_weighted_failover: spec.skip_weighted_failover.unwrap_or(false), + same_group_fallbacks_only: spec.same_group_fallbacks_only.unwrap_or(false), + model_group_retry_policy: spec.model_group_retry_policy, + retry_policy_raises: spec.retry_policy_raises.unwrap_or_default(), + fallbacks: spec.fallbacks, + context_window_fallbacks: spec.context_window_fallbacks, + content_policy_fallbacks: spec.content_policy_fallbacks, + max_fallbacks: spec.max_fallbacks, + mock_failure: spec.mock_failure, + cooldown_time: spec.cooldown_time, + health_check_probe: spec.health_check_probe.unwrap_or(false), + outcome_callbacks: spec.outcome_callbacks.unwrap_or_default(), + payload: None, + }) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ChangesSpec { + /// Every deployment, replacing the current list before the upserts and deletes. + #[serde(default)] + replace: Option>, + /// Deployments added at the end, beside any that share their id. + #[serde(default)] + append: Vec, + #[serde(default)] + upsert: Vec, + #[serde(default)] + delete: Vec, + /// A new dynamic document (the same keys `RouterCoreHandle(static, dynamic)` takes). + #[serde(default)] + settings: Option, + /// The registry indexes the host wrote (`IndexOverrides`); `{}` drops them. + #[serde(default)] + index: Option, + /// The deployment at the id map's position for this id, removed with every index shifted. + #[serde(default)] + remove: Option, + /// A write to the indexes the host hands out (`HostIndexes`), field by field. + #[serde(default)] + host_index: Option, +} + +pub(crate) fn parse_changes(value: Value) -> Result { + let spec: ChangesSpec = shaped(Document::Changes, object(Document::Changes, value)?)?; + Ok(ChangeSet { + replace: spec.replace, + appends: spec.append, + upserts: spec.upsert, + deletes: spec.delete, + settings: spec.settings.map(parse_dynamic_config).transpose()?, + index: spec.index, + remove: spec.remove, + host_index: spec.host_index, + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::unknown_key(json!({"routing_strategy_args": {"ttl": 1}}), "routing_strategy_args")] + #[case::dynamic_key_in_static(json!({"num_retries": 2}), "num_retries")] + fn an_unread_static_key_declines_naming_the_key(#[case] json: Value, #[case] key: &str) { + let error = parse_router_config(json).unwrap_err(); + assert!(error.declined()); + assert!(matches!(&error, Error::UnknownKey { key: named, .. } if named == key)); + } + + #[rstest] + #[case::null_unknown(json!({"routing_strategy_args": null}))] + #[case::ignored(json!({"set_verbose": true, "debug_level": "INFO"}))] + #[case::pre_call_checks_off(json!({"enable_pre_call_checks": false}))] + #[case::pre_call_checks_on(json!({"enable_pre_call_checks": true}))] + fn unset_and_ignored_keys_are_accepted(#[case] json: Value) { + parse_router_config(json).unwrap(); + } + + #[rstest] + #[case::not_object(json!([]), false)] + #[case::bad_shape(json!({"retry_after": "soon"}), true)] + fn a_config_that_is_not_an_object_is_a_value_error_and_a_wrong_shape_declines( + #[case] json: Value, + #[case] declined: bool, + ) { + assert_eq!(parse_router_config(json).unwrap_err().declined(), declined); + } + + #[rstest] + fn python_router_defaults_read_as_unset() { + let dynamic = parse_dynamic_config(json!({ + "num_retries": null, "fallbacks": [], "context_window_fallbacks": [], + "content_policy_fallbacks": [], "model_group_alias": {"gpt": "gpt-4"}, + })) + .unwrap(); + assert_eq!(dynamic.num_retries, None); + assert!(dynamic.context_window_fallbacks.is_none()); + assert!(dynamic.content_policy_fallbacks.is_none()); + assert_eq!(dynamic.model_alias_map["gpt"], "gpt-4"); + } + + #[rstest] + #[case::own_model(json!({"model": "gpt", "num_retries": 1}), None, "gpt")] + #[case::override_model(json!({"model": "gpt"}), Some("other"), "other")] + fn a_request_reads_its_model( + #[case] json: Value, + #[case] model: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!(parse_request(json, model).unwrap().model, expected); + } + + #[rstest] + #[case::negative("negative", UncountedRetries::Negative)] + #[case::not_positive("not_positive", UncountedRetries::NotPositive)] + #[case::raises("raises", UncountedRetries::Raises)] + fn a_request_reads_a_num_retries_that_is_no_count( + #[case] spelled: &str, + #[case] expected: UncountedRetries, + ) { + let request = + parse_request(json!({"model": "gpt", "uncounted_retries": spelled}), None).unwrap(); + assert_eq!(request.uncounted_retries, Some(expected)); + assert_eq!(request.num_retries, None); + } + + #[rstest] + #[case::unknown_field(json!({"model": "gpt", "temperature": 1}))] + #[case::no_model(json!({"team_id": "t"}))] + fn a_bad_request_is_a_value_error(#[case] json: Value) { + assert!(!parse_request(json, None).unwrap_err().declined()); + } + + #[rstest] + fn a_change_set_reads_upserts_and_deletes() { + let changes = parse_changes(json!({ + "upsert": [{"model_name": "m", "litellm_params": {"model": "openai/x"}}], + "delete": ["old"], + })) + .unwrap(); + assert_eq!(changes.upserts.len(), 1); + assert_eq!(changes.deletes, ["old"]); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/host.rs b/litellm-rust/crates/python-bridge/src/router/host.rs new file mode 100644 index 00000000000..16b425d69c6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/host.rs @@ -0,0 +1,1569 @@ +//! The host behind `RouterCoreHandle`. Rust never awaits Python: each host operation the call +//! loop needs becomes a [`HostCall`] parked in the call's [`Exchange`], and the call future stays +//! pending until Python runs it and hands the [`HostCallResponse`] back through [`Driver::resume`]. +//! +//! The driver polls the call future with a no-op waker while the call's exchange is bound to the +//! current thread, so a host future finds its exchange without the router passing it through. +//! +//! The routing cache is a host operation too: [`StepCache`] parks every [`RouterCache`] op as a +//! `cache` step, and Python runs it against the router's own `DualCache` (memory plus Redis), so +//! Python and Rust read and write the same cooldowns, fail counters and usage. + +use std::{ + cell::RefCell, + future::{Future, poll_fn}, + pin::Pin, + sync::{Arc, Mutex, MutexGuard, PoisonError}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use async_trait::async_trait; +use litellm_router_core::{ + Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, Deployment, Error, Host, Output, + Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, RouterEvent, RouterRequest, + RouterResponse, RoutingContext, SecretRef, SelectStage, ThreadRandomness, + build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo, + precall::PreCallFacts, +}; +use pyo3::{PyResult, Python, types::PyAnyMethods}; +use serde_json::{Map, Value}; + +/// The `kind` of a step dict. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum StepKind { + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "sleep")] + Sleep, + #[strum(serialize = "allow_fallback")] + AllowFallback, + #[strum(serialize = "refuse_fallbacks")] + RefuseFallbacks, + #[strum(serialize = "select")] + Select, + #[strum(serialize = "pre_route")] + PreRoute, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "secrets")] + Secrets, + #[strum(serialize = "validate")] + Validate, + #[strum(serialize = "provider")] + Provider, + #[strum(serialize = "added")] + Added, + #[strum(serialize = "model_info")] + ModelInfo, + #[strum(serialize = "done")] + Done, + #[strum(serialize = "failed")] + Failed, +} + +/// Which selection stage Python is asked to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Stage { + /// Pre-call checks and sub-routers: return the deployments that remain. + Plugins, + /// A strategy router-core does not implement: return the candidates with the pick first. + Strategy(String), + /// Tag routing: return the deployments that remain. + Tags, + /// `enable_pre_call_checks`: return the facts the filter decides by. + PreCall, + /// `_deployment_declares_no_vision`: return the candidates that declare no vision support. + TextOnly, + /// `_deployment_serves_route`: return the candidates that cannot serve `handler`'s call. + RouteIncapable { handler: String }, + /// `_decisions_deployment_refuses_media`: return the candidates that refuse the call's media. + MediaRefusing, + /// The routing plugins: run them when `run`, then return the candidate models they left. + RoutingPlugins { run: bool }, + /// The host's `CustomLogger` deployment filters: return the ids that remain, or a failure. + /// They read the ids this request already failed on, as Python's request kwargs carry them. + CallbackFilters { + excluded_ids: Vec, + skipped_ids: Vec, + target_order: Option, + }, + /// `f"{provider}/{model}"` for a name no wildcard pattern matches as spelled: return it as + /// the one entry of `models`. + ProviderRequest, +} + +/// The `op` of a `cache` step: one per [`RouterCache`] method. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum CacheOp { + /// `async_get_cache(key)`: answered with the stored JSON value, or `None`. + #[strum(serialize = "get")] + Get, + /// `async_set_cache(key, value, ttl)`: answered with `None`. + #[strum(serialize = "set")] + Set, + /// `async_increment(key, value, ttl)`: answered with the new number. The TTL applies when + /// the key is created. + #[strum(serialize = "increment")] + Increment, + /// `async_increment_usage(keys, value, ttl)`: `value` is one amount per key, added as + /// `_increment_deployment_usage` adds them. Answered with `None`. + #[strum(serialize = "increment_usage")] + IncrementUsage, + /// `async_batch_get_cache(keys)`: answered with one JSON value or `None` per key, in order. + #[strum(serialize = "batch_get")] + BatchGet, + /// `async_delete_cache(key)`: answered with `None`. + #[strum(serialize = "delete")] + Delete, +} + +/// Python's outcome of a `cache` step. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum CacheOutcome { + /// `get`: the stored value (`None` when missing). + Value(Option), + /// `batch_get`: one entry per key. + Values(Vec>), + /// `increment`: the counter after the increment. + Number(f64), + /// `set`, `delete` and `increment_usage`. + Stored, +} + +impl CacheOutcome { + /// The op this outcome answers (`Stored` answers both `set` and `delete`). + fn answers(&self, op: CacheOp) -> bool { + matches!( + (self, op), + (Self::Value(_), CacheOp::Get) + | (Self::Values(_), CacheOp::BatchGet) + | (Self::Number(_), CacheOp::Increment) + | ( + Self::Stored, + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage + ) + ) + } +} + +/// A fallback chain's dict targets, which can carry an api key. +#[derive(Clone, PartialEq, veil::Redact)] +pub(crate) struct DictTargets(#[redact(with = "[REDACTED]")] pub(crate) Vec>); + +/// What a call is blocked on, for Python to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum HostCall { + Attempt { + deployment_id: String, + /// The picked deployment as configured: `model_name`, `litellm_params`, `model_info` + /// (with its `id`). The host builds the request from it; it never looks the deployment up + /// in a Python router. + deployment: Value, + /// The name the attempt resolved: the requested name or the fallback target. + model: String, + model_group: String, + /// The `order` level of a same-group order fallback. + target_order: Option, + /// Every deployment `model` resolved to for this caller, the picked one included. + candidate_ids: Vec, + retry: u32, + fallback_depth: u32, + fallback_level: u32, + timeout: Option, + /// Request params the fallback walk's dict targets merged in. + params: Option>, + /// The walk level whose request metadata the attempt carries. + scope: u32, + max_retries: u32, + model_group_size: usize, + }, + Sleep { + seconds: f64, + }, + AllowFallback { + target: String, + }, + RefuseFallbacks { + /// The walk level whose chain the targets are. + level: u32, + targets: DictTargets, + }, + Select { + stage: Stage, + model_group: String, + /// The name the request resolved from, which strategies key their state by. + name: String, + /// Whose selector a strategy reads: `default`, `group` (with `group`) or `override`. + scope: &'static str, + group: Option, + /// The deployments the name resolved to, before health filters (the tag stage reads them). + pool: Vec, + candidates: Vec, + }, + /// Pre-routing before a pick resolves `model`: answered with the model group to resolve + /// instead (a string), a custom strategy's deployment document (a mapping), or `None`. + PreRoute { + model: String, + fallback_depth: u32, + retry: u32, + }, + /// One [`RouterCache`] op against the host's cache. `key` is set for every op but + /// `batch_get`, which sets `keys`; `value` is set for `set` (the JSON to store) and + /// `increment` (the amount); `ttl` is in seconds. + Cache { + op: CacheOp, + key: Option, + keys: Option>, + value: Option, + ttl: Option, + /// The op reads or writes this worker's in-memory tier only (`local_only=True`). + local: bool, + }, + /// The deployment build's `os.environ/` references: answered with one value per ref. + Secrets { + index: usize, + refs: Vec, + }, + /// The deployment build's `validate`, `provider` and `added` steps for one deployment. + Build { + kind: StepKind, + path: BuildPath, + index: usize, + entry: Value, + declared_id: Option, + duplicate_id: bool, + /// The provider the `provider` step resolved (`added` only). + provider: Option, + }, + /// One lookup of a model-info query ([`ModelInfoLookup`], as its `lookup` field and + /// arguments): answered as the lookup documents. + ModelInfo { + lookup: Value, + }, +} + +impl HostCall { + pub(crate) fn kind(&self) -> StepKind { + match self { + Self::Attempt { .. } => StepKind::Attempt, + Self::Sleep { .. } => StepKind::Sleep, + Self::AllowFallback { .. } => StepKind::AllowFallback, + Self::RefuseFallbacks { .. } => StepKind::RefuseFallbacks, + Self::Select { .. } => StepKind::Select, + Self::PreRoute { .. } => StepKind::PreRoute, + Self::Cache { .. } => StepKind::Cache, + Self::Secrets { .. } => StepKind::Secrets, + Self::Build { kind, .. } => *kind, + Self::ModelInfo { .. } => StepKind::ModelInfo, + } + } +} + +/// Python's response to a [`HostCall`]. +#[derive(Debug)] +pub(crate) enum HostCallResponse { + /// The attempt's total tokens, or its failure. + Attempt(Result), + /// A successful attempt whose response the provider blocked, or a stream whose outcome the + /// host reports when it ends (`deferred`), with its tokens. + Served { + total_tokens: u64, + blocked: Option, + deferred: bool, + tracks_usage: Option, + }, + Slept, + AllowFallback(bool), + /// Whether the chain a `refuse_fallbacks` step named is refused. + RefuseFallbacks(bool), + /// The candidate ids that remain, pick first. + Select(Vec), + /// The model group to resolve instead, a custom strategy's deployment, or `None`. + PreRoute(Option), + /// The facts a `pre_call` select step asked for. + PreCall(PreCallFacts), + /// The candidate models a `routing_plugins` select step reported. + Models(Option>), + /// The ids a `callback_filters` select step left, or the error a filter raised. + Filtered(Result, AttemptError>), + /// What a pick's select or pre-route step, or a fallback check, raised on the host instead of + /// answering: the walk takes it as the try's failure. + Raised(AttemptError), + Cache(CacheOutcome), + /// One value per ref of a `secrets` step. + Secrets(Vec), + Validated(Built), + Provided(Provided), + Added(Built), + /// A `model_info` lookup's response, as the host spelled it. + ModelInfo(Value), +} + +impl HostCallResponse { + fn kind(&self) -> StepKind { + match self { + Self::Attempt(_) | Self::Served { .. } => StepKind::Attempt, + // Answers whichever step raised it (`check`). + Self::Raised(_) => StepKind::Select, + Self::Slept => StepKind::Sleep, + Self::AllowFallback(_) => StepKind::AllowFallback, + Self::RefuseFallbacks(_) => StepKind::RefuseFallbacks, + Self::Select(_) | Self::PreCall(_) | Self::Models(_) | Self::Filtered(_) => { + StepKind::Select + } + Self::PreRoute(_) => StepKind::PreRoute, + Self::Cache(_) => StepKind::Cache, + Self::Secrets(_) => StepKind::Secrets, + Self::Validated(_) => StepKind::Validate, + Self::Provided(_) => StepKind::Provider, + Self::Added(_) => StepKind::Added, + Self::ModelInfo(_) => StepKind::ModelInfo, + } + } + + fn check(&self, call: &HostCall) -> Result<(), DriveError> { + if matches!(self, Self::Raised(_)) + && matches!( + call.kind(), + StepKind::Select | StepKind::PreRoute | StepKind::AllowFallback + ) + { + return Ok(()); + } + if self.kind() != call.kind() { + return Err(DriveError::WrongAnswer { + expected: call.kind(), + got: self.kind(), + }); + } + if let (Self::Select(ids) | Self::Filtered(Ok(ids)), HostCall::Select { candidates, .. }) = + (self, call) + && let Some(id) = ids.iter().find(|id| !candidates.contains(id)) + { + return Err(DriveError::UnknownCandidate { id: id.clone() }); + } + if let (Self::Secrets(values), HostCall::Secrets { refs, .. }) = (self, call) + && values.len() != refs.len() + { + return Err(DriveError::SecretsLength { + expected: refs.len(), + got: values.len(), + }); + } + if let (Self::Cache(outcome), HostCall::Cache { op, keys, .. }) = (self, call) { + if !outcome.answers(*op) { + return Err(DriveError::WrongCacheOutcome { op: *op }); + } + if let (CacheOutcome::Values(values), Some(keys)) = (outcome, keys) + && values.len() != keys.len() + { + return Err(DriveError::BatchLength { + expected: keys.len(), + got: values.len(), + }); + } + } + Ok(()) + } +} + +/// How a driven future ended well. +#[derive(Debug)] +pub(crate) enum Finish { + /// `start`: the call's response. + Call(RouterResponse), + /// `select`: the deployment picked and the model group it serves. + Selected { + deployment_id: String, + model_group: String, + /// The deployment as served: a priority group's member carries its priority as `order`. + deployment: Value, + }, + /// `healthy`: the deployments a pick would choose among, as served. + Healthy { + /// A deployment named by its id, which the listing returns alone rather than as a list. + single: bool, + deployment_ids: Vec, + deployments: Vec, + }, + /// `report_failure`: the cooldown applied, if the deployment was cooled down. + Reported { cooldown: Option }, + /// `report_success` / `count_success`: the success counter's key, if one was counted. + Counted { key: Option }, + /// `build`: the deployments it installed, in order. + Built(Vec), + /// `router_model_info`: the merged model info (`None` when the lookup answered `None`). + RouterModelInfo(Option), + /// `deployment_model_info`: the merged model info, if any. + DeploymentModelInfo(Option>), + /// A model-info query that ends with one JSON answer (`model_group_info`, + /// `configured_token_limits`, `model_listing_info`). + Value(Value), +} + +/// One step of a call: what it is blocked on, or how it ended. +#[derive(Debug)] +pub(crate) enum Step { + HostCall(HostCall), + Done(Finish), + Failed(Error), +} + +/// A misuse of the step protocol by the caller. +#[derive(Debug, thiserror::Error)] +pub(crate) enum DriveError { + #[error("the router call already finished")] + Finished, + #[error("the router call has not made a host call yet, so the outcome must be None")] + UnexpectedResponse, + #[error("the router call is waiting for an outcome of its `{expected}` step")] + MissingResponse { expected: StepKind }, + #[error("the router call is waiting for an outcome of its `{expected}` step, not `{got}`")] + WrongAnswer { expected: StepKind, got: StepKind }, + #[error("deployment `{id}` was not one of the select step's candidates")] + UnknownCandidate { id: String }, + #[error("the outcome does not answer a `{op}` cache step")] + WrongCacheOutcome { op: CacheOp }, + #[error("a `batch_get` cache step for {expected} keys was answered with {got} values")] + BatchLength { expected: usize, got: usize }, + #[error("a `secrets` step for {expected} references was answered with {got} values")] + SecretsLength { expected: usize, got: usize }, + #[error("the router call is pending on something other than a step")] + Stalled, +} + +impl DriveError { + /// Whether the caller passed a bad outcome, rather than driving a call that cannot go on. + pub(crate) fn is_request(&self) -> bool { + match self { + Self::UnexpectedResponse + | Self::MissingResponse { .. } + | Self::WrongAnswer { .. } + | Self::UnknownCandidate { .. } + | Self::WrongCacheOutcome { .. } + | Self::BatchLength { .. } + | Self::SecretsLength { .. } => true, + Self::Finished | Self::Stalled => false, + } + } +} + +/// The host call a router call is blocked on and the response that releases it. +#[derive(Default)] +struct Exchange { + call: Option, + response: Option, + /// Events the call reported since the last step, handed to Python with the next one. + events: Vec, +} + +type Shared = Arc>; + +fn lock(exchange: &Mutex) -> MutexGuard<'_, Exchange> { + exchange.lock().unwrap_or_else(PoisonError::into_inner) +} + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// Binds an exchange to this thread for one poll and restores the previous one after. +struct Bound(Option); + +impl Bound { + fn new(exchange: &Shared) -> Self { + Self(CURRENT.replace(Some(exchange.clone()))) + } +} + +impl Drop for Bound { + fn drop(&mut self) { + CURRENT.set(self.0.take()); + } +} + +/// Park the host call `call` in the bound exchange and wait for its response. +/// `response`, or the failure a step answered with instead ([`HostCallResponse::Raised`]). +fn answered(response: HostCallResponse) -> Result { + match response { + HostCallResponse::Raised(failure) => Err(failure), + response => Ok(response), + } +} + +async fn call_host(call: HostCall) -> HostCallResponse { + let mut call = Some(call); + poll_fn(move |_| { + CURRENT.with_borrow(|current| { + let Some(exchange) = current else { + return Poll::Pending; + }; + let mut exchange = lock(exchange); + if let Some(call) = call.take() { + exchange.call = Some(call); + return Poll::Pending; + } + exchange.response.take().map_or(Poll::Pending, Poll::Ready) + }) + }) + .await +} + +const CHECKED: &str = "the driver checks each response against its host call"; + +fn ttl_seconds(ttl: Option) -> Option { + ttl.map(|ttl| ttl.as_secs_f64()) +} + +/// An increment amount as Python's counters store it: an int when it is whole. +fn amount_value(amount: f64) -> Value { + if amount.fract() == 0.0 && amount.abs() < i64::MAX as f64 { + Value::from(amount as i64) + } else { + Value::from(amount) + } +} + +/// The host's cache: every op is a `cache` step Python runs against the router's `DualCache`. +pub(crate) struct StepCache; + +async fn cache_step( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, +) -> CacheOutcome { + call_cache(op, key, keys, value, ttl, false).await +} + +async fn call_cache( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, + local: bool, +) -> CacheOutcome { + let call = HostCall::Cache { + op, + key: key.map(str::to_owned), + keys: keys.map(<[String]>::to_vec), + value, + ttl: ttl_seconds(ttl), + local, + }; + let HostCallResponse::Cache(outcome) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + outcome +} + +#[async_trait] +impl RouterCache for StepCache { + async fn async_get_cache(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + cache_step(CacheOp::Get, Some(key), None, None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Set, Some(key), None, Some(value), ttl).await; + Ok(()) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + cache_step(CacheOp::Increment, Some(key), None, value, ttl).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } + + async fn async_batch_get_cache( + &self, + keys: &[String], + ) -> Result>, litellm_cache::Error> { + let CacheOutcome::Values(values) = + cache_step(CacheOp::BatchGet, None, Some(keys), None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Delete, Some(key), None, None, None).await; + Ok(()) + } + + async fn async_increment_usage( + &self, + increments: &[(String, f64)], + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + let keys: Vec = increments.iter().map(|(key, _)| key.clone()).collect(); + let amounts = increments + .iter() + .map(|(_, amount)| amount_value(*amount)) + .collect(); + let value = Some(Value::Array(amounts)); + cache_step(CacheOp::IncrementUsage, None, Some(&keys), value, ttl).await; + Ok(()) + } + + async fn async_get_local(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + call_cache(CacheOp::Get, Some(key), None, None, None, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_increment_local( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + call_cache(CacheOp::Increment, Some(key), None, value, ttl, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } +} + +/// A deployment as Python spells a `model_list` entry, with its id in `model_info`. +pub(crate) fn deployment_record(deployment: &Deployment) -> Value { + deployment.record() +} + +/// The process's `random` module: a Python router draws its shuffle and retry jitter there, so +/// a seeded process picks the same deployments on either core. The bare module functions share +/// one generator, so these calls interleave with every other draw the process makes. +pub(crate) struct PythonRandomness; + +impl Randomness for PythonRandomness { + fn next_f64(&self) -> f64 { + Python::attach(|py| -> PyResult { + py.import("random")?.call_method0("random")?.extract() + }) + .unwrap_or_else(|_| ThreadRandomness.next_f64()) + } + + fn index_below(&self, len: usize) -> usize { + // `random.choice` is `seq[self._randbelow(len(seq))]`, and so is `randrange(len)`. + Python::attach(|py| -> PyResult { + py.import("random")? + .call_method1("randrange", (len,))? + .extract() + }) + .unwrap_or_else(|_| ThreadRandomness.index_below(len)) + } +} + +/// The [`Host`] every `RouterCoreHandle` router runs on. +pub(crate) struct StepHost; + +#[async_trait] +impl Host for StepHost { + async fn invoke(&self, attempt: &Attempt) -> Result { + let call = HostCall::Attempt { + deployment_id: attempt.deployment.id.clone(), + deployment: deployment_record(&attempt.deployment), + model: attempt.name.clone(), + model_group: attempt.model_group.clone(), + target_order: attempt.target_order, + candidate_ids: attempt.candidate_ids.clone(), + retry: attempt.retry, + fallback_depth: attempt.fallback_depth, + fallback_level: attempt.fallback_level, + timeout: attempt.timeout.map(|timeout| timeout.as_secs_f64()), + params: attempt + .params + .as_ref() + .map(|params| Box::new(Value::Object(Map::clone(params)))), + scope: attempt.scope, + max_retries: attempt.max_retries, + model_group_size: attempt.model_group_size, + }; + let (result, blocked, deferred, tracks_usage) = match call_host(call).await { + HostCallResponse::Attempt(result) => (result, None, false, None), + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } => (Ok(total_tokens), blocked, deferred, tracks_usage), + _ => unreachable!("{CHECKED}"), + }; + result.map(|total_tokens| Output { + value: Payload(Arc::new(())), + total_tokens, + blocked, + deferred, + tracks_usage, + }) + } + + async fn select( + &self, + stage: &SelectStage, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: match stage { + SelectStage::Plugins => Stage::Plugins, + SelectStage::Strategy(strategy) => Stage::Strategy(strategy.to_string()), + SelectStage::Tags => Stage::Tags, + SelectStage::TextOnly => Stage::TextOnly, + SelectStage::RouteIncapable { handler } => Stage::RouteIncapable { + handler: handler.clone(), + }, + SelectStage::MediaRefusing => Stage::MediaRefusing, + }, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::Select(ids) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(ids + .iter() + .filter_map(|id| candidates.iter().find(|d| d.id == *id).cloned()) + .collect()) + } + + async fn pre_route(&self, pre: &PreRoute<'_>) -> Result, Error> { + let call = HostCall::PreRoute { + model: pre.name.to_owned(), + fallback_depth: pre.fallback_depth, + retry: pre.retry, + }; + let HostCallResponse::PreRoute(routed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(routed) + } + + async fn routing_plugin_models( + &self, + ctx: &RoutingContext<'_>, + run: bool, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::RoutingPlugins { run }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models) + } + + async fn provider_request(&self, ctx: &RoutingContext<'_>) -> Result, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::ProviderRequest, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models.into_iter().flatten().next()) + } + + async fn pre_call_facts( + &self, + ctx: &RoutingContext<'_>, + candidates: &[Arc], + ) -> Result { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::PreCall, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::PreCall(facts) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(facts) + } + + async fn callback_filters( + &self, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let joined = |own: &[String], walked: &[String]| [own, walked].concat(); + let call = HostCall::Select { + stage: Stage::CallbackFilters { + excluded_ids: joined(&ctx.request.excluded_ids, ctx.excluded_ids), + skipped_ids: joined(&ctx.request.retry_skipped_ids, ctx.skipped_ids), + target_order: ctx.target_order, + }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::Filtered(result) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + let ids = result.map_err(Error::Attempt)?; + Ok(ids + .iter() + .filter_map(|id| candidates.iter().find(|d| d.id == *id).cloned()) + .collect()) + } + + async fn allow_fallback( + &self, + _request: &RouterRequest, + target: &str, + ) -> Result { + let call = HostCall::AllowFallback { + target: target.to_owned(), + }; + let HostCallResponse::AllowFallback(allowed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(allowed) + } + + async fn refuses_fallbacks(&self, level: u32, targets: &[Map]) -> bool { + let call = HostCall::RefuseFallbacks { + level, + targets: DictTargets(targets.to_vec()), + }; + let HostCallResponse::RefuseFallbacks(refused) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + refused + } + + /// A zero sleep is asked too: Python's retry loop awaits `asyncio.sleep(0)` between sibling + /// retries, which yields to the event loop. + async fn sleep(&self, duration: Duration) { + let call = HostCall::Sleep { + seconds: duration.as_secs_f64(), + }; + let HostCallResponse::Slept = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + } + + /// Kept for Python only when it acts on it: an exhausted retry loop is stamped on the + /// error Python raises, a fallback hop tells it which target a failed walk ended on, and a + /// cooldown or a pick is reported to the host's Prometheus and service loggers (a pick also + /// joins the router cache's per-deployment count, a failed walk the router's error log). + fn observe(&self, event: &RouterEvent) { + if !matches!( + event, + RouterEvent::RetriesExhausted { .. } + | RouterEvent::Retry { .. } + | RouterEvent::Fallback { .. } + | RouterEvent::FallingBack { .. } + | RouterEvent::OrderChainFailed { .. } + | RouterEvent::FallbacksFailed { .. } + | RouterEvent::LevelFailed { .. } + | RouterEvent::PickFailed { .. } + | RouterEvent::RetriesOpened { .. } + | RouterEvent::RetryBudget { .. } + | RouterEvent::MockFailed { .. } + | RouterEvent::RetryLogged { .. } + | RouterEvent::RetryPolicyRaised { .. } + | RouterEvent::RetryCountRaised { .. } + | RouterEvent::FallbackDone { .. } + | RouterEvent::Blocked { .. } + | RouterEvent::CooldownSet { .. } + | RouterEvent::Selected { .. } + ) { + return; + } + CURRENT.with_borrow(|current| { + if let Some(exchange) = current { + lock(exchange).events.push(event.clone()); + } + }); + } + + fn cache(&self) -> Arc { + Arc::new(StepCache) + } + + async fn secrets(&self, index: usize, refs: &[SecretRef]) -> Result, Error> { + let HostCallResponse::Secrets(values) = call_host(HostCall::Secrets { + index, + refs: refs.to_vec(), + }) + .await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn validate(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Validated(built) = + call_host(build_host_call(StepKind::Validate, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn provider(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Provided(provided) = + call_host(build_host_call(StepKind::Provider, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(provided) + } + + async fn added(&self, build: &BuildAsk<'_>, provider: Option<&str>) -> Result { + let HostCallResponse::Added(built) = + call_host(build_host_call(StepKind::Added, build, provider)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + let lookup = serde_json::to_value(lookup).unwrap_or(Value::Null); + let HostCallResponse::ModelInfo(response) = call_host(HostCall::ModelInfo { lookup }).await + else { + unreachable!("{CHECKED}"); + }; + Ok(response) + } +} + +fn build_host_call(kind: StepKind, build: &BuildAsk<'_>, provider: Option<&str>) -> HostCall { + HostCall::Build { + kind, + path: build.path, + index: build.index, + entry: Value::Object(build.entry.clone()), + declared_id: build.declared_id.map(str::to_owned), + duplicate_id: build.duplicate_id, + provider: provider.map(str::to_owned), + } +} + +type CallFuture = Pin> + Send>>; + +/// Poll `future` once with `exchange` bound. +fn poll_bound(future: Pin<&mut F>, exchange: &Shared) -> Poll { + let _bound = Bound::new(exchange); + future.poll(&mut Context::from_waker(Waker::noop())) +} + +/// Drives one router call a step at a time. +pub(crate) struct Driver { + future: Option, + exchange: Shared, + pending_call: Option, +} + +impl Driver { + pub(crate) fn new(call: impl Future> + Send + 'static) -> Self { + Self { + future: Some(Box::pin(call)), + exchange: Shared::default(), + pending_call: None, + } + } + + /// The host call the router call is blocked on, if it made one. + pub(crate) fn pending_call(&self) -> Option<&HostCall> { + self.pending_call.as_ref() + } + + /// The events reported since the last call. + pub(crate) fn take_events(&self) -> Vec { + std::mem::take(&mut lock(&self.exchange).events) + } + + /// Hand back the response to the pending host call (`None` on the first call) and run the call to + /// its next step or its end. + pub(crate) fn resume( + &mut self, + response: Option, + ) -> Result { + let Some(future) = self.future.as_mut() else { + return Err(DriveError::Finished); + }; + match (&self.pending_call, response) { + (None, None) => {} + (None, Some(_)) => return Err(DriveError::UnexpectedResponse), + (Some(call), None) => { + return Err(DriveError::MissingResponse { + expected: call.kind(), + }); + } + (Some(call), Some(response)) => { + response.check(call)?; + lock(&self.exchange).response = Some(response); + } + } + self.pending_call = None; + match poll_bound(future.as_mut(), &self.exchange) { + Poll::Ready(result) => { + self.future = None; + Ok(match result { + Ok(finish) => Step::Done(finish), + Err(error) => Step::Failed(error), + }) + } + Poll::Pending => { + let Some(call) = lock(&self.exchange).call.take() else { + self.future = None; + return Err(DriveError::Stalled); + }; + self.pending_call = Some(call.clone()); + Ok(Step::HostCall(call)) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, + }; + + use litellm_router_core::{ + FailureKind, Op, Router, RouterConfig, RouterDynamicConfig, SeededRandomness, + }; + use rstest::{fixture, rstest}; + use serde_json::{Value, json}; + + use super::*; + + fn router(config: Value, dynamic: Value) -> Router { + let config: RouterConfig = serde_json::from_value(config).unwrap(); + let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap(); + Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(SeededRandomness::new(7))) + .build() + .unwrap() + } + + fn deployment(group: &str, id: &str) -> Value { + json!({"model_name": group, "litellm_params": {"model": format!("openai/{id}")}, "model_info": {"id": id}}) + } + + fn start(router: &Router, model: &str) -> Driver { + let router = router.clone(); + let request = RouterRequest { + model: model.to_owned(), + ..RouterRequest::default() + }; + Driver::new(async move { + router + .call(Op::from("acompletion"), request) + .await + .map(Finish::Call) + }) + } + + fn rate_limited() -> HostCallResponse { + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::RateLimit, Some(429)))) + } + + /// A dict-backed stand-in for Python's `DualCache` that answers `cache` steps and keeps + /// every one it answered. + #[derive(Default)] + struct Store { + entries: HashMap, + calls: Vec, + } + + impl Store { + fn respond(&mut self, call: &HostCall) -> HostCallResponse { + let HostCall::Cache { + op, + key, + keys, + value, + .. + } = call + else { + panic!("not a cache step: {call:?}"); + }; + self.calls.push(call.clone()); + let key = key.clone().unwrap_or_default(); + HostCallResponse::Cache(match op { + CacheOp::Get => CacheOutcome::Value(self.entries.get(&key).cloned()), + CacheOp::Set => { + self.entries.insert(key, value.clone().unwrap()); + CacheOutcome::Stored + } + CacheOp::Increment => { + let current = self.entries.get(&key).and_then(Value::as_f64); + let next = + current.unwrap_or(0.0) + value.as_ref().and_then(Value::as_f64).unwrap(); + self.entries.insert(key, amount_value(next)); + CacheOutcome::Number(next) + } + CacheOp::IncrementUsage => { + let amounts = value.as_ref().and_then(Value::as_array).unwrap(); + for (key, amount) in keys.iter().flatten().zip(amounts) { + let current = self.entries.get(key).and_then(Value::as_f64); + let next = current.unwrap_or(0.0) + amount.as_f64().unwrap(); + self.entries.insert(key.clone(), amount_value(next)); + } + CacheOutcome::Stored + } + CacheOp::BatchGet => CacheOutcome::Values( + keys.iter() + .flatten() + .map(|key| self.entries.get(key).cloned()) + .collect(), + ), + CacheOp::Delete => { + self.entries.remove(&key); + CacheOutcome::Stored + } + }) + } + + /// Resume with `response`, answering every cache step on the way, up to the next other + /// step. + fn resume(&mut self, driver: &mut Driver, response: Option) -> Step { + let mut step = driver.resume(response).unwrap(); + while let Step::HostCall(call @ HostCall::Cache { .. }) = &step { + let response = self.respond(call); + step = driver.resume(Some(response)).unwrap(); + } + step + } + } + + #[fixture] + fn pair() -> Router { + router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({}), + ) + } + + #[rstest] + fn a_first_attempt_success_is_done(pair: Router) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let Step::HostCall(HostCall::Attempt { + deployment_id, + model_group, + retry: 0, + fallback_depth: 0, + .. + }) = store.resume(&mut driver, None) + else { + panic!("the call should start with an attempt"); + }; + assert_eq!(model_group, "primary"); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(12)))) + else { + panic!("a successful attempt should finish the call"); + }; + assert_eq!(response.deployment_id, deployment_id); + assert_eq!(response.output.total_tokens, 12); + assert_eq!( + (response.attempted_retries, response.attempted_fallbacks), + (0, 0) + ); + assert!(matches!(driver.resume(None), Err(DriveError::Finished))); + } + + #[rstest] + fn cooldown_reads_are_cache_steps(pair: Router) { + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall(HostCall::Cache { + op: CacheOp::BatchGet, + key: None, + keys: Some(keys), + value: None, + ttl: None, + local: false, + }) = step + else { + panic!("the call should read cooldowns first, got {step:?}"); + }; + assert_eq!(keys, ["deployment:a1:cooldown", "deployment:a2:cooldown"]); + } + + #[rstest] + #[case::lone_deployment_backs_off(vec![deployment("primary", "a1")], true)] + #[case::sibling_retries_at_once( + vec![deployment("primary", "a1"), deployment("primary", "a2")], + false + )] + fn a_rate_limit_retries_after_the_backoff( + #[case] model_list: Vec, + #[case] backs_off: bool, + ) { + let router = router(json!({"model_list": model_list}), json!({"num_retries": 1})); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let step = store.resume(&mut driver, Some(rate_limited())); + let Step::HostCall(HostCall::Sleep { seconds }) = step else { + panic!("a retry should sleep first, got {step:?}"); + }; + assert_eq!(seconds > 0.0, backs_off, "slept {seconds}s"); + let step = store.resume(&mut driver, Some(HostCallResponse::Slept)); + assert!( + matches!(step, Step::HostCall(HostCall::Attempt { retry: 1, .. })), + "expected the retry, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))) + else { + panic!("the retry should finish the call"); + }; + assert_eq!(response.attempted_retries, 1); + } + + #[rstest] + fn a_cooldown_python_wrote_is_read_back_by_the_next_call() { + let pair_without_retries = router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + store.entries.insert( + "deployment:a1:cooldown".into(), + json!({"exception_received": "slow down", "status_code": "429", "timestamp": now, "cooldown_time": 60}), + ); + for _ in 0..8 { + let mut call = start(&pair_without_retries, "primary"); + let Step::HostCall(HostCall::Attempt { deployment_id, .. }) = + store.resume(&mut call, None) + else { + panic!("the call should attempt the healthy deployment"); + }; + assert_eq!(deployment_id, "a2", "the cooled-down deployment was picked"); + } + } + + /// The host only runs the attempt; the core records its outcome (cooldowns, fail counters, + /// usage) through the host's cache. + #[rstest] + #[case::failure(rate_limited())] + #[case::success(HostCallResponse::Attempt(Ok(7)))] + fn attempt_outcomes_are_recorded_by_the_core(#[case] outcome: HostCallResponse) { + let tracked = |id: &str| { + json!({ + "model_name": "primary", + "litellm_params": {"model": format!("openai/{id}"), "tpm": 100, "cooldown_time": 7}, + "model_info": {"id": id, "allowed_fails": 3}, + }) + }; + let router = router( + json!({"model_list": [tracked("a1"), tracked("a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let step = store.resume(&mut driver, Some(outcome)); + assert!( + matches!(step, Step::Done(_) | Step::Failed(_)), + "got {step:?}" + ); + let writes: Vec<&HostCall> = store + .calls + .iter() + .filter(|call| { + matches!(call, HostCall::Cache { op, .. } if !matches!(op, CacheOp::Get | CacheOp::BatchGet)) + }) + .collect(); + assert!(!writes.is_empty(), "the core wrote no outcome"); + } + + #[rstest] + fn a_failed_group_falls_back_once_python_allows_it() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::AllowFallback { target }) if target == "backup"), + "expected the fallback check, got {step:?}" + ); + let step = store.resume(&mut driver, Some(HostCallResponse::AllowFallback(true))); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, fallback_depth: 1, .. }) if deployment_id == "b1"), + "expected the fallback attempt, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(3)))) + else { + panic!("the fallback should finish the call"); + }; + assert_eq!(response.model_group, "backup"); + assert_eq!(response.attempted_fallbacks, 1); + } + + #[rstest] + fn a_refused_fallback_fails_with_the_attempt_error() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + store.resume(&mut driver, Some(bad_request)); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::AllowFallback(false))) + else { + panic!("a refused fallback should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + #[rstest] + #[case::none_while_waiting(None, StepKind::Attempt)] + #[case::wrong_kind(Some(HostCallResponse::Slept), StepKind::Attempt)] + #[case::cache_outcome(Some(HostCallResponse::Cache(CacheOutcome::Stored)), StepKind::Attempt)] + fn a_mismatched_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: Option, + #[case] expected: StepKind, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + store.resume(&mut driver, None); + let error = driver.resume(response).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call().map(HostCall::kind), Some(expected)); + assert!(matches!( + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))), + Step::Done(_) + )); + } + + #[rstest] + #[case::wrong_op(HostCallResponse::Cache(CacheOutcome::Stored))] + #[case::number_for_batch(HostCallResponse::Cache(CacheOutcome::Number(1.0)))] + #[case::short_batch(HostCallResponse::Cache(CacheOutcome::Values(vec![None])))] + #[case::not_a_cache_outcome(HostCallResponse::Slept)] + fn a_mismatched_cache_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: HostCallResponse, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall( + call @ HostCall::Cache { + op: CacheOp::BatchGet, + .. + }, + ) = &step + else { + panic!("expected the cooldown read, got {step:?}"); + }; + let error = driver.resume(Some(response)).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call(), Some(call)); + let response = store.respond(call); + assert_eq!( + store.resume(&mut driver, Some(response)).unwrap_kind(), + StepKind::Attempt + ); + } + + #[rstest] + fn a_strategy_python_runs_asks_for_a_selection_and_keeps_the_pick() { + let router = router( + json!({ + "model_list": [deployment("primary", "a1"), deployment("primary", "a2")], + "routing_strategy": "least-busy", + }), + json!({}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + let step = store.resume(&mut driver, None); + let Step::HostCall(HostCall::Select { + stage: Stage::Strategy(strategy), + candidates, + .. + }) = step + else { + panic!("a host strategy should ask for a selection, got {step:?}"); + }; + assert_eq!(strategy, "least-busy"); + assert_eq!(candidates, ["a1", "a2"]); + let step = store.resume( + &mut driver, + Some(HostCallResponse::Select(vec!["a2".to_owned()])), + ); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, .. }) if deployment_id == "a2"), + "expected the picked deployment, got {step:?}" + ); + } + + #[rstest] + fn a_selection_is_driven_through_cache_steps(pair: Router) { + let request = RouterRequest { + model: "primary".to_owned(), + ..RouterRequest::default() + }; + let mut store = Store::default(); + let mut driver = Driver::new(async move { + pair.select(&Op::from("acompletion"), &request) + .await + .map(|deployment| Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: deployment_record(&deployment), + }) + }); + let Step::Done(Finish::Selected { + deployment_id, + model_group, + .. + }) = store.resume(&mut driver, None) + else { + panic!("a selection should finish with the pick"); + }; + assert!(["a1", "a2"].contains(&deployment_id.as_str())); + assert_eq!(model_group, "primary"); + assert!(matches!( + store.calls.as_slice(), + [HostCall::Cache { + op: CacheOp::BatchGet, + .. + }] + )); + } + + #[rstest] + fn dict_targets_keep_their_api_key_out_of_debug() { + let target = json!({"model": "b", "api_key": "credential-value"}); + let Value::Object(target) = target else { + unreachable!("a JSON object"); + }; + let shown = format!("{:?}", DictTargets(vec![target])); + assert!(!shown.contains("credential-value")); + assert!(shown.contains("[REDACTED]")); + } + + #[rstest] + fn a_chain_with_a_dict_target_asks_python_before_any_hop() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": [{"model": "backup", "api_key": "k"}]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::RefuseFallbacks { level: 0, targets }) if targets.0.len() == 1), + "expected the chain's check, got {step:?}" + ); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::RefuseFallbacks(true))) + else { + panic!("a refused chain should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + trait StepKindOf { + fn unwrap_kind(self) -> StepKind; + } + + impl StepKindOf for Step { + fn unwrap_kind(self) -> StepKind { + match self { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + } + } + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/mod.rs b/litellm-rust/crates/python-bridge/src/router/mod.rs new file mode 100644 index 00000000000..c69259fcf1c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/mod.rs @@ -0,0 +1,2051 @@ +//! `RouterCoreHandle`: router-core behind a step-driven handle. Python drives each call through +//! [`RouterCall::next`], running every attempt, sleep, fallback check, selection and cache op the +//! call loop asks for and handing the outcome back; Rust never awaits Python. `select` is driven +//! the same way, since reading cooldowns is a cache step. + +mod config; +mod host; +mod retries; +mod stream; + +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, SystemTime}, +}; + +use litellm_error::Report; +use litellm_host_python::{from_py_argument, json_loads, to_py}; +use litellm_router_core::lookup::{ListScope, Lookup, Row}; +use litellm_router_core::{ + AttemptError, Blocked, BuildPath, Built, Deployment, DeploymentSpec, Error, FailureKind, + FailureReport, Op, OutcomeCallbacks, PreRouted, Provided, RetryCause, Router, RouterEvent, + SecretAt, SuccessReport, + build::Install, + health::{UsageMetric, current_minute, usage_key}, + registry::{DeploymentFilter, ResolveKind, ResolveScope}, + strategy::routing_context, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::{PyBool, PyDict, PyList, PyString}, +}; +use serde_json::Value; + +use self::host::{ + CacheOp, CacheOutcome, DriveError, Driver, Finish, HostCall, HostCallResponse, + PythonRandomness, Stage, Step, StepHost, StepKind, +}; +use crate::errors::RustBridgeDeclined; + +/// The `error_kind` of a `failed` step. +#[derive(Clone, Copy, Debug, strum::IntoStaticStr)] +enum ErrorKind { + #[strum(serialize = "config")] + Config, + #[strum(serialize = "unknown_model")] + UnknownModel, + #[strum(serialize = "no_healthy_deployments")] + NoHealthyDeployments, + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "retired")] + Retired, + #[strum(serialize = "paused")] + Paused, + #[strum(serialize = "not_pass_through")] + NotPassThrough, + #[strum(serialize = "no_pass_through_deployments")] + NoPassThroughDeployments, + #[strum(serialize = "only_markers")] + OnlyMarkers, + #[strum(serialize = "endpoint_not_supported")] + EndpointNotSupported, + #[strum(serialize = "ptu_reserved")] + PtuReserved, + #[strum(serialize = "context_window_exceeded")] + ContextWindowExceeded, + #[strum(serialize = "unsupported_params")] + UnsupportedParams, + #[strum(serialize = "routing_plugins_left_nothing")] + RoutingPluginsLeftNothing, + #[strum(serialize = "routing_plugins_need_async")] + RoutingPluginsNeedAsync, + #[strum(serialize = "multiple_teams")] + MultipleTeams, + #[strum(serialize = "reserved")] + Reserved, + #[strum(serialize = "invalid_router_weights")] + InvalidRouterWeights, + #[strum(serialize = "non_numeric_weight")] + NonNumericWeight, + #[strum(serialize = "missing_routing_group")] + MissingRoutingGroup, + #[strum(serialize = "default_deployment_raises")] + DefaultDeploymentRaises, + #[strum(serialize = "index_out_of_range")] + IndexOutOfRange, + #[strum(serialize = "model_info")] + ModelInfo, +} + +impl From<&Error> for ErrorKind { + fn from(error: &Error) -> Self { + match error { + Error::Config(_) => Self::Config, + Error::UnknownModel { .. } => Self::UnknownModel, + Error::NoHealthyDeployments { .. } => Self::NoHealthyDeployments, + Error::Attempt(_) => Self::Attempt, + Error::Cache(_) => Self::Cache, + Error::Retired => Self::Retired, + Error::Paused { .. } => Self::Paused, + Error::NotPassThrough { .. } => Self::NotPassThrough, + Error::NoPassThroughDeployments { .. } => Self::NoPassThroughDeployments, + Error::OnlyMarkers { .. } => Self::OnlyMarkers, + Error::EndpointNotSupported { .. } => Self::EndpointNotSupported, + Error::PtuReserved { .. } => Self::PtuReserved, + Error::ContextWindowExceeded { .. } => Self::ContextWindowExceeded, + Error::UnsupportedParams { .. } => Self::UnsupportedParams, + Error::RoutingPluginsLeftNothing { .. } => Self::RoutingPluginsLeftNothing, + Error::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync, + Error::MultipleTeams { .. } => Self::MultipleTeams, + Error::Reserved { .. } => Self::Reserved, + Error::InvalidRouterWeights => Self::InvalidRouterWeights, + Error::NonNumericWeight { .. } => Self::NonNumericWeight, + Error::MissingRoutingGroup { .. } => Self::MissingRoutingGroup, + Error::DefaultDeploymentRaises { .. } => Self::DefaultDeploymentRaises, + Error::IndexOutOfRange { .. } => Self::IndexOutOfRange, + Error::ModelInfo { .. } => Self::ModelInfo, + } + } +} + +fn config_error_to_pyerr(error: config::Error) -> PyErr { + let message = Report::new(&error).to_string(); + if error.declined() { + RustBridgeDeclined::new_err(message) + } else { + PyValueError::new_err(message) + } +} + +fn drive_error_to_pyerr(error: DriveError) -> PyErr { + let message = Report::new(&error).to_string(); + if error.is_request() { + PyValueError::new_err(message) + } else { + PyRuntimeError::new_err(message) + } +} + +/// A document Python passes as a mapping or as its JSON text. +fn document(value: &Bound<'_, PyAny>) -> PyResult { + match value.cast::() { + Ok(text) => { + let parsed = json_loads(value.py(), text.to_str()?.as_bytes())?; + from_py_argument(parsed.bind(value.py())) + } + Err(_) => from_py_argument(value), + } +} + +/// The `OutcomeCallbacks` a reported outcome came through; none is a report no callback makes. +fn outcome_callbacks(value: Option<&Bound<'_, PyAny>>) -> PyResult { + let Some(value) = value else { + return Ok(OutcomeCallbacks::default()); + }; + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// A router-core router whose host operations Python runs. +#[pyclass(frozen)] +pub(crate) struct RouterCoreHandle { + router: Router, +} + +#[pymethods] +impl RouterCoreHandle { + #[new] + fn new(config: &Bound<'_, PyAny>, dynamic: &Bound<'_, PyAny>) -> PyResult { + let (config, dynamic) = (document(config)?, document(dynamic)?); + let build = || { + let config = config::parse_router_config(config)?; + let dynamic = config::parse_dynamic_config(dynamic)?; + Ok(Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(PythonRandomness)) + .build()?) + }; + build() + .map(|router| Self { router }) + .map_err(config_error_to_pyerr) + } + + /// Start a call; nothing runs until the first `next(None)`. + fn start(&self, op: &str, request: &Bound<'_, PyAny>) -> PyResult { + let op = Op::new(op); + let request = + config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router.call(op, request).await.map(Finish::Call) + })) + } + + /// Pick the deployment `model` routes to now, without calling it. Driven like `start`: its + /// steps are the cache reads and selections the pick needs, and it ends with `done` + /// (`deployment_id`, `model_group`) or `failed`. + fn select(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let deployment = router.select(&Op::from("acompletion"), &request).await?; + Ok(Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: host::deployment_record(&deployment), + }) + })) + } + + /// List the deployments a pick for `model` would choose among, driven like `select`. + fn healthy(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let candidates = router.healthy(&Op::from("acompletion"), &request).await?; + // A deployment named by its id is listed alone, as a dict (`_core_async_get_healthy_deployments`). + let single = candidates.kind.is_single() && candidates.healthy.len() == 1; + let deployments = candidates.healthy; + Ok(Finish::Healthy { + single, + deployment_ids: deployments.iter().map(|d| d.id.clone()).collect(), + deployments: deployments + .iter() + .map(|d| host::deployment_record(d)) + .collect(), + }) + })) + } + + /// Apply `{"upsert": [deployments], "delete": [ids], "settings": {dynamic document}}` and + /// return the new registry version. + fn apply(&self, changes: &Bound<'_, PyAny>) -> PyResult { + let changes = config::parse_changes(document(changes)?).map_err(config_error_to_pyerr)?; + self.router + .apply(changes) + .map_err(|error| config_error_to_pyerr(error.into())) + } + + /// Build `entries` (model_list entries) through the host, driven like `start`: its steps + /// are `secrets`, `validate`, `provider` and `added`, and it ends with `done` (`built`: one + /// `{"index", "id", "provider"}` per deployment installed). `path` is `create` + /// (`set_model_list`) or `add` (`add_deployment`); `install` is `replace`, `append` or + /// `upsert`. + #[pyo3(signature = (entries, path, resolve_secrets, install))] + fn build( + &self, + entries: &Bound<'_, PyAny>, + path: &str, + resolve_secrets: bool, + install: &str, + ) -> PyResult { + let entries: Vec> = + serde_json::from_value(document(entries)?) + .map_err(|error| PyValueError::new_err(format!("bad model_list: {error}")))?; + let path: BuildPath = path + .parse() + .map_err(|_| PyValueError::new_err(format!("`{path}` is not a build path")))?; + let install: Install = install + .parse() + .map_err(|_| PyValueError::new_err(format!("`{install}` is not a build install")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .build(path, entries, resolve_secrets, install) + .await + .map(Finish::Built) + })) + } + + /// `get_router_model_info` for the deployment `query` describes (a + /// `RouterModelInfoQuery` document), driven like `start`: its steps are `model_info` + /// lookups, and it ends with `done` (`info`: the merged map or None, `user_keys`: the keys + /// whose values are the deployment's own). + fn router_model_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::model_info::RouterModelInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad model-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .router_model_info(&query) + .await + .map(Finish::RouterModelInfo) + })) + } + + /// `get_deployment_model_info` for deployment `model_id` serving `model_name`, driven like + /// `router_model_info`; it ends with `done` (`info`: the merged map or None). + fn deployment_model_info( + &self, + model_id: &str, + model_name: &Bound<'_, PyAny>, + ) -> PyResult { + let model_id = model_id.to_owned(); + let model_name = from_py_argument::(model_name)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .deployment_model_info(&model_id, &model_name) + .await + .map(Finish::DeploymentModelInfo) + })) + } + + /// `_set_model_group_info` for the deployments `query` names (a `GroupInfoQuery` document), + /// driven like `router_model_info`; it ends with `done` (`value`: None when no deployment + /// matched, else `{"assignments": {field: value}}`, what the merge set on the group the host + /// built from its first deployment). + fn model_group_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::group_info::GroupInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad group-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let info = router.model_group_info(&query).await?; + Ok(Finish::Value(info.map_or( + Value::Null, + |info| serde_json::json!({"assignments": info.assignments}), + ))) + })) + } + + /// `get_configured_token_limits` for the deployment with `id` and its `configured` model_info, + /// driven like `router_model_info`; it ends with `done` (`value`: `[max_input, max_output]`). + fn configured_token_limits( + &self, + id: &Bound<'_, PyAny>, + configured: &Bound<'_, PyAny>, + ) -> PyResult { + let id = from_py_argument::(id)?; + let Value::Object(configured) = document(configured)? else { + return Err(PyValueError::new_err( + "the configured model_info must be a dict", + )); + }; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let (input, output) = router.configured_token_limits(id, &configured).await?; + Ok(Finish::Value(serde_json::json!([input, output]))) + })) + } + + /// `get_model_listing_info` over the `deployments` a listed name names, driven like + /// `router_model_info`; it ends with `done` (`value`: `{"cost_map_keys", "max_input_tokens", + /// "max_output_tokens"}`). + fn model_listing_info(&self, deployments: &Bound<'_, PyAny>) -> PyResult { + let deployments: Vec = from_py_argument(deployments)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let listing = router.model_listing_info(&deployments).await?; + Ok(Finish::Value(serde_json::json!({ + "cost_map_keys": listing.cost_map_keys, + "max_input_tokens": listing.max_input_tokens, + "max_output_tokens": listing.max_output_tokens, + }))) + })) + } + + /// `get_configured_mode` / `get_configured_display_name`: `value` when it is a string that is + /// not blank. + #[staticmethod] + fn configured_text(value: &Bound<'_, PyAny>) -> PyResult> { + let value = from_py_argument::(value).ok(); + Ok(litellm_router_core::group_info::configured_text( + value.as_ref(), + )) + } + + /// `get_configured_service_tiers`: each of `model_infos`' `service_tiers`, unvalidated. + #[staticmethod] + fn configured_service_tiers( + py: Python<'_>, + model_infos: &Bound<'_, PyAny>, + ) -> PyResult>> { + let model_infos: Vec> = from_py_argument(model_infos)?; + litellm_router_core::group_info::configured_service_tiers(&model_infos) + .iter() + .map(|tiers| to_py(py, tiers)) + .collect() + } + + /// The routing-cache key deployment `id` serving litellm `model` counts `metric` (`tpm`, + /// `rpm`, `itpm` or `otpm`) under in `minute` (`HH-MM`). + fn usage_key(&self, id: &str, model: &str, metric: &str, minute: &str) -> PyResult { + let metric: UsageMetric = metric + .parse() + .map_err(|_| PyValueError::new_err(format!("`{metric}` is not a usage metric")))?; + Ok(usage_key(id, model, metric, minute)) + } + + /// Keep the providers `other`'s build resolved for the deployments this router serves too. + fn adopt_providers(&self, other: &Self) { + self.router.adopt_providers(&other.router); + } + + fn registry_version(&self) -> u64 { + self.router.registry().version() + } + + /// The version the deployments before the newest appends last changed at: a list read at + /// or after it is current once it reads `deployments_from(len(list))`. + fn registry_rewritten(&self) -> u64 { + self.router.registry().rewritten() + } + + /// Record a failure the router did not see itself (a health check, an interceptor, a + /// logging callback), driven like `start`: its steps are cache ops, and it ends with `done` + /// (`cooled`, `cooldown_time`). `counts_request` also counts the failed request against the deployment's + /// per-minute requests, as an async call's failure callback does. `callbacks` is the logged + /// request's `OutcomeCallbacks`. + #[pyo3(signature = (id, failure, deployment_cooldown=false, count_only=false, requested_group=None, cooldown_time=None, uncounted=false, counts_request=false, usage_model=None, callbacks=None))] + #[allow(clippy::too_many_arguments)] + fn report_failure( + &self, + id: &str, + failure: &Bound<'_, PyDict>, + deployment_cooldown: bool, + count_only: bool, + requested_group: Option, + cooldown_time: Option, + uncounted: bool, + counts_request: bool, + usage_model: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let failure = match attempt_outcome(failure)? { + Err(failure) => failure, + Ok(_) => { + return Err(PyValueError::new_err( + "a reported failure needs `ok: False`", + )); + } + }; + let router = self.router.clone(); + let id = id.to_owned(); + let report = FailureReport { + deployment_cooldown, + cooldown_time, + count_only, + requested_group, + uncounted, + counts_request, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_failure(&id, &failure, report) + .await + .map(|cooldown| Finish::Reported { cooldown }) + })) + } + + /// Record a success a logging callback saw, driven like `report_failure`; it ends with + /// `done` (`key`: the success counter's key, or None for an unknown deployment). + /// `usage_model` is the model its usage is keyed by, and `tracks_usage` the host's answer to + /// whether it counts any (see `Output::tracks_usage`). `callbacks`: see `report_failure`. + #[pyo3(signature = (id, total_tokens, requests, usage_model=None, tracks_usage=None, callbacks=None))] + fn report_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + usage_model: Option, + tracks_usage: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let report = SuccessReport { + tracks_usage, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_success(&id, total_tokens, requests, report) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// Count a success against `id`, configured or not; ends with `done` (`key`). `callbacks`: + /// see `report_failure`. + #[pyo3(signature = (id, callbacks=None))] + fn count_success( + &self, + id: &str, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let callbacks = outcome_callbacks(callbacks)?; + Ok(RouterCall::new(async move { + router + .count_success(&id, callbacks) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// The usage keys of this minute for every deployment of `model_group`: + /// `[(id, tpm_key, rpm_key)]`, or None when the group has no deployment. + fn usage_keys(&self, model_group: &str) -> Option> { + let minute = current_minute(SystemTime::now()); + let group = self.router.registry().group(model_group)?; + Some( + group + .iter() + .map(|deployment| { + let model = &deployment.params.model; + ( + deployment.id.clone(), + usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), + usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), + ) + }) + .collect(), + ) + } + + /// Every configured deployment, as `model_list` entries with their ids, in config order. + fn deployments<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments(&DeploymentFilter::default()) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployments from position `start` on, as `deployments()` lists them. + fn deployments_from<'py>(&self, py: Python<'py>, start: usize) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments_from(start) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployment with this id, or None. + fn deployment(&self, py: Python<'_>, id: &str) -> PyResult>> { + self.router + .registry() + .deployment(id) + .map(|deployment| to_py(py, &host::deployment_record(&deployment))) + .transpose() + } + + // -- registry lookups (router-core `lookup.rs`): rows the host reads its stored entries at -- + + /// `_core_get_all_deployments`, as rows. + #[pyo3(signature = (model_name, team_id=None, rename=None))] + fn lookup_all<'py>( + &self, + py: Python<'py>, + model_name: &str, + team_id: Option<&str>, + rename: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).all(model_name, team_id, rename)) + } + + /// `_team_deployments_across_teams`: the rows, and how many teams own them. + fn lookup_across_teams<'py>( + &self, + py: Python<'py>, + model: &str, + ) -> PyResult<(Bound<'py, PyAny>, usize)> { + let registry = self.router.registry(); + let (rows, teams) = Lookup::new(®istry).across_teams(model); + Ok((rows_to_py(py, &rows)?, teams)) + } + + /// `_core_config_deployments`, as rows. + fn lookup_config<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).config()) + } + + /// `_core_get_model_ids`: the ids, or the row the read raises at. + #[pyo3(signature = (model_name=None, exclude_team_models=false))] + fn lookup_model_ids<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + exclude_team_models: bool, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).model_ids(model_name, exclude_team_models) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_list`, as rows. + #[pyo3(signature = (model_name, team_id, scope))] + fn lookup_model_list<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).model_list(model_name, team_id, &scope), + ) + } + + /// `_core_get_model_names`, or the row the read raises at. + fn lookup_model_names<'py>( + &self, + py: Python<'py>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(Vec, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).model_names(team_id, &scope) { + Ok(names) => Ok((names, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_access_groups`: `(group, names)` pairs in the order the groups appear, + /// or the row the read raises at. + fn lookup_access_groups<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + access_group: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(AccessGroups, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).access_groups(model_name, access_group, team_id, &scope) { + Ok(groups) => Ok((groups, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the whole list lists any row, and the name of its first row whose litellm model + /// is `model_id` (or ends in `/model_id` or `:model_id`); or the row the read raises at. + fn lookup_name_serving<'py>( + &self, + py: Python<'py>, + model_id: &str, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(bool, Option, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).name_serving(model_id, &scope) { + Ok((listed, name)) => Ok((listed, name, None)), + Err(raw) => Ok((false, None, Some(row_to_py(py, &raw)?))), + } + } + + /// Whether `model_names` (the group names) holds `name`. + fn has_model_name(&self, name: &str) -> bool { + self.router + .registry() + .host_indexes() + .model_names + .contains(&Value::String(name.to_owned())) + } + + /// `_core_team_model_has_alternatives`, or the row the read raises at. + fn lookup_team_model_has_alternatives<'py>( + &self, + py: Python<'py>, + id: &str, + ) -> PyResult<(bool, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_model_has_alternatives(id) { + Ok(found) => Ok((found, None)), + Err(raw) => Ok((false, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the group's first deployment the team may use, or the row the read + /// raises at. + fn lookup_group_usable_by_team<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).group_deployment_usable_by_team(model_group, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the team public name's first deployment, or the row the read raises at. + fn lookup_team_public_name<'py>( + &self, + py: Python<'py>, + model_id: &str, + team_id: &str, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_public_name_deployment(model_id, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// `_routable_deployments`, as rows. + fn lookup_routable<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).routable(model_group, team_id)) + } + + /// `_core_wildcard_route_deployments`: the team router's routes, else the global router's. + fn lookup_wildcard_route<'py>( + &self, + py: Python<'py>, + model: &str, + team_id: Option<&str>, + requests: Vec, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).wildcard_route(model, team_id, &requests), + ) + } + + /// `_core_get_candidate_model_ids_for_route` for the resolved name: the members' ids when it + /// names a callable routing group (`members`), else for a name that is no model group the + /// team's or the wildcard route's, else the group's; or the row the read raises at. + fn lookup_candidate_ids<'py>( + &self, + py: Python<'py>, + resolved: &str, + team_id: Option<&str>, + members: Option>, + requests: Vec, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + let lookup = Lookup::new(®istry); + match lookup.candidate_ids(resolved, team_id, members.as_deref(), &requests) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the id map holds `id` (`_has_model_id`). + fn has_model_id(&self, id: &str) -> bool { + self.router.registry().host_indexes().has_id(id) + } + + /// What the id map holds for `id`, as written (a position, or what a caller wrote), or None. + fn id_position<'py>(&self, py: Python<'py>, id: &str) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .id_entry(id) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// The id map's entries, in order: `(id, what it holds)`. + fn id_positions<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let pairs: Vec = registry + .host_indexes() + .id_map + .iter() + .map(|(id, held)| Value::Array(vec![Value::String(id.clone()), held.clone()])) + .collect(); + to_py(py, &Value::Array(pairs)).map(|value| value.into_bound(py)) + } + + /// What the name index holds for `name`, as written, or None when it does not index it. + fn named_positions<'py>( + &self, + py: Python<'py>, + name: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .named(name) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// What the team index holds for `(team, public)`, as written, or None. + fn team_positions<'py>( + &self, + py: Python<'py>, + team: &str, + public: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .team_named(team, public) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// `team_public_model_names_of`: the public names the team index holds for `team`. + fn team_public_names_of(&self, team: &str) -> Vec { + self.router + .registry() + .host_indexes() + .team_public_names_of(team) + } + + /// The positions of the deployments whose `litellm_params.model` is `model`. + fn litellm_model_positions(&self, model: &str) -> Vec { + self.router + .registry() + .deployments(&DeploymentFilter::default()) + .iter() + .enumerate() + .filter(|(_, deployment)| { + deployment.raw_params.get("model").and_then(Value::as_str) == Some(model) + }) + .map(|(position, _)| position) + .collect() + } + + /// `PatternMatchRouter.route` over the wildcard router (or `team`'s): the first pattern's + /// deployments, each with the model the wildcards matched, or None. + #[pyo3(signature = (team, request, filtered=None))] + fn pattern_route<'py>( + &self, + py: Python<'py>, + team: Option<&str>, + request: &str, + filtered: Option>, + ) -> PyResult>> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router + .and_then(|router| router.route(request, filtered.as_deref())) + .map(|routed| to_py(py, &Value::Array(routed)).map(|value| value.into_bound(py))) + .transpose() + } + + /// `get_models_by_pattern`: the models the first matching (request, pattern) pair routes to. + fn models_by_pattern(&self, team: Option<&str>, requests: Vec) -> Option> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router.and_then(|router| router.models_by_pattern(&requests)) + } + + /// The indexes the host hands out (`HostIndexes`): id map, name and team indexes, group and + /// deployment names, team public names, provider default ids and the pattern routers. + fn host_indexes<'py>(&self, py: Python<'py>) -> PyResult> { + let indexes = serde_json::to_value(self.router.registry().host_indexes()) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + to_py(py, &indexes).map(|value| value.into_bound(py)) + } + + /// `provider_default_deployment_ids` as the indexes hold it now: the ids of the wildcard + /// deployments the cooldown check skips. + fn provider_default_ids(&self) -> Vec { + self.router + .registry() + .host_indexes() + .provider_default_ids + .clone() + } + + /// The provider the host resolved when it built the deployment with this id (its + /// `provider` build step), or None when there is no such deployment or it never went + /// through a host build. + fn provider(&self, id: &str) -> Option { + self.router + .registry() + .deployment(id) + .and_then(|deployment| deployment.resolved_provider.clone()) + } + + /// What `name` resolves to: `{"model_group", "kind", "ids"}`, or None for an unknown name. + #[pyo3(signature = (name, specific_deployment=false))] + fn resolve<'py>( + &self, + py: Python<'py>, + name: &str, + specific_deployment: bool, + ) -> PyResult>> { + let registry = self.router.registry(); + let Some(resolution) = registry.resolve( + name, + ResolveScope { + specific_deployment, + }, + ) else { + return Ok(None); + }; + let dict = PyDict::new(py); + dict.set_item("model_group", resolution.group)?; + dict.set_item("kind", format!("{:?}", resolution.kind))?; + dict.set_item( + "ids", + PyList::new(py, resolution.deployments.iter().map(|d| d.id.clone()))?, + )?; + Ok(Some(dict)) + } + + /// The deployments a pick for `model` would resolve to for `request`, before any team, + /// cooldown or strategy filter (`_core_get_routable_deployments`); empty when it refuses. + fn routable<'py>( + &self, + py: Python<'py>, + model: &str, + request: &Bound<'py, PyAny>, + ) -> PyResult> { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let records = litellm_router_core::resolve::routable( + &self.router.registry(), + &self.router.settings().dynamic, + &request, + ) + .iter() + .map(|d| to_py(py, &host::deployment_record(d))) + .collect::>>()?; + PyList::new(py, records) + } + + /// `_get_routing_context` for a pick of `model`: `(strategy, scope, group)`, spelled as a + /// strategy select step spells them. A name that resolves to nothing reads its routing group + /// membership as is. + #[pyo3(signature = (model, routing_strategy=None))] + fn routing_context( + &self, + model: &str, + routing_strategy: Option<&str>, + ) -> (String, &'static str, Option) { + let registry = self.router.registry(); + let settings = self.router.settings(); + let (kind, group) = registry + .resolve(model, ResolveScope::default()) + .map_or((ResolveKind::Group, model.to_owned()), |resolution| { + (resolution.kind, resolution.group) + }); + let (strategy, scope) = routing_context( + ®istry, + &settings.routing_strategy, + routing_strategy, + &kind, + &group, + ); + let (scope, group) = scope.wire(); + (strategy.to_string(), scope, group.map(str::to_owned)) + } + + fn retire(&self) { + self.router.retire(); + } + + /// What may still take over a stream from `origin` that fails before content, and the + /// retry budget for a failure of `kind` (see `stream::policy`). + #[pyo3(signature = (request, origin, kind=None))] + fn stream_policy<'py>( + &self, + py: Python<'py>, + request: &Bound<'py, PyAny>, + origin: &Bound<'py, PyAny>, + kind: Option<&str>, + ) -> PyResult> { + stream::policy(py, &self.router, request, origin, kind) + } + + /// Retry a stream from `origin` that failed before content within its group, driven like + /// `start`: `done` when a retry returned a response, `failed` with the newest failure. + fn retry_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Retry, + op, + request, + origin, + failure, + ) + } + + /// The fallback walk for a stream from `origin` that failed before content, driven like + /// `start`. + fn fallback_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Fallback, + op, + request, + origin, + failure, + ) + } + + /// `get_num_retries_from_retry_policy` for a failure shaped like an attempt outcome. + #[pyo3(signature = (failure, model_group=None))] + fn retry_policy_retries( + &self, + failure: &Bound<'_, PyDict>, + model_group: Option<&str>, + ) -> PyResult> { + retries::policy_retries(&self.router, failure, model_group) + } + + /// `_core_get_allowed_fails_from_policy` for a failure shaped like an attempt outcome. + fn allowed_fails_for(&self, failure: &Bound<'_, PyDict>) -> PyResult> { + retries::allowed_fails(&self.router, failure) + } + + /// `should_retry_this_error` as a predicate, for a failure shaped like an attempt outcome. + fn should_retry( + &self, + failure: &Bound<'_, PyDict>, + counts: (usize, usize), + fallbacks: (bool, bool, bool), + ) -> PyResult { + retries::should_retry(failure, counts, fallbacks) + } +} + +/// One router call, driven a step at a time by Python. +#[pyclass(frozen)] +pub(crate) struct RouterCall { + driver: Mutex, +} + +impl RouterCall { + fn new( + future: impl std::future::Future> + Send + 'static, + ) -> Self { + Self { + driver: Mutex::new(Driver::new(future)), + } + } +} + +#[pymethods] +impl RouterCall { + /// Hand back the outcome of the last step (`None` on the first call) and return the next + /// step, or the call's end. + #[pyo3(signature = (outcome))] + fn next<'py>( + &self, + py: Python<'py>, + outcome: Option>, + ) -> PyResult> { + let mut driver = self.driver.lock().unwrap_or_else(PoisonError::into_inner); + let response = match driver.pending_call() { + None => match outcome { + None => None, + Some(_) => return Err(drive_error_to_pyerr(DriveError::UnexpectedResponse)), + }, + Some(call) => Some(host_call_response(call, outcome)?), + }; + let step = driver.resume(response).map_err(drive_error_to_pyerr)?; + let dict = step_dict(py, step)?; + dict.set_item("events", events_list(py, driver.take_events())?)?; + Ok(dict) + } +} + +/// `(group, names)` pairs, in the order the groups appear. +type AccessGroups = Vec<(String, Vec)>; + +/// Lookup rows as the host reads them. +fn rows_to_py<'py>(py: Python<'py>, rows: &[Row]) -> PyResult> { + let rows = Value::Array(rows.iter().map(Row::wire).collect()); + to_py(py, &rows).map(|value| value.into_bound(py)) +} + +fn row_to_py<'py>(py: Python<'py>, row: &Row) -> PyResult> { + to_py(py, &row.wire()).map(|value| value.into_bound(py)) +} + +/// `{"aliases": [{alias, target}], "groups": [{name, members}], "wildcard": [requests]}`. +fn list_scope(scope: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(scope)?) + .map_err(|error| PyValueError::new_err(format!("bad lookup scope: {error}"))) +} + +/// Read Python's outcome for `call`. +fn host_call_response( + call: &HostCall, + outcome: Option>, +) -> PyResult { + let missing = || { + drive_error_to_pyerr(DriveError::MissingResponse { + expected: call.kind(), + }) + }; + // `{"raised": True, ...}`: the step's work raised, reported as an attempt's failure. + if let HostCall::Select { .. } | HostCall::PreRoute { .. } | HostCall::AllowFallback { .. } = + call + && let Some(failure) = outcome.as_ref().and_then(|o| o.cast::().ok()) + && failure.contains("raised")? + { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Raised(error)), + Ok(_) => Err(PyValueError::new_err("a `raised` outcome is a failure")), + }; + } + match call { + HostCall::Sleep { .. } => Ok(HostCallResponse::Slept), + HostCall::AllowFallback { .. } => Ok(HostCallResponse::AllowFallback( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::RefuseFallbacks { .. } => Ok(HostCallResponse::RefuseFallbacks( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::Select { + stage: Stage::PreCall, + .. + } => { + let facts = from_py_argument(&outcome.ok_or_else(missing)?)?; + serde_json::from_value(facts) + .map(HostCallResponse::PreCall) + .map_err(|error| PyValueError::new_err(format!("bad pre-call facts: {error}"))) + } + HostCall::Select { + stage: Stage::RoutingPlugins { .. } | Stage::ProviderRequest, + .. + } => { + // `{"models": [...] | None}`: a bare None would read as no outcome. + let outcome = outcome.ok_or_else(missing)?; + let models = outcome.cast::()?.get_item("models")?; + Ok(HostCallResponse::Models(match models { + Some(models) if !models.is_none() => Some(models.extract()?), + _ => None, + })) + } + HostCall::Select { + stage: Stage::CallbackFilters { .. }, + .. + } => { + // The remaining ids, or a failure mapping as an attempt reports one. + let outcome = outcome.ok_or_else(missing)?; + if let Ok(failure) = outcome.cast::() { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Filtered(Err(error))), + Ok(_) => Err(PyValueError::new_err( + "a `callback_filters` outcome is the remaining ids or a failure", + )), + }; + } + Ok(HostCallResponse::Filtered(Ok(outcome.extract()?))) + } + HostCall::Select { .. } => Ok(HostCallResponse::Select( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::PreRoute { .. } => pre_route_outcome(outcome).map(HostCallResponse::PreRoute), + HostCall::Attempt { .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let blocked = match outcome.get_item("blocked")? { + Some(value) if !value.is_none() => { + let name: String = value.extract()?; + Some(name.parse::().map_err(|_| { + PyValueError::new_err(format!( + "`{name}` is not a way a response is blocked" + )) + })?) + } + Some(_) | None => None, + }; + let deferred = outcome + .get_item("deferred")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + let tracks_usage = outcome + .get_item("tracks_usage")? + .map(|flag| flag.extract::>()) + .transpose()? + .flatten(); + attempt_outcome(outcome).map(|result| match result { + Ok(total_tokens) if blocked.is_some() || deferred || tracks_usage.is_some() => { + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } + } + result => HostCallResponse::Attempt(result), + }) + } + HostCall::Cache { op, .. } => cache_outcome(*op, outcome).map(HostCallResponse::Cache), + HostCall::Secrets { .. } => Ok(HostCallResponse::Secrets(from_py_argument( + &outcome.ok_or_else(missing)?, + )?)), + // `None` is a response here (no row, no entry), so a missing outcome reads as one. + HostCall::ModelInfo { .. } => Ok(HostCallResponse::ModelInfo(match outcome { + Some(outcome) if !outcome.is_none() => from_py_argument(&outcome)?, + _ => Value::Null, + })), + HostCall::Build { kind, .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let invalid = outcome + .get_item("invalid")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + match kind { + StepKind::Provider => Ok(HostCallResponse::Provided(if invalid { + Provided::Invalid + } else { + Provided::Provider(match outcome.get_item("provider")? { + Some(provider) if !provider.is_none() => Some(provider.extract()?), + _ => None, + }) + })), + _ => { + let built = if invalid { + Built::Invalid + } else { + match outcome.get_item("entry")? { + Some(entry) if !entry.is_none() => { + Built::Entry(from_py_argument(&entry)?) + } + _ => Built::Skip, + } + }; + Ok(if *kind == StepKind::Validate { + HostCallResponse::Validated(built) + } else { + HostCallResponse::Added(built) + }) + } + } + } + } +} + +/// A pre-route response: `None`, a model group name, or a custom strategy's deployment document. +fn pre_route_outcome(outcome: Option>) -> PyResult> { + let Some(outcome) = outcome.filter(|outcome| !outcome.is_none()) else { + return Ok(None); + }; + if let Ok(model) = outcome.cast::() { + return Ok(Some(PreRouted::Model(model.to_str()?.to_owned()))); + } + let spec: DeploymentSpec = serde_json::from_value(document(&outcome)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let deployment = Deployment::from_spec(spec) + .map_err(|error| PyValueError::new_err(Report::new(&error).to_string()))?; + Ok(Some(PreRouted::Deployment(Arc::new(deployment)))) +} + +/// `get`: the JSON value or None; `batch_get`: a list of JSON values or None, one per key; +/// `increment`: the new int or float; `set` and `delete`: None. +fn cache_outcome(op: CacheOp, outcome: Option>) -> PyResult { + let outcome = outcome.filter(|outcome| !outcome.is_none()); + let json = |value: Option| value.filter(|value| !value.is_null()); + match op { + CacheOp::Get => outcome + .map(|outcome| from_py_argument::(&outcome)) + .transpose() + .map(|value| CacheOutcome::Value(json(value))), + CacheOp::BatchGet => { + let outcome = outcome.ok_or_else(|| { + PyValueError::new_err("a `batch_get` cache outcome must be a list, not None") + })?; + if !outcome.is_instance_of::() { + return Err(PyValueError::new_err( + "a `batch_get` cache outcome must be a list with one entry per key", + )); + } + let values: Vec> = from_py_argument(&outcome)?; + Ok(CacheOutcome::Values(values.into_iter().map(json).collect())) + } + CacheOp::Increment => outcome + .filter(|outcome| !outcome.is_instance_of::()) + .and_then(|outcome| outcome.extract::().ok()) + .map(CacheOutcome::Number) + .ok_or_else(|| { + PyValueError::new_err("an `increment` cache outcome must be the new int or float") + }), + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage => match outcome { + None => Ok(CacheOutcome::Stored), + Some(_) => Err(PyValueError::new_err(format!( + "a `{op}` cache step has no outcome, so pass None" + ))), + }, + } +} + +/// `{"ok": True, "total_tokens": int}` (an attempt's may add `"blocked": str`, `"deferred": bool` +/// and `"tracks_usage": bool`) or `{"ok": False, "kind": str, "status": int | None, +/// "retry_after": float | None, "num_retries": int | None, "detail": str | None, +/// "outside_deployment": bool, "caller_scoped": bool, "status_absent": bool, +/// "uncounted_request": bool}`. +pub(crate) fn attempt_outcome(outcome: &Bound<'_, PyDict>) -> PyResult> { + fn item<'py, T: FromPyObjectOwned<'py>>( + outcome: &Bound<'py, PyDict>, + key: &str, + ) -> PyResult> { + match outcome.get_item(key)? { + Some(value) if !value.is_none() => Ok(Some(value.extract().map_err(Into::into)?)), + Some(_) | None => Ok(None), + } + } + let ok: bool = item(outcome, "ok")? + .ok_or_else(|| PyValueError::new_err("an attempt outcome needs `ok`"))?; + if ok { + return Ok(Ok(item(outcome, "total_tokens")?.unwrap_or(0))); + } + let kind: String = item(outcome, "kind")? + .ok_or_else(|| PyValueError::new_err("a failed attempt outcome needs `kind`"))?; + let kind: FailureKind = kind + .parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind")))?; + let retry_after = item::(outcome, "retry_after")? + .map(|seconds| { + Duration::try_from_secs_f64(seconds) + .map_err(|_| PyValueError::new_err("`retry_after` must be a non-negative number")) + }) + .transpose()?; + let mut failure = AttemptError::new(kind, item(outcome, "status")?); + failure.retry_after = retry_after; + failure.num_retries = item(outcome, "num_retries")?; + failure.failed_deployment_id = item(outcome, "failed_deployment_id")?; + failure.detail = item::(outcome, "detail")?.map(Into::into); + failure.recorded = item::(outcome, "recorded")?.unwrap_or(false); + failure.names_provider = item::(outcome, "llm_provider")?.unwrap_or(false); + failure.agentic_followup = item::(outcome, "agentic_followup")?.unwrap_or(false); + failure.outside_deployment = item::(outcome, "outside_deployment")?.unwrap_or(false); + failure.status_absent = item::(outcome, "status_absent")?.unwrap_or(false); + failure.uncounted_request = item::(outcome, "uncounted_request")?.unwrap_or(false); + failure.caller_scoped = item::(outcome, "caller_scoped")?.unwrap_or(false); + failure.classes = item::>(outcome, "classes")? + .unwrap_or_default() + .into_iter() + .map(Into::into) + .collect(); + Ok(Err(failure)) +} + +/// `[{"kind": "retries_exhausted", "model_group": str, "attempted": int, "max": int} +/// | {"kind": "fallback", "to": str, "order": int | None, "level": int}]`. +/// The `cause` of a `retry_logged` or `retry_policy_raised` event, with what names it. +fn cause_items(dict: &Bound<'_, PyDict>, cause: RetryCause) -> PyResult<()> { + match cause { + RetryCause::Attempt => dict.set_item("cause", "attempt"), + RetryCause::Level => dict.set_item("cause", "level"), + RetryCause::Other => dict.set_item("cause", "other"), + RetryCause::Pick(picked) => { + dict.set_item("cause", "pick")?; + pick_items(dict, &picked.0) + } + } +} + +/// A pick's error, as the `failed` step names it (`error_kind` and what names it), with the +/// model its message names (`failed_model`) when the error carries one. +fn pick_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + error_items(dict, error)?; + match error { + Error::NoHealthyDeployments { model } | Error::UnknownModel { model } => { + dict.set_item("failed_model", model) + } + _ => Ok(()), + } +} + +/// `error_kind` and the fields that name the error. +fn error_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + dict.set_item("error", Report::new(error).to_string())?; + dict.set_item("error_kind", <&'static str>::from(ErrorKind::from(error)))?; + match error { + Error::EndpointNotSupported { model, message } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", message) + } + Error::OnlyMarkers { model } + | Error::UnknownModel { model } + | Error::NoPassThroughDeployments { model } + | Error::PtuReserved { model } + | Error::MultipleTeams { model } + | Error::DefaultDeploymentRaises { model } + | Error::IndexOutOfRange { model } => dict.set_item("error_model", model), + Error::ContextWindowExceeded { model, detail } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", detail) + } + Error::UnsupportedParams { model, params } => { + dict.set_item("error_model", model)?; + dict.set_item("error_params", PyList::new(dict.py(), params)?) + } + Error::RoutingPluginsLeftNothing { models } => { + dict.set_item("error_params", PyList::new(dict.py(), models)?) + } + Error::Reserved { + model, + until, + timezone, + } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", format!("{until} {timezone}")) + } + Error::NonNumericWeight { metric } => dict.set_item("error_detail", metric), + Error::MissingRoutingGroup { group } => dict.set_item("error_model", group), + Error::ModelInfo { exception, detail } => { + dict.set_item("error_exception", *exception)?; + dict.set_item("error_detail", detail) + } + _ => Ok(()), + } +} + +fn events_list(py: Python<'_>, events: Vec) -> PyResult> { + let list = PyList::empty(py); + for event in events { + let dict = PyDict::new(py); + match event { + RouterEvent::RetriesExhausted { + model_group, + attempted, + max, + level, + picked, + } => { + dict.set_item("kind", "retries_exhausted")?; + dict.set_item("model_group", model_group)?; + dict.set_item("attempted", attempted)?; + dict.set_item("max", max)?; + dict.set_item("level", level)?; + dict.set_item("picked", picked)?; + } + RouterEvent::Retry { retry, .. } => { + dict.set_item("kind", "retry")?; + dict.set_item("retry", retry)?; + } + RouterEvent::FallingBack { level, model } => { + dict.set_item("kind", "falling_back")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + } + RouterEvent::OrderChainFailed { level, last_hop } => { + dict.set_item("kind", "order_chain_failed")?; + dict.set_item("level", level)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::FallbacksFailed { + level, + chain, + last_hop, + } => { + dict.set_item("kind", "fallbacks_failed")?; + dict.set_item("level", level)?; + dict.set_item("chain", chain)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::Fallback { + from, + to, + order, + depth, + level, + scope, + parent, + max_fallbacks, + unroutable, + } => { + dict.set_item("kind", "fallback")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("order", order)?; + dict.set_item("depth", depth)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("max_fallbacks", max_fallbacks)?; + if let Some(model) = unroutable { + dict.set_item("unroutable", to_py(py, &model)?)?; + } + } + RouterEvent::Blocked { + level, + model, + blocked, + } => { + dict.set_item("kind", "blocked")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("blocked", <&'static str>::from(blocked))?; + } + RouterEvent::FallbackDone { + from, + to, + level, + scope, + parent, + succeeded, + } => { + dict.set_item("kind", "fallback_done")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("succeeded", succeeded)?; + } + RouterEvent::RetryLogged { + scope, + level, + model, + attempted_retries, + cause, + } => { + dict.set_item("kind", "retry_logged")?; + dict.set_item("scope", scope)?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("attempted_retries", attempted_retries)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryPolicyRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_policy_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryCountRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_count_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetriesOpened { + scope, + max_retries, + model_group_size, + } => { + dict.set_item("kind", "retries_opened")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + } + RouterEvent::RetryBudget { scope, max_retries } => { + dict.set_item("kind", "retry_budget")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + } + RouterEvent::PickFailed { + level, + model, + error, + } => { + dict.set_item("kind", "pick_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::LevelFailed { + level, + model, + error, + handling, + } => { + dict.set_item("kind", "level_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("handling", handling)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::MockFailed { + mock, + level, + model, + num_retries, + } => { + dict.set_item("kind", "mock_failed")?; + dict.set_item("mock", <&'static str>::from(mock))?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("num_retries", num_retries)?; + } + RouterEvent::CooldownSet { + deployment_id, + cooldown, + } => { + dict.set_item("kind", "cooldown_set")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("cooldown_time", cooldown.as_secs_f64())?; + } + RouterEvent::Selected { + deployment_id, + pick, + .. + } => { + dict.set_item("kind", "selected")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("pick_seconds", pick.as_secs_f64())?; + } + _ => continue, + } + list.append(dict)?; + } + Ok(list) +} + +fn step_dict(py: Python<'_>, step: Step) -> PyResult> { + let dict = PyDict::new(py); + let kind = match &step { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + }; + dict.set_item("kind", <&'static str>::from(kind))?; + match step { + Step::HostCall(call) => host_call_items(py, &dict, call)?, + Step::Done(finish) => done_items(&dict, finish)?, + Step::Failed(error) => { + let attempt = error.attempt(); + error_items(&dict, &error)?; + dict.set_item("attempt_failed", attempt.is_some())?; + dict.set_item("status", attempt.and_then(|failure| failure.status))?; + dict.set_item( + "failure_kind", + attempt.map(|failure| <&'static str>::from(failure.kind)), + )?; + } + } + Ok(dict) +} + +fn host_call_items(py: Python<'_>, dict: &Bound<'_, PyDict>, call: HostCall) -> PyResult<()> { + match call { + HostCall::Attempt { + deployment_id, + deployment, + model, + model_group, + target_order, + candidate_ids, + retry, + fallback_depth, + fallback_level, + timeout, + params, + scope, + max_retries, + model_group_size, + } => { + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("deployment", to_py(py, &deployment)?)?; + dict.set_item("model", model)?; + dict.set_item("model_group", model_group)?; + dict.set_item("target_order", target_order)?; + dict.set_item("candidate_ids", PyList::new(py, candidate_ids)?)?; + dict.set_item("retry", retry)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("fallback_level", fallback_level)?; + dict.set_item("timeout", timeout)?; + dict.set_item("params", params.map(|p| to_py(py, &p)).transpose()?) + } + HostCall::Sleep { seconds } => dict.set_item("seconds", seconds), + HostCall::AllowFallback { target } => dict.set_item("target", target), + HostCall::RefuseFallbacks { level, targets } => { + dict.set_item("level", level)?; + let targets = targets.0.iter().cloned().map(Value::Object).collect(); + dict.set_item("targets", to_py(py, &Value::Array(targets))?) + } + HostCall::Select { + stage, + model_group, + name, + scope, + group, + pool, + candidates, + } => { + let (stage, strategy) = match stage { + Stage::Plugins => ("plugins", None), + Stage::Strategy(strategy) => ("strategy", Some(strategy)), + Stage::Tags => ("tags", None), + Stage::PreCall => ("pre_call", None), + Stage::TextOnly => ("text_only", None), + Stage::RouteIncapable { handler } => { + dict.set_item("handler", handler)?; + ("route_incapable", None) + } + Stage::MediaRefusing => ("media_refusing", None), + Stage::ProviderRequest => ("provider_request", None), + Stage::CallbackFilters { + excluded_ids, + skipped_ids, + target_order, + } => { + dict.set_item("excluded_ids", excluded_ids)?; + dict.set_item("skipped_ids", skipped_ids)?; + dict.set_item("target_order", target_order)?; + ("callback_filters", None) + } + Stage::RoutingPlugins { run } => { + dict.set_item("run", run)?; + ("routing_plugins", None) + } + }; + dict.set_item("pool", PyList::new(py, pool)?)?; + dict.set_item("stage", stage)?; + dict.set_item("strategy", strategy)?; + dict.set_item("model_group", model_group)?; + dict.set_item("model", name)?; + dict.set_item("scope", scope)?; + dict.set_item("group", group)?; + dict.set_item("candidates", PyList::new(py, candidates)?) + } + HostCall::PreRoute { + model, + fallback_depth, + retry, + } => { + dict.set_item("model", model)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("retry", retry) + } + HostCall::Cache { + op, + key, + keys, + value, + ttl, + local, + } => { + dict.set_item("op", <&'static str>::from(op))?; + dict.set_item("key", key)?; + dict.set_item("keys", keys.map(|keys| PyList::new(py, keys)).transpose()?)?; + dict.set_item("value", value.map(|value| to_py(py, &value)).transpose()?)?; + dict.set_item("ttl", ttl)?; + dict.set_item("local", local) + } + HostCall::Secrets { index, refs } => { + dict.set_item("index", index)?; + let refs = refs + .into_iter() + .map(|secret| { + let item = PyDict::new(py); + item.set_item("name", secret.name)?; + match secret.at { + SecretAt::Param(key) => { + item.set_item("key", key)?; + item.set_item("env_only", false)?; + } + SecretAt::DataSource { source, key } => { + item.set_item("source", source)?; + item.set_item("key", key)?; + item.set_item("env_only", true)?; + } + } + Ok(item) + }) + .collect::>>()?; + dict.set_item("refs", PyList::new(py, refs)?) + } + HostCall::Build { + kind: _, + path, + index, + entry, + declared_id, + duplicate_id, + provider, + } => { + dict.set_item("path", <&'static str>::from(path))?; + dict.set_item("index", index)?; + dict.set_item("entry", to_py(py, &entry)?)?; + dict.set_item("declared_id", declared_id)?; + dict.set_item("duplicate_id", duplicate_id)?; + dict.set_item("provider", provider) + } + HostCall::ModelInfo { lookup } => { + if let Value::Object(lookup) = lookup { + for (key, value) in lookup { + dict.set_item(key, to_py(py, &value)?)?; + } + } + Ok(()) + } + } +} + +fn done_items(dict: &Bound<'_, PyDict>, finish: Finish) -> PyResult<()> { + match finish { + Finish::Call(response) => { + dict.set_item("deployment_id", response.deployment_id)?; + dict.set_item("model_group", response.model_group)?; + dict.set_item("attempted_retries", response.attempted_retries)?; + dict.set_item("max_retries", response.max_retries)?; + dict.set_item("attempted_fallbacks", response.attempted_fallbacks)?; + dict.set_item("total_tokens", response.output.total_tokens) + } + Finish::Selected { + deployment_id, + model_group, + deployment, + } => { + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("model_group", model_group)?; + dict.set_item("deployment", to_py(dict.py(), &deployment)?) + } + Finish::Reported { cooldown } => { + dict.set_item("cooled", cooldown.is_some())?; + dict.set_item("cooldown_time", cooldown.map(|c| c.as_secs_f64())) + } + Finish::Built(built) => { + let py = dict.py(); + let records = built + .into_iter() + .map(|deployment| { + let item = PyDict::new(py); + item.set_item("index", deployment.index)?; + item.set_item( + "id", + deployment.spec.model_info.get("id").and_then(Value::as_str), + )?; + item.set_item("provider", deployment.spec.provider)?; + Ok(item) + }) + .collect::>>()?; + dict.set_item("built", PyList::new(py, records)?) + } + Finish::Counted { key } => dict.set_item("key", key), + Finish::RouterModelInfo(merged) => { + let py = dict.py(); + match merged { + Some(merged) => { + dict.set_item("info", to_py(py, &Value::Object(merged.info))?)?; + dict.set_item("user_keys", merged.user_keys) + } + None => dict.set_item("info", py.None()), + } + } + Finish::DeploymentModelInfo(info) => { + let py = dict.py(); + match info { + Some(info) => dict.set_item("info", to_py(py, &Value::Object(info))?), + None => dict.set_item("info", py.None()), + } + } + Finish::Value(value) => dict.set_item("value", to_py(dict.py(), &value)?), + Finish::Healthy { + single, + deployment_ids, + deployments, + } => { + dict.set_item("single", single)?; + dict.set_item("deployment_ids", deployment_ids)?; + let records = deployments + .iter() + .map(|d| to_py(dict.py(), d)) + .collect::>>()?; + dict.set_item("deployments", PyList::new(dict.py(), records)?) + } + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ffi::c_str, prelude::*, types::PyDict}; + use rstest::rstest; + + const CONFIG: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}} + ]}"#; + + const PAIR: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}} + ]}"#; + + #[rstest] + fn python_drives_calls_and_selections_through_the_handle() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("native", crate::native_module(py)).unwrap(); + locals.set_item("config", CONFIG).unwrap(); + locals.set_item("pair", PAIR).unwrap(); + py.run( + c_str!( + r#" +import time +store = {} +asked = [] + +def answer_cache(step): + asked.append(step) + op, key = step["op"], step["key"] + if op == "get": + return store.get(key) + if op == "batch_get": + return [store.get(k) for k in step["keys"]] + if op == "set": + store[key] = step["value"] + return None + if op == "increment": + store[key] = store.get(key, 0) + step["value"] + return store[key] + if op == "delete": + store.pop(key, None) + return None + raise AssertionError(op) + +def drive(call, outcome): + step = call.next(outcome) + while step["kind"] == "cache": + step = call.next(answer_cache(step)) + return step + +handle = native.RouterCoreHandle(config, {"num_retries": 1}) +picked = drive(handle.select("primary", {}), None) +assert (picked["kind"], picked["deployment_id"], picked["model_group"]) == ("done", "a1", "primary"), picked +assert [step["op"] for step in asked] == ["batch_get"] +assert list(asked[0]) == ["kind", "op", "key", "keys", "value", "ttl", "local", "events"], list(asked[0]) +call = handle.start("acompletion", '{"model": "primary"}') +version = handle.registry_version() +assert handle.apply({"delete": ["gone"]}) == version + 1 +steps = [drive(call, None)] +steps.append(drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})) +steps.append(drive(call, None)) +steps.append(drive(call, {"ok": True, "total_tokens": 5})) +kinds = [step["kind"] for step in steps] + +cooled = native.RouterCoreHandle(pair, {"num_retries": 0}) +call = cooled.start("acompletion", {"model": "primary"}) +attempt = drive(call, None) +tried = attempt["deployment_id"] +assert attempt["deployment"]["model_info"]["id"] == tried, attempt +assert attempt["deployment"]["litellm_params"]["model"].startswith("openai/"), attempt +assert [d["model_info"]["id"] for d in cooled.deployments()] == ["a1", "a2"] +assert cooled.deployment("a2")["model_name"] == "primary" +assert cooled.resolve("primary")["ids"] == ["a1", "a2"] +assert cooled.resolve("nope") is None +assert drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})["kind"] == "failed" +# The core records the failure itself: the 429 cools the deployment down in the host's store. +assert f"deployment:{tried}:cooldown" in store, store +other = drive(cooled.select("primary", {}), None) +assert other["kind"] == "done" and other["deployment_id"] != tried, other + +call = handle.select("primary", {}) +step = call.next(None) +assert step["op"] == "batch_get" +for bad in ([], [None, None], "nope", None): + try: + call.next(bad) + except ValueError: + pass + else: + raise AssertionError(f"{bad!r} should not answer a batch_get of one key") +assert drive(call, [None])["kind"] == "done" +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let kinds: Vec = locals + .get_item("kinds") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(kinds, ["attempt", "sleep", "attempt", "done"]); + }); + } + + #[rstest] + #[case::unknown_key(r#"{"routing_strategy_args": {}}"#, true)] + #[case::bad_json("{", false)] + #[case::not_an_object("[]", false)] + fn construction_failures_raise_the_class_python_reads( + #[case] config: &str, + #[case] declined: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let error = crate::native_module(py) + .getattr("RouterCoreHandle") + .and_then(|handle| handle.call1((config, "{}"))) + .unwrap_err(); + assert_eq!( + error.is_instance_of::(py), + declined, + "{error}" + ); + assert_eq!( + error.is_instance_of::(py), + !declined, + "{error}" + ); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/retries.rs b/litellm-rust/crates/python-bridge/src/router/retries.rs new file mode 100644 index 00000000000..949aae96b34 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/retries.rs @@ -0,0 +1,55 @@ +//! The retry decisions `RouterCoreHandle` exposes as members: `get_num_retries_from_retry_policy` +//! and `should_retry_this_error`, decided by router-core for a failure Python describes the way it +//! describes a failed attempt. + +use litellm_router_core::{ + AttemptError, Router, + retry::{RetryContext, resolve_retry_policy, should_retry as core_should_retry}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +fn failure(outcome: &Bound<'_, PyDict>) -> PyResult { + match super::attempt_outcome(outcome)? { + Err(failure) => Ok(failure), + Ok(_) => Err(PyValueError::new_err("a failure needs `ok: False`")), + } +} + +pub(crate) fn policy_retries( + router: &Router, + outcome: &Bound<'_, PyDict>, + model_group: Option<&str>, +) -> PyResult> { + let failure = failure(outcome)?; + let settings = router.settings(); + let policy = match model_group { + Some(group) => resolve_retry_policy( + settings.retry_policy.as_ref(), + group, + Some(&settings.model_group_retry_policy), + ), + None => settings.retry_policy.as_ref(), + }; + Ok(policy.and_then(|policy| policy.retries_for(&failure))) +} + +pub(crate) fn allowed_fails(router: &Router, outcome: &Bound<'_, PyDict>) -> PyResult> { + Ok(router.settings().allowed_fails_for(&failure(outcome)?)) +} + +pub(crate) fn should_retry( + outcome: &Bound<'_, PyDict>, + (healthy, all): (usize, usize), + (context_window, content_policy, regular): (bool, bool, bool), +) -> PyResult { + Ok(core_should_retry( + &failure(outcome)?, + RetryContext { + healthy, + all, + has_context_window_fallbacks: context_window, + has_content_policy_fallbacks: content_policy, + has_regular_fallbacks: regular, + }, + )) +} diff --git a/litellm-rust/crates/python-bridge/src/router/stream.rs b/litellm-rust/crates/python-bridge/src/router/stream.rs new file mode 100644 index 00000000000..f85d73e0778 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/stream.rs @@ -0,0 +1,94 @@ +//! The stream-recovery entry points of `RouterCoreHandle`: what may still take over a stream that +//! failed before content, the same-group stream retry, and the fallback walk re-entered from a +//! stream. The host's stream wrappers call them; the calls are driven like `start`. + +use litellm_router_core::{ + FailureKind, Op, Router, + call::{StreamOrigin, StreamPolicy}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +use super::{RouterCall, attempt_outcome, config, config_error_to_pyerr, document, host::Finish}; + +/// A stream recovery a host wrapper asks for. +#[derive(Clone, Copy)] +pub(super) enum Recovery { + /// `_astream_retry_same_group`. + Retry, + /// The fallback walk re-entered from the stream wrapper. + Fallback, +} + +fn origin(value: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(format!("a stream origin: {error}"))) +} + +fn kind(kind: Option<&str>) -> PyResult> { + kind.map(|kind| { + kind.parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind"))) + }) + .transpose() +} + +/// `{"can_retry", "can_fall_back", "refusal_fallback", "retry_budget": int | None, +/// "policy_applies": bool}`. +pub(super) fn policy<'py>( + py: Python<'py>, + router: &Router, + request: &Bound<'py, PyAny>, + stream_origin: &Bound<'py, PyAny>, + failure_kind: Option<&str>, +) -> PyResult> { + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let StreamPolicy { + can_retry, + can_fall_back, + refusal_fallback, + retry_budget, + } = router.stream_policy(&request, &origin(stream_origin)?, kind(failure_kind)?); + let dict = PyDict::new(py); + dict.set_item("can_retry", can_retry)?; + dict.set_item("can_fall_back", can_fall_back)?; + dict.set_item("refusal_fallback", refusal_fallback)?; + dict.set_item("retry_budget", retry_budget.map(|(budget, _)| budget))?; + dict.set_item( + "policy_applies", + retry_budget.is_some_and(|(_, applies)| applies), + )?; + Ok(dict) +} + +/// A recovery call for a stream from `stream_origin` that failed with `failure`. +pub(super) fn recover( + router: &Router, + recovery: Recovery, + op: &str, + request: &Bound<'_, PyAny>, + stream_origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, +) -> PyResult { + let op = Op::new(op); + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let stream_origin = origin(stream_origin)?; + let Err(failure) = attempt_outcome(failure)? else { + return Err(PyValueError::new_err("a stream failure needs `ok: False`")); + }; + let router = router.clone(); + Ok(RouterCall::new(async move { + let response = match recovery { + Recovery::Retry => { + router + .retry_stream(op, request, stream_origin, failure) + .await + } + Recovery::Fallback => { + router + .fallback_stream(op, request, stream_origin, failure) + .await + } + }; + response.map(Finish::Call) + })) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs index d78d141f0f2..1bcdc18a765 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use host::ChatCompletionsPythonHost; use litellm_auth::AuthServices; -use litellm_cache_response::{CachePolicy, ScopedCache}; +use litellm_cache_response::CachePolicy; use litellm_callbacks_legacy_python::LoggingOperation; use litellm_host::{call::HostedMachine, protocol::Protocol}; use litellm_inference_chat::{ChatCompletionsRoute, route::ChatCompletions}; @@ -57,10 +57,6 @@ impl InferenceRoute for ChatCompletionsRoute { Self::new(http, auth, secrets) } - fn with_cache(self, cache: ScopedCache) -> Self { - self.with_cache(cache) - } - fn machine( self, call: ::Request, diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index ca30c94b02f..1245c126c4e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use litellm_auth::AuthServices; -use litellm_cache_response::{CachePolicy, CacheScope, ScopedCache}; +use litellm_cache_response::CachePolicy; use litellm_callbacks_legacy_python::LoggingOperation; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_host::{call::HostedMachine, protocol::Protocol}; @@ -142,7 +142,6 @@ pub(super) trait InferenceRoute: Sized + 'static { auth: Arc, secrets: Arc, ) -> Self; - fn with_cache(self, cache: ScopedCache) -> Self; fn machine( self, call: ::Request, @@ -176,12 +175,8 @@ where crate::http::resources().auth.clone(), crate::secrets::source(py)?, ); - let (cache, cache_options) = crate::cache::configured_native(py, arguments, call_type)?; - let route = match cache { - Some(cache) => route.with_cache(ScopedCache::new(cache, CacheScope::Shared)), - None => route, - }; - Ok(route.machine(request, cache_options.policy)) + crate::cache::ensure_cache_disabled(py, arguments, call_type)?; + Ok(route.machine(request, CachePolicy::default())) }, host, hooks, diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 27ce8c943f0..af614f17360 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -34,11 +34,9 @@ fn merge_headers( forwarded: Option>, extra_headers: Option>, ) -> Option> { - let merged: Map = forwarded - .into_iter() - .flatten() - .chain(extra_headers.into_iter().flatten()) - .collect(); + let merged = litellm_inference_messages::merge_messages_header_sources( + forwarded.into_iter().chain(extra_headers), + ); (!merged.is_empty()).then_some(merged) } @@ -107,10 +105,19 @@ impl MessagesPythonHost { .map(|value| python_timeout_seconds(py, value.unbind())) .transpose()? .flatten(); - let custom_llm_provider = string("custom_llm_provider")?; - let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; let api_key = string("api_key")?; let api_base = string("api_base")?; + // Python's handler resolves a bare model name (`claude-sonnet-4-5`) through + // `get_llm_provider`'s model lists, which the native prefix lookup does not carry. + let custom_llm_provider = match string("custom_llm_provider")? { + Some(provider) => Some(provider), + None => py + .import(ROUTE_HOST_MODULE)? + .getattr("inferred_provider")? + .call1((&model, api_base.as_deref()))? + .extract()?, + }; + let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index c788760b268..a9ae01bd7ee 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR ) } +/// Python's sync Messages handler raises for every native provider before it sends anything +/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies +/// that until both sides serve sync calls. +const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls"; + #[pyfunction] -pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { - run_messages(py, call, false) +#[pyo3(signature = (call))] +pub(crate) fn messages(call: NativeCall<'_>) -> PyResult> { + drop(call); + Err(pyo3::exceptions::PyValueError::new_err( + SYNC_NOT_IMPLEMENTED, + )) } #[pyfunction] diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 9c045d65c13..cc4d85b4d26 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use host::ResponsesPythonHost; use litellm_auth::AuthServices; -use litellm_cache_response::{CachePolicy, ScopedCache}; +use litellm_cache_response::CachePolicy; use litellm_callbacks_legacy_python::LoggingOperation; use litellm_host::{call::HostedMachine, protocol::Protocol}; use litellm_inference_responses::{ResponsesRoute, route::Responses}; @@ -49,10 +49,6 @@ impl InferenceRoute for ResponsesRoute { Self::new(http, auth, secrets) } - fn with_cache(self, cache: ScopedCache) -> Self { - self.with_cache(cache) - } - fn machine( self, call: ::Request, diff --git a/litellm-rust/crates/router-core/Cargo.toml b/litellm-rust/crates/router-core/Cargo.toml index af97bd52c57..ff5e2e8161f 100644 --- a/litellm-rust/crates/router-core/Cargo.toml +++ b/litellm-rust/crates/router-core/Cargo.toml @@ -7,3 +7,21 @@ repository.workspace = true description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop" [dependencies] +arc-swap.workspace = true +async-trait.workspace = true +litellm-cache.workspace = true +litellm-router-types.workspace = true +jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-zoneinfo", "tzdb-bundle-always"] } +parking_lot.workspace = true +rand.workspace = true +serde.workspace = true +serde_with.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +sha2.workspace = true +strum.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/router-core/src/access.rs b/litellm-rust/crates/router-core/src/access.rs new file mode 100644 index 00000000000..1b91fa417da --- /dev/null +++ b/litellm-rust/crates/router-core/src/access.rs @@ -0,0 +1,236 @@ +//! Who may reach a deployment: model access windows (`litellm_core_utils/access_windows.py`, +//! a deployment reserved for some teams during a daily wall-clock window) and model access +//! groups (`_filter_deployments_by_model_access_groups`, a caller allowed only through access +//! groups is kept to its groups' deployments). + +use std::{collections::BTreeSet, sync::Arc}; + +use jiff::{Timestamp, tz::TimeZone}; +use serde_json::Value; + +use crate::registry::Deployment; + +/// One `model_info.access_windows` entry (`ModelAccessWindow`), validated when the deployment +/// was added. +#[derive(Clone, Debug, PartialEq)] +pub struct AccessWindow { + /// Microseconds after midnight. + pub start: u64, + pub end: u64, + pub timezone: String, + pub team_ids: Vec, +} + +const MICROS_PER_SECOND: u64 = 1_000_000; + +/// A `datetime.time` as pydantic reads it: `HH:MM[:SS[.ffffff]]`, or seconds after midnight. +fn wall_clock(value: &Value) -> Option { + match value { + Value::Number(seconds) => { + let seconds = seconds.as_f64()?; + (0.0..86_400.0) + .contains(&seconds) + .then(|| (seconds * MICROS_PER_SECOND as f64).round() as u64) + } + Value::String(text) => { + let mut parts = text.splitn(3, ':'); + let hour: u64 = parts.next()?.parse().ok()?; + let minute: u64 = parts.next()?.parse().ok()?; + let (second, micros) = match parts.next() { + None => (0, 0), + Some(rest) => { + let (whole, fraction) = rest.split_once('.').unwrap_or((rest, "")); + let digits: String = fraction.chars().take(6).collect(); + let micros = if digits.is_empty() { + 0 + } else { + format!("{digits:0<6}").parse().ok()? + }; + (whole.parse().ok()?, micros) + } + }; + (hour < 24 && minute < 60 && second < 60) + .then(|| ((hour * 60 + minute) * 60 + second) * MICROS_PER_SECOND + micros) + } + _ => None, + } +} + +impl AccessWindow { + fn parse(value: &Value) -> Option { + let window = value.as_object()?; + Some(Self { + start: wall_clock(window.get("start")?)?, + end: wall_clock(window.get("end")?)?, + timezone: window.get("timezone")?.as_str()?.to_owned(), + team_ids: window + .get("team_ids")? + .as_array()? + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + }) + } + + /// `is_window_active`: the window's wall clock contains `now`, wrapping past midnight when + /// it ends before it starts. + pub fn is_active(&self, now: Timestamp) -> bool { + let Ok(zone) = TimeZone::get(&self.timezone) else { + return false; + }; + let local = now.to_zoned(zone).time(); + let micros = ((u64::from(local.hour().unsigned_abs()) * 60 + + u64::from(local.minute().unsigned_abs())) + * 60 + + u64::from(local.second().unsigned_abs())) + * MICROS_PER_SECOND + + u64::from(local.subsec_nanosecond().unsigned_abs()) / 1_000; + if self.start < self.end { + self.start <= micros && micros < self.end + } else { + micros >= self.start || micros < self.end + } + } + + /// The window's end as `{end:%H:%M}`. + pub fn end_hhmm(&self) -> String { + let minutes = self.end / MICROS_PER_SECOND / 60; + format!("{:02}:{:02}", minutes / 60, minutes % 60) + } +} + +/// A deployment's access windows. +pub fn access_windows(deployment: &Deployment) -> Vec { + deployment + .model_info + .raw + .get("access_windows") + .and_then(Value::as_array) + .map(|windows| windows.iter().filter_map(AccessWindow::parse).collect()) + .unwrap_or_default() +} + +/// `_reservation_blocking_window`: the first active window, unless the caller's team is named +/// by one of the active windows. +fn blocking_window( + deployment: &Deployment, + team_id: Option<&str>, + now: Timestamp, +) -> Option { + let active: Vec = access_windows(deployment) + .into_iter() + .filter(|window| window.is_active(now)) + .collect(); + if let Some(team) = team_id + && active + .iter() + .any(|window| window.team_ids.iter().any(|id| id == team)) + { + return None; + } + active.into_iter().next() +} + +/// `filter_reserved_deployments`: the deployments no active window withholds from the caller, +/// and the first window that withheld one. +pub fn filter_reserved( + deployments: Vec>, + team_id: Option<&str>, + now: Timestamp, +) -> (Vec>, Option) { + let mut blocking = None; + let kept = deployments + .into_iter() + .filter( + |deployment| match blocking_window(deployment, team_id, now) { + None => true, + Some(window) => { + blocking.get_or_insert(window); + false + } + }, + ) + .collect(); + (kept, blocking) +} + +/// `_filter_deployments_by_model_access_groups`: a caller whose allowed models (`models` plus +/// `team_models`) reach `model` only through access groups is kept to the deployments in those +/// groups. `model_groups` are the access groups the rows listed for `model` carry. +pub fn filter_by_access_groups( + model: &str, + deployments: Vec>, + allowed_models: &[String], + model_groups: impl FnOnce() -> BTreeSet, +) -> Vec> { + if deployments.is_empty() || allowed_models.is_empty() { + return deployments; + } + let allowed = |name: &str| allowed_models.iter().any(|m| m == name); + if allowed(model) || allowed("*") || allowed("all-proxy-models") { + return deployments; + } + let groups: BTreeSet = model_groups() + .into_iter() + .filter(|group| allowed(group)) + .collect(); + if groups.is_empty() { + return deployments; + } + deployments + .into_iter() + .filter(|d| { + d.model_info + .access_groups + .iter() + .any(|g| groups.contains(g)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::minutes(json!("09:30"), Some(34_200_000_000))] + #[case::seconds(json!("09:30:15"), Some(34_215_000_000))] + #[case::fraction(json!("00:00:00.5"), Some(500_000))] + #[case::number(json!(60), Some(60_000_000))] + #[case::bad(json!("25:00"), None)] + fn wall_clock_reads_pydantic_times(#[case] raw: Value, #[case] expected: Option) { + assert_eq!(wall_clock(&raw), expected); + } + + #[rstest] + #[case::inside("2026-01-01T10:00:00Z", true)] + #[case::at_end("2026-01-01T17:00:00Z", false)] + #[case::before("2026-01-01T08:59:59Z", false)] + fn a_window_is_active_inside_its_wall_clock(#[case] now: &str, #[case] expected: bool) { + let window = AccessWindow { + start: wall_clock(&json!("09:00")).unwrap(), + end: wall_clock(&json!("17:00")).unwrap(), + timezone: "UTC".to_owned(), + team_ids: vec!["t".to_owned()], + }; + assert_eq!(window.is_active(now.parse().unwrap()), expected); + assert_eq!(window.end_hhmm(), "17:00"); + } + + #[test] + fn an_overnight_window_wraps_past_midnight() { + let window = AccessWindow { + start: wall_clock(&json!("22:00")).unwrap(), + end: wall_clock(&json!("02:00")).unwrap(), + timezone: "UTC".to_owned(), + team_ids: vec!["t".to_owned()], + }; + assert!(window.is_active("2026-01-01T23:00:00Z".parse().unwrap())); + assert!(window.is_active("2026-01-01T01:00:00Z".parse().unwrap())); + assert!(!window.is_active("2026-01-01T12:00:00Z".parse().unwrap())); + } +} diff --git a/litellm-rust/crates/router-core/src/build.rs b/litellm-rust/crates/router-core/src/build.rs new file mode 100644 index 00000000000..36bad012b3a --- /dev/null +++ b/litellm-rust/crates/router-core/src/build.rs @@ -0,0 +1,386 @@ +//! The deployment build: how configured entries become the registry's deployments. +//! +//! The core owns the loop (`set_model_list`, `_create_deployment` and `_add_deployment`): which +//! values are secret references, the id of a deployment that declares none, one deployment per +//! `organization`, and the order everything happens in. What needs the embedding process is a +//! host step, asked for one deployment at a time so an error surfaces exactly where the Python +//! router raises it: +//! - [`Host::secrets`]: the `os.environ/` references the entry holds; +//! - [`Host::validate`]: the host's checks, and the entry as it normalizes it; +//! - [`Host::provider`]: the provider that serves it; +//! - [`Host::added`]: the host's side effects, and the entry as the registry stores it. + +use serde_json::{Map, Value}; + +use crate::{ + error::{ConfigError, Error}, + host::{BuildAsk, BuildPath, Built, Host, Provided, SecretAt, SecretRef}, + registry::{DeploymentSpec, deployment_id}, +}; + +/// The `litellm_params` keys whose `os.environ/` value is a secret pointer the identity source +/// dereferences at use time, so the build leaves it as written. The router's copy of +/// `_WIF_POINTER_FIELDS` (litellm/types/router.py), which the proxy's config loader also reads. +pub const SECRET_POINTER_PARAMS: &[&str] = &[ + "anthropic_issuer_signing_key_ref", + "anthropic_keycloak_client_secret_ref", +]; + +const SECRET_PREFIX: &str = "os.environ/"; + +/// How a build's deployments join the registry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum Install { + /// They become every deployment (`set_model_list`). + Replace, + /// They are added at the end (a deployment retried once its provider resolves). + Append, + /// Each replaces the deployment with its id, if any, and is added at the end + /// (`add_deployment`, an upsert). + Upsert, +} + +/// One deployment the build accepted, as the registry stores it. +#[derive(Clone, Debug, PartialEq)] +pub struct BuiltDeployment { + /// The entry index in the list the build was given. + pub index: usize, + pub spec: DeploymentSpec, +} + +/// Build `entries` (model_list entries) through `host`, in order. `resolve_secrets` resolves +/// top-level `os.environ/` references first, as `set_model_list` does; `add_deployment` and +/// upserts take their params as given. `install` takes each deployment as soon as it is +/// accepted, so it joins the registry before the next one is built, as the Python router adds +/// each in turn. +pub async fn build( + host: &dyn Host, + path: BuildPath, + entries: Vec>, + resolve_secrets: bool, + mut install: impl FnMut(&BuiltDeployment) -> Result<(), Error> + Send, +) -> Result, Error> { + let declared: Vec> = entries.iter().map(declared_id).collect(); + let mut built = Vec::new(); + for (index, mut entry) in entries.into_iter().enumerate() { + let model_name = text(entry.get("model_name")); + let mut params = match entry.remove("litellm_params") { + Some(Value::Object(params)) => params, + _ => Map::new(), + }; + if resolve_secrets { + let refs = param_refs(¶ms); + if !refs.is_empty() { + let values = host.secrets(index, &refs).await?; + for (secret, value) in refs.into_iter().zip(values) { + if let SecretAt::Param(key) = secret.at { + params.insert(key, value); + } + } + } + } + let mut info = match entry.remove("model_info") { + Some(Value::Object(info)) => info, + _ => Map::new(), + }; + let declared_id = declared[index].clone(); + if !info.contains_key("id") { + info.insert( + "id".to_owned(), + Value::String(deployment_id(&model_name, ¶ms)), + ); + } + // `set_model_list` adds one deployment per organization of a list, each with the + // organization set on the same params (so every one keeps the list's id). + let organizations = match (path, params.get("organization")) { + (BuildPath::Create, Some(Value::Array(organizations))) => organizations.clone(), + _ => vec![Value::Null], + }; + let duplicate_id = path == BuildPath::Create + && declared_id.as_ref().is_some_and(|id| { + declared + .iter() + .filter(|other| other.as_ref() == Some(id)) + .count() + > 1 + }); + for organization in organizations { + if !organization.is_null() { + params.insert("organization".to_owned(), organization); + } + let mut deployment = entry.clone(); + deployment.insert("model_name".to_owned(), Value::String(model_name.clone())); + deployment.insert("litellm_params".to_owned(), Value::Object(params.clone())); + deployment.insert("model_info".to_owned(), Value::Object(info.clone())); + if let Some(spec) = build_one( + host, + path, + index, + &deployment, + declared_id.as_deref(), + duplicate_id, + ) + .await? + { + let deployment = BuiltDeployment { index, spec }; + install(&deployment)?; + built.push(deployment); + } + } + } + Ok(built) +} + +/// One deployment's steps: validate, provider, the data-source references, added. +async fn build_one( + host: &dyn Host, + path: BuildPath, + index: usize, + entry: &Map, + declared_id: Option<&str>, + duplicate_id: bool, +) -> Result, Error> { + let ask = |entry| BuildAsk { + path, + index, + entry, + declared_id, + duplicate_id, + }; + let validated = match host.validate(&ask(entry)).await? { + Built::Entry(validated) => validated, + Built::Skip | Built::Invalid => return Ok(None), + }; + let provider = match host.provider(&ask(&validated)).await? { + Provided::Provider(provider) => provider, + Provided::Invalid => return Ok(None), + }; + let refs = data_source_refs(&validated); + if !refs.is_empty() { + let _ = host.secrets(index, &refs).await?; + } + let stored = match host.added(&ask(&validated), provider.as_deref()).await? { + Built::Entry(stored) => stored, + Built::Skip | Built::Invalid => return Ok(None), + }; + let model_name = text(stored.get("model_name")); + let mut spec: DeploymentSpec = serde_json::from_value(Value::Object(stored)) + .map_err(|source| ConfigError::InvalidDeployment { model_name, source })?; + spec.provider = provider; + Ok(Some(spec)) +} + +/// The top-level `litellm_params` values `set_model_list` resolves. +fn param_refs(params: &Map) -> Vec { + params + .iter() + .filter(|(key, _)| !SECRET_POINTER_PARAMS.contains(&key.as_str())) + .filter_map(|(key, value)| { + let name = value + .as_str() + .filter(|name| name.starts_with(SECRET_PREFIX))?; + Some(SecretRef { + name: name.to_owned(), + at: SecretAt::Param(key.clone()), + }) + }) + .collect() +} + +/// The `dataSources[].parameters.endpoint` / `key` references `_add_deployment` reads from the +/// environment. +fn data_source_refs(entry: &Map) -> Vec { + let sources = entry + .get("litellm_params") + .and_then(|params| params.get("dataSources")) + .and_then(Value::as_array); + let mut refs = Vec::new(); + for (source, data_source) in sources.into_iter().flatten().enumerate() { + let Some(parameters) = data_source.get("parameters").and_then(Value::as_object) else { + continue; + }; + for key in ["endpoint", "key"] { + if let Some(name) = parameters + .get(key) + .and_then(Value::as_str) + .filter(|name| name.starts_with(SECRET_PREFIX)) + { + refs.push(SecretRef { + name: name.to_owned(), + at: SecretAt::DataSource { + source, + key: key.to_owned(), + }, + }); + } + } + } + refs +} + +/// The `model_info.id` an entry declares, as Python's `str()` spells it. +fn declared_id(entry: &Map) -> Option { + match entry.get("model_info")?.get("id")? { + Value::Null => None, + Value::String(id) => Some(id.clone()), + other => Some(other.to_string()), + } +} + +/// A name as Python's `str()` spells it. +fn text(value: Option<&Value>) -> String { + match value { + Some(Value::String(text)) => text.clone(), + Some(Value::Null) | None => "None".to_owned(), + Some(other) => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use serde_json::json; + + use super::*; + use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + error::AttemptError, + host::{Attempt, Output, RouterEvent, RouterRequest, RoutingContext, SelectStage}, + registry::Deployment, + }; + + /// Records every build step and resolves `os.environ/X` to `secret-X`. + #[derive(Default)] + struct Recording { + steps: Mutex>, + } + + #[async_trait] + impl Host for Recording { + async fn invoke(&self, _attempt: &Attempt) -> Result { + unreachable!("the build runs no attempt") + } + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + Ok(true) + } + async fn sleep(&self, _duration: std::time::Duration) {} + fn observe(&self, _event: &RouterEvent) {} + fn cache(&self) -> Arc { + Arc::new(InMemoryRouterCache::default()) + } + async fn secrets(&self, _index: usize, refs: &[SecretRef]) -> Result, Error> { + self.steps.lock().unwrap().push(format!("secrets {refs:?}")); + Ok(refs + .iter() + .map(|r| json!(format!("secret-{}", &r.name[SECRET_PREFIX.len()..]))) + .collect()) + } + async fn validate(&self, ask: &BuildAsk<'_>) -> Result { + self.steps.lock().unwrap().push(format!( + "validate {} {}", + ask.index, ask.entry["litellm_params"] + )); + Ok(Built::Entry(ask.entry.clone())) + } + async fn provider(&self, ask: &BuildAsk<'_>) -> Result { + self.steps + .lock() + .unwrap() + .push(format!("provider {}", ask.index)); + Ok(Provided::Provider(Some("azure".to_owned()))) + } + async fn added(&self, ask: &BuildAsk<'_>, provider: Option<&str>) -> Result { + self.steps + .lock() + .unwrap() + .push(format!("added {} {provider:?}", ask.index)); + Ok(Built::Entry(ask.entry.clone())) + } + } + + fn entries(value: Value) -> Vec> { + serde_json::from_value(value).unwrap() + } + + #[tokio::test] + async fn create_resolves_secrets_then_builds_one_deployment_per_organization() { + let host = Recording::default(); + let built = build( + &host, + BuildPath::Create, + entries(json!([{ + "model_name": "g", + "litellm_params": { + "model": "azure/x", + "api_key": "os.environ/KEY", + "anthropic_issuer_signing_key_ref": "os.environ/POINTER", + "organization": ["a", "b"], + }, + }])), + true, + |_| Ok(()), + ) + .await + .unwrap(); + let steps = host.steps.lock().unwrap().clone(); + assert_eq!(steps.len(), 7, "{steps:?}"); + assert!(steps[0].starts_with("secrets") && steps[0].contains("KEY")); + assert!(!steps[0].contains("POINTER")); + assert!(steps[1].contains(r#""organization":"a""#)); + assert!(steps[1].contains("secret-KEY")); + assert_eq!(steps[3], "added 0 Some(\"azure\")"); + assert!(steps[4].contains(r#""organization":"b""#)); + assert_eq!(built.len(), 2); + assert_eq!(built[0].spec.provider.as_deref(), Some("azure")); + assert_eq!( + built[0].spec.model_info["id"], + built[1].spec.model_info["id"] + ); + let deployment = Deployment::from_spec(built[0].spec.clone()).unwrap(); + assert_eq!(deployment.provider(), "azure"); + } + + #[tokio::test] + async fn add_leaves_params_as_given_and_asks_for_data_source_references() { + let host = Recording::default(); + let built = build( + &host, + BuildPath::Add, + entries(json!([{ + "model_name": "g", + "litellm_params": { + "model": "azure/x", + "api_key": "os.environ/KEY", + "organization": ["a", "b"], + "dataSources": [{"parameters": {"endpoint": "os.environ/END", "key": "plain"}}], + }, + "model_info": {"id": "given"}, + }])), + false, + |_| Ok(()), + ) + .await + .unwrap(); + let steps = host.steps.lock().unwrap().clone(); + assert_eq!(steps.len(), 4, "{steps:?}"); + assert!(steps[0].contains("os.environ/KEY")); + assert!(steps[2].contains("DataSource { source: 0, key: \"endpoint\" }")); + assert_eq!(built.len(), 1); + assert_eq!(built[0].spec.model_info["id"], "given"); + } +} diff --git a/litellm-rust/crates/router-core/src/cache.rs b/litellm-rust/crates/router-core/src/cache.rs new file mode 100644 index 00000000000..9811e34a3e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/cache.rs @@ -0,0 +1,159 @@ +//! [`RouterCache`]: the DualCache-style store all routing state lives in, under the keys, +//! encodings and TTLs the Python router uses, so Python and Rust workers share it. Method names +//! follow `litellm-cache`'s `BaseCache`, `BatchCache`, `CounterCache` and `DeleteCache`. + +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use litellm_cache::Error; +use parking_lot::Mutex; +use serde_json::Value; + +#[async_trait] +pub trait RouterCache: Send + Sync { + async fn async_get_cache(&self, key: &str) -> Result, Error>; + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), Error>; + + /// Add `amount` to the number at `key` (0 when missing) and return the new value. The TTL + /// applies when the key is created, as Redis `INCRBYFLOAT` + `EXPIRE NX` does. + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result; + + async fn async_batch_get_cache(&self, keys: &[String]) -> Result>, Error>; + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error>; + + /// Read this worker's in-memory tier only (`local_only=True`). + async fn async_get_local(&self, key: &str) -> Result, Error> { + self.async_get_cache(key).await + } + + /// Add to a counter in this worker's in-memory tier only (`local_only=True`). + async fn async_increment_local( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + self.async_increment(key, amount, ttl).await + } + + /// `_increment_deployment_usage`'s write: add each `(key, amount)` in one pipeline. With a + /// shared tier, this worker's in-memory tier then holds the shared totals, so its reads see + /// what other workers counted up to now. + async fn async_increment_usage( + &self, + increments: &[(String, f64)], + ttl: Option, + ) -> Result<(), Error> { + for (key, amount) in increments { + self.async_increment(key, *amount, ttl).await?; + } + Ok(()) + } +} + +#[derive(Default)] +struct Entry { + value: Value, + expires: Option, +} + +/// An in-process [`RouterCache`], for tests and for state Python keeps `local_only`. +#[derive(Clone, Default)] +pub struct InMemoryRouterCache { + entries: Arc>>, +} + +impl InMemoryRouterCache { + fn live(entries: &mut HashMap, key: &str) -> Option { + let expired = entries + .get(key) + .is_some_and(|entry| entry.expires.is_some_and(|at| at <= Instant::now())); + if expired { + entries.remove(key); + } + entries.get(key).map(|entry| entry.value.clone()) + } + + pub fn get(&self, key: &str) -> Option { + Self::live(&mut self.entries.lock(), key) + } + + pub fn increment(&self, key: &str, amount: f64, ttl: Option) -> f64 { + let mut entries = self.entries.lock(); + let current = Self::live(&mut entries, key).and_then(|value| value.as_f64()); + let next = current.unwrap_or(0.0) + amount; + let value = serde_json::Number::from_f64(next) + .map(|number| { + if next.fract() == 0.0 && next.abs() < i64::MAX as f64 { + Value::from(next as i64) + } else { + Value::Number(number) + } + }) + .unwrap_or(Value::Null); + let expires = match (current, entries.get(key)) { + (Some(_), Some(entry)) => entry.expires, + _ => ttl.map(|ttl| Instant::now() + ttl), + }; + entries.insert(key.to_owned(), Entry { value, expires }); + next + } +} + +#[async_trait] +impl RouterCache for InMemoryRouterCache { + async fn async_get_cache(&self, key: &str) -> Result, Error> { + Ok(self.get(key)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), Error> { + let expires = ttl.map(|ttl| Instant::now() + ttl); + self.entries + .lock() + .insert(key.to_owned(), Entry { value, expires }); + Ok(()) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + Ok(self.increment(key, amount, ttl)) + } + + async fn async_batch_get_cache(&self, keys: &[String]) -> Result>, Error> { + let mut entries = self.entries.lock(); + Ok(keys + .iter() + .map(|key| Self::live(&mut entries, key)) + .collect()) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { + self.entries.lock().remove(key); + Ok(()) + } +} diff --git a/litellm-rust/crates/router-core/src/call.rs b/litellm-rust/crates/router-core/src/call.rs new file mode 100644 index 00000000000..287bb98d8b7 --- /dev/null +++ b/litellm-rust/crates/router-core/src/call.rs @@ -0,0 +1,1555 @@ +//! The call loop: select, run the attempt, then retry in the same group or fall back to another +//! until it succeeds or the policy is exhausted. Mirrors `Router.async_function_with_fallbacks`, +//! `async_function_with_fallbacks_common_utils`, `async_function_with_retries`, +//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and +//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`. + +use std::{ + borrow::Cow, + future::Future, + pin::Pin, + sync::Arc, + time::{Duration, Instant}, +}; + +use serde_json::{Map, Value}; + +use tracing::warn; + +use crate::{ + Core, RouterStateSnapshot, + config::Op, + error::{AttemptError, ConfigError, Error, FailureKind, Unsupported}, + fallback::{FallbackKeys, FallbackTarget, Fallbacks}, + health::{GroupShape, should_retry_status, team_model_has_alternatives}, + host::{ + Attempt, Blocked, Output, PickError, RetryCause, RouterEvent, RouterRequest, + UncountedRetries, + }, + host_index::python_str, + lookup::Lookup, + mock::MockFailure, + outcome::OutcomeWrite, + prerouting::{PreRoute, PreRouted}, + registry::{ResolveKind, ResolveScope}, + retry::{RetryContext, resolve_retry_policy, should_retry}, + select::{Candidates, SelectInputs}, +}; + +mod stream; + +pub use self::stream::{StreamOrigin, StreamPolicy}; + +/// `INITIAL_RETRY_DELAY`, `MAX_RETRY_DELAY` and `JITTER` in `litellm/constants.py`. +const INITIAL_RETRY_DELAY: f64 = 0.5; +const MAX_RETRY_DELAY: f64 = 8.0; +const JITTER: f64 = 0.75; +/// `calculate_retry_after` obeys a provider's retry-after up to a minute. +const MAX_HONOURED_RETRY_AFTER: Duration = Duration::from_secs(60); + +#[derive(Clone, Debug)] +pub struct RouterResponse { + pub output: Output, + pub deployment_id: String, + pub model_group: String, + pub attempted_retries: u32, + /// The retry budget of the loop a retry succeeded in (`x-litellm-max-retries`); `None` + /// when the first attempt succeeded. + pub max_retries: Option, + pub attempted_fallbacks: u32, +} + +/// A failed attempt with the deployment it ran on, when one was picked. +struct Failed { + error: Box, + deployment_id: Option, +} + +impl From for Failed { + fn from(error: Error) -> Self { + Self { + error: Box::new(error), + deployment_id: None, + } + } +} + +/// Where a level of the fallback chain sits: `depth` is `run_async_fallback`'s counter, one more +/// for every target tried, `level` how many fallbacks deep the chain is, and `scope` the level's +/// own copy of the request metadata (every hop copies its parent's). +#[derive(Clone, Copy, Debug, Default)] +struct FallbackHop { + depth: u32, + level: u32, + scope: u32, +} + +/// What the whole chain shares: the attempts made (`attempted_targets`) and the scopes handed out. +#[derive(Debug, Default)] +struct FallbackChain { + attempted: Vec, + scopes: u32, +} + +/// What one level of the chain serves: a group, or the same group at a later `order`, with the +/// request params the chain's dict targets merged so far (`kwargs.update(target)`). +#[derive(Clone, Debug)] +struct Target { + group: String, + order: Option, + params: Option>>, + /// `_excluded_deployment_ids` a weighted failover hop put on the request: never picked. + excluded: Arc<[String]>, + /// `_failover_excluded_ids` in the level's metadata: the deployments weighted failover + /// already excluded on the way here. + failover_excluded: Arc<[String]>, +} + +/// A step of a level's fallback chain: a configured target, or an order level of the group. +#[derive(Clone, Debug)] +enum Next { + Fallback(FallbackTarget), + Order(String, i64), + /// A weighted failover hop: the same group without the deployments already excluded. + Failover(String, Arc<[String]>), +} + +/// An attempt the chain already made (`fallback_attempt_key`). +#[derive(Clone, Debug, PartialEq)] +enum Tried { + Target(FallbackTarget), + Order(String, i64), + Failover(String, Vec), +} + +/// The fallback lists a call reads: the request's own where it passed them, else the router's. +struct Lists<'a> { + fallbacks: Option<&'a Fallbacks>, + context_window: Option<&'a Fallbacks>, + content_policy: Option<&'a Fallbacks>, + keys: &'a FallbackKeys, +} + +impl<'a> Lists<'a> { + fn of(snapshot: &'a RouterStateSnapshot, request: &'a RouterRequest) -> Self { + let dynamic = &snapshot.settings.dynamic; + let pick = |own: &'a Option>, router: Option<&'a Fallbacks>| match own { + Some(own) => own.as_ref(), + None => router, + }; + Self { + fallbacks: pick(&request.fallbacks, Some(&dynamic.fallbacks)), + context_window: pick( + &request.context_window_fallbacks, + dynamic.context_window_fallbacks.as_ref(), + ), + content_policy: pick( + &request.content_policy_fallbacks, + dynamic.content_policy_fallbacks.as_ref(), + ), + keys: &dynamic.fallback_keys, + } + } + + fn has_regular(&self) -> bool { + self.fallbacks.is_some_and(|list| !list.is_empty()) + } +} + +/// `has_unattempted_fallback_target` over the chain `list` resolves for the level's lookup +/// groups: before the first hop nothing counts as tried, and a dict target with its own params +/// is never skipped. +fn unattempted_chain( + list: &Fallbacks, + lists: &Lists<'_>, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], +) -> bool { + let mut lookup = vec![target.group.as_str()]; + if request.model != target.group { + lookup.push(request.model.as_str()); + } + list.lookup_groups(&lookup, lists.keys) + .is_some_and(|targets| { + at.level == 0 + || targets.iter().any(|next| match next.identity() { + FallbackTarget::Params(_) => true, + identity => !attempted.contains(&Tried::Target(identity)), + }) + }) +} + +/// The deployments `_core_get_model_list(model_name=group, team_id=team_id)` lists: the name's own +/// (a team's public name among them), an alias's target's, a routing group's members, and only +/// when none of those serve it, the wildcard routes (`_get_wildcard_deployments`), the proxy-wide +/// pattern's and then the team's. +fn group_members( + snapshot: &RouterStateSnapshot, + group: &str, + team_id: Option<&str>, +) -> Vec> { + let registry = &snapshot.registry; + let all = |name: &str, team_id: Option<&str>| { + crate::resolve::all_deployments(registry, name, team_id).unwrap_or_default() + }; + let members = |routing_group: &crate::registry::RoutingGroupSpec| { + crate::routing_groups::members(routing_group, |member| all(member, None)) + }; + let mut rows = all(group, team_id); + if let Some(alias_target) = registry.alias_of(group) { + rows.extend(match registry.routing_group_named(alias_target) { + Some(routing_group) => members(routing_group), + None => all(alias_target, None), + }); + } + if let Some(routing_group) = registry.routing_group_named(group) { + rows.extend(members(routing_group)); + } + if rows.is_empty() { + if let Some((_, routed)) = registry.wildcard_for(group) { + rows.extend(routed); + } + if let Some(team) = team_id { + rows.extend(Lookup::new(registry).team_wildcard(group, team)); + } + } + rows +} + +/// What `_maybe_run_weighted_failover` catches and defers to the regular fallbacks: +/// `openai.APIError` (every litellm provider error, a failure the host reported no classes for +/// among them), `RouterRateLimitError` and `RouterRateLimitErrorBasic`. Anything else propagates. +fn defers_to_regular_fallbacks(error: &Error) -> bool { + retried_failure(error).is_some_and(|failure| { + failure.classes.is_empty() + || [ + "APIError", + "BadRequestError", + "ServiceUnavailableError", + "RouterRateLimitError", + "RouterRateLimitErrorBasic", + ] + .iter() + .any(|class| failure.is_a(class)) + }) +} + +/// `is_bad_request_without_fallback` with `fallback_on_unsupported_params` off: an unsupported +/// params error, or a provider's plain 400 bad request. +fn is_bad_request_without_fallback(failure: &AttemptError) -> bool { + if !failure.is_a("BadRequestError") { + return false; + } + if failure.is_a("UnsupportedParamsError") { + return true; + } + if [ + "ContextWindowExceededError", + "ContentPolicyViolationError", + "UnsupportedEndpointError", + ] + .iter() + .any(|class| failure.is_a(class)) + { + return false; + } + failure.status == Some(400) && failure.names_provider +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// `KeyError`'s classes, most specific first. +/// The failure a retry loop decides by: an attempt's, or the exception a failed pick raised +/// (Python's loop retries a pick's error like any other: no deployment available, a stale +/// routing group's `KeyError`, ...). +fn retried_failure(error: &Error) -> Option> { + match error { + Error::Attempt(failure) => Some(Cow::Borrowed(failure)), + _ => error.pick_failure().map(Cow::Owned), + } +} + +impl Core { + pub(crate) async fn call( + &self, + op: Op, + request: Arc, + ) -> Result { + let snapshot = self.snapshot.load_full(); + let mut chain = FallbackChain::default(); + let target = Target { + group: request.model.clone(), + order: None, + params: None, + excluded: Arc::from([]), + failover_excluded: Arc::from([]), + }; + let at = FallbackHop::default(); + // A fallbacks mock stands in for the requested group's retry loop: no pick, no attempt. + let Some(mock) = request.mock_failure.filter(|mock| mock.before_retries()) else { + return self + .with_fallbacks(&snapshot, &op, &request, target, at, &mut chain) + .await; + }; + let failed = Failed::from(self.mock_failed(mock, &target, at, None)); + self.fall_back(&snapshot, &op, &request, target, at, &mut chain, failed) + .await + } + + /// One level of `async_function_with_fallbacks`: the target's retry loop, then its + /// fallbacks if every attempt failed. + #[allow(clippy::too_many_arguments)] + fn with_fallbacks<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a Arc, + target: Target, + at: FallbackHop, + chain: &'a mut FallbackChain, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let failed = match self + .with_retries(snapshot, op, request, &target, at, &chain.attempted) + .await + { + Ok(response) => return Ok(response), + Err(failed) => failed, + }; + self.fall_back(snapshot, op, request, target, at, chain, failed) + .await + }) + } + + /// The level after its own failure: the no-fallback checks, weighted failover, then the + /// fallback targets. A level that fails raises its own error, whatever its fallbacks raised, + /// except an order chain's failure on any error but a context window one, which raises what + /// the chain's last hop raised ([`RouterEvent::OrderChainFailed`]). + /// A stream that fails before content re-enters here (`fallback_stream`), so the group's + /// retry loop does not run again. + #[allow(clippy::too_many_arguments)] + async fn fall_back( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: Target, + at: FallbackHop, + chain: &mut FallbackChain, + failed: Failed, + ) -> Result { + let Failed { + error, + deployment_id: failed_id, + } = failed; + let error = *error; + self.level_failed(at, &target, &error, false); + if self.raises_without_fallback(snapshot, request, &error) { + return Err(error); + } + // `async_function_with_fallbacks_common_utils` lists the group's deployments + // (`_core_get_model_list`) for its order fallbacks: a written index position past the + // model list raises there, before any fallback. + if let Err(raised) = crate::resolve::all_deployments( + &snapshot.registry, + &target.group, + request.listing_team_id.as_deref(), + ) { + self.level_failed(at, &target, &raised, true); + return Err(raised); + } + let (targets, order_path) = self.fallback_targets(snapshot, request, &target, &error); + if !order_path { + self.host.observe(&RouterEvent::FallingBack { + level: at.level, + model: target.group.clone(), + }); + } + let mut target = target; + // `_maybe_run_weighted_failover`: its own `run_async_fallback`, before the regular one. + let failover = if order_path { + None + } else { + match self + .weighted_failover(snapshot, request, &target, &error, failed_id.as_deref()) + .await + { + Ok(failover) => failover, + Err(raised) => { + // `_get_routing_context` raised inside the level's fallback handling: the + // level raises that error, before any regular fallback. + self.level_failed(at, &target, &raised, true); + return Err(raised); + } + } + }; + if let Some(failover) = failover { + let excluded = match &failover { + Next::Failover(_, excluded) => excluded.clone(), + _ => Arc::from([]), + }; + target.failover_excluded = excluded; + match self + .run_fallbacks(snapshot, op, request, &target, at, chain, vec![failover]) + .await + { + Ok(response) => return Ok(response), + Err(raised) if defers_to_regular_fallbacks(raised.as_ref().unwrap_or(&error)) => {} + // Outside the `try` around the regular fallbacks: the level raises it as it is. + Err(raised) => { + self.host.observe(&RouterEvent::OrderChainFailed { + level: at.level, + last_hop: raised.is_some(), + }); + return Err(raised.unwrap_or(error)); + } + } + } + let Some(targets) = targets else { + self.host.observe(&RouterEvent::FallbacksFailed { + level: at.level, + chain: false, + last_hop: false, + }); + return Err(error); + }; + let last_hop_error = match self + .run_fallbacks(snapshot, op, request, &target, at, chain, targets) + .await + { + Ok(response) => return Ok(response), + Err(raised) => raised, + }; + if !order_path { + self.host.observe(&RouterEvent::FallbacksFailed { + level: at.level, + chain: true, + last_hop: last_hop_error.is_some(), + }); + return Err(error); + } + // The order path's `raise`, outside the `try` that re-raises the original error: only a + // context window error stays the one the caller sees. + let context_window = retried_failure(&error) + .is_some_and(|failure| failure.is_a("ContextWindowExceededError")); + let raised = last_hop_error.filter(|_| !context_window); + self.host.observe(&RouterEvent::OrderChainFailed { + level: at.level, + last_hop: raised.is_some(), + }); + Err(raised.unwrap_or(error)) + } + + /// `run_async_fallback` over one chain: the depth is checked once on entry, then every + /// target tried counts one more. When the chain served nothing, the error its last hop + /// raised (`error_from_fallbacks`), or `None` when no hop ran. + #[allow(clippy::too_many_arguments)] + fn run_fallbacks<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a Arc, + target: &'a Target, + at: FallbackHop, + chain: &'a mut FallbackChain, + targets: Vec, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let max_fallbacks = request + .max_fallbacks + .unwrap_or(snapshot.settings.max_fallbacks); + if at.depth >= max_fallbacks { + return Err(None); + } + let failed = Tried::Target(FallbackTarget::Name(target.group.clone())); + if !chain.attempted.contains(&failed) { + chain.attempted.push(failed); + } + // Before the loop, or the refusal would be one target's failure: no target of the + // chain is tried, and the walk raises a `ValueError` the host holds. + let dicts: Vec> = targets + .iter() + .filter_map(|next| match next { + Next::Fallback(FallbackTarget::Params(dict)) => Some(dict.clone()), + _ => None, + }) + .collect(); + if !dicts.is_empty() && self.host.refuses_fallbacks(at.level, &dicts).await { + let refused = AttemptError::host_raised(&["ValueError"]); + return Err(Some(Error::Attempt(refused))); + } + let mut params = target.params.clone(); + let mut hop = at.depth; + let mut last_hop_error = None; + // `kwargs["model"]` as a dict target's non-string `model` left it: later targets + // without a `model` keep it. + let mut unroutable: Option = None; + for next in targets { + let (tried, group, order) = match &next { + Next::Order(group, order) => ( + Tried::Order(group.clone(), *order), + group.clone(), + Some(*order), + ), + Next::Failover(group, excluded) => { + let mut key = excluded.to_vec(); + key.sort(); + (Tried::Failover(group.clone(), key), group.clone(), None) + } + Next::Fallback(FallbackTarget::Name(name)) if *name == target.group => continue, + // A provider-scoped request never leaves its group; a dict target with no + // `model` names no group, so it is skipped too. + Next::Fallback(fallback) + if request.same_group_fallbacks_only + && fallback.model() != Some(target.group.as_str()) => + { + continue; + } + Next::Fallback(fallback) => { + if let Some(model) = fallback.model() + && model != target.group + { + // A check that raises ends the loop outside its `try`, as the + // chain's own failure. + match self.host.allow_fallback(request, model).await { + Ok(true) => {} + Ok(false) => continue, + Err(raised) => return Err(Some(Error::Attempt(raised))), + } + } + let group = fallback.model().unwrap_or(&target.group).to_owned(); + (Tried::Target(fallback.identity()), group, None) + } + }; + if chain.attempted.contains(&tried) { + continue; + } + chain.attempted.push(tried); + if let Next::Fallback(fallback) = &next + && let Some(merged) = fallback.params() + { + let mut combined = params.as_deref().cloned().unwrap_or_default(); + combined.extend(merged.iter().map(|(k, v)| (k.clone(), v.clone()))); + params = Some(Arc::new(combined)); + } + match &next { + Next::Fallback(fallback) if fallback.model().is_none() => { + if let Some(model) = fallback.unroutable_model() { + unroutable = Some(model.clone()); + } + } + _ => unroutable = None, + } + let group = unroutable.as_ref().map_or(group, python_str); + hop += 1; + self.host.observe(&RouterEvent::RetryLogged { + scope: at.scope, + level: at.level, + model: target.group.clone(), + attempted_retries: None, + cause: RetryCause::Level, + }); + chain.scopes += 1; + self.host.observe(&RouterEvent::Fallback { + from: target.group.clone(), + to: group.clone(), + order, + depth: hop, + level: at.level + 1, + scope: chain.scopes, + parent: at.scope, + max_fallbacks, + unroutable: unroutable.clone(), + }); + let excluded = match &next { + Next::Failover(_, excluded) => excluded.clone(), + _ => target.excluded.clone(), + }; + let next = Target { + group: group.clone(), + order, + params: params.clone(), + excluded, + failover_excluded: target.failover_excluded.clone(), + }; + let parent = at; + let at = FallbackHop { + depth: hop, + level: at.level + 1, + scope: chain.scopes, + }; + // A non-string `model` is a hop that fails before any attempt, on an error its + // own level's fallback lookup raises too, so it walks no chain of its own. + let result = match &unroutable { + Some(_) => Err(Error::UnknownModel { + model: group.clone(), + }), + None => { + self.with_fallbacks(snapshot, op, request, next, at, chain) + .await + } + }; + self.host.observe(&RouterEvent::FallbackDone { + from: target.group.clone(), + to: group, + level: parent.level, + scope: at.scope, + parent: parent.scope, + succeeded: result.is_ok(), + }); + match result { + Ok(mut response) => { + response.attempted_fallbacks = hop; + return Ok(response); + } + Err(raised) => last_hop_error = Some(raised), + } + } + Err(last_hop_error) + }) + } + + /// `_maybe_run_weighted_failover`'s hop for a failed deployment, when it applies: the + /// simple-shuffle group re-picked without every deployment already excluded, while one + /// that is not cooling down remains. + async fn weighted_failover( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + error: &Error, + failed_id: Option<&str>, + ) -> Result, Error> { + if !snapshot.settings.enable_weighted_failover { + return Ok(None); + } + if retried_failure(error).is_some_and(|failure| { + failure.is_a("ContextWindowExceededError") + || failure.is_a("ContentPolicyViolationError") + }) { + return Ok(None); + } + // `_get_routing_context` runs first, whatever failed: a stale routing-group member map + // raises its `KeyError` here. + snapshot.missing_routing_group(request, &target.group)?; + if request.skip_weighted_failover || error.attempt().is_none() { + return Ok(None); + } + let kind = snapshot + .registry + .resolve(&target.group, ResolveScope::default()) + .map_or(ResolveKind::Group, |resolution| resolution.kind); + let (strategy, _) = crate::strategy::routing_context( + &snapshot.registry, + &snapshot.settings.routing_strategy, + request.routing_strategy.as_deref(), + &kind, + &target.group, + ); + if strategy != crate::config::RoutingStrategy::SimpleShuffle { + return Ok(None); + } + let Some(failed_id) = failed_id else { + return Ok(None); + }; + let mut excluded: Vec = target.failover_excluded.to_vec(); + if !excluded.iter().any(|id| id == failed_id) { + excluded.push(failed_id.to_owned()); + } + let listing_team = request.listing_team_id.as_deref(); + let ids: Vec = group_members(snapshot, &target.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .map(|d| d.id.clone()) + .collect(); + let cooling = self + .health + .active_cooldowns(&snapshot.registry.ids()) + .await + .unwrap_or_default(); + let remaining = ids + .iter() + .any(|id| !excluded.contains(id) && !cooling.iter().any(|(cooled, _)| cooled == id)); + Ok(remaining.then(|| Next::Failover(target.group.clone(), excluded.into()))) + } + + /// `async_function_with_fallbacks_common_utils` raises before looking for any fallback: + /// fallbacks are off, the error is a guardrail's, a bad request without + /// `fallback_on_unsupported_params`, or no fallback serves the error at all. + fn raises_without_fallback( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + error: &Error, + ) -> bool { + if request.disable_fallbacks { + return true; + } + // Python decides by the exception alone, so a pick's error falls back like an attempt's: + // an `UnsupportedParamsError` raises, and the refusals name no provider. Only an error no + // pick raises (no Python exception stands behind it) has no fallback. + retried_failure(error).is_none_or(|failure| { + failure.kind == FailureKind::GuardrailIntervention + || (!snapshot.settings.fallback_on_unsupported_params + && is_bad_request_without_fallback(&failure)) + }) + } + + /// The fallback targets for a failure, or `None` when it must be raised as is + /// (`async_function_with_fallbacks_common_utils`). + fn fallback_targets( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + error: &Error, + ) -> (Option>, bool) { + // A pick's error decides by its exception's classes as an attempt's does: a pre-call + // context window refusal takes the context window fallbacks, any other the generic ones. + let failure = retried_failure(error); + let is_a = |name: &str| failure.as_ref().is_some_and(|f| f.is_a(name)); + let lists = Lists::of(snapshot, request); + // `fallback_lookup_groups`: the group being served, then the group the request was for. + let mut lookup = vec![target.group.as_str()]; + if request.model != target.group { + lookup.push(request.model.as_str()); + } + let chain = |list: &Fallbacks| { + if list.is_client_side(lists.keys) { + Some(list.as_chain()) + } else { + list.lookup_groups(&lookup, lists.keys) + } + }; + let fallbacks_of = |targets: Vec| { + targets.into_iter().map(Next::Fallback).collect::>() + }; + // `context_window_fallback_model_group`: only a chain the group matched takes precedence. + let context_window_targets = is_a("ContextWindowExceededError") + .then(|| lists.context_window?.lookup_groups(&lookup, lists.keys)) + .flatten(); + let skip_order = is_a("ContentPolicyViolationError") || context_window_targets.is_some(); + if !skip_order { + // A routing group's members, with a priority group's priorities as their order. + let listing_team = request.listing_team_id.as_deref(); + let mut orders: Vec = group_members(snapshot, &target.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .filter_map(|d| d.order()) + .collect(); + orders.sort_unstable(); + orders.dedup(); + if orders.len() > 1 { + let current = target.order.unwrap_or(orders[0]); + let mut targets: Vec = orders + .into_iter() + .filter(|order| *order > current) + .map(|order| Next::Order(target.group.clone(), order)) + .collect(); + if let Some(list) = lists.fallbacks { + targets.extend(fallbacks_of(chain(list).unwrap_or_default())); + } + if !targets.is_empty() { + return (Some(targets), true); + } + } + } + if let Some(list) = lists.fallbacks + && list.is_client_side(lists.keys) + && context_window_targets.is_none() + { + return (Some(fallbacks_of(list.as_chain())), false); + } + if let Some(targets) = context_window_targets { + return (Some(fallbacks_of(targets)), false); + } + // A context window list with no match falls through to the regular fallbacks; a content + // policy list with no match raises. + if is_a("ContentPolicyViolationError") + && let Some(list) = lists.content_policy + { + return ( + list.lookup_groups(&lookup, lists.keys).map(fallbacks_of), + false, + ); + } + ( + lists + .fallbacks + .and_then(|list| list.lookup_groups(&lookup, lists.keys)) + .map(fallbacks_of), + false, + ) + } + + /// A level's first attempt, or the rate-limit mock in its place + /// (`_handle_mock_testing_rate_limit_error`), which the retry loop handles like a failed + /// attempt. + #[allow(clippy::too_many_arguments)] + async fn first_attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + max_retries: u32, + place: (FallbackHop, &[Tried]), + ) -> Result { + if request.mock_failure != Some(MockFailure::RateLimit) { + return self + .attempt( + snapshot, + op, + request, + target, + inputs, + (0, max_retries), + place, + ) + .await; + } + // A lone deployment's `num_retries` rides on the mock error, as on a real one. + let lone = snapshot + .registry + .group(&target.group) + .filter(|group| group.len() == 1) + .and_then(|group| group[0].raw_params.get("num_retries")?.as_u64()) + .and_then(|retries| u32::try_from(retries).ok()); + Err(self + .mock_failed(MockFailure::RateLimit, target, place.0, lone) + .into()) + } + + async fn with_retries( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> Result { + let place = (at, attempted); + let mut inputs = SelectInputs { + target_order: target.order, + excluded_ids: target.excluded.clone(), + requested: Some(target.group.clone()), + fallback_hop: at.level > 0, + ..SelectInputs::default() + }; + let counting = SelectInputs { + count_only: true, + ..SelectInputs::default() + }; + let settings = &snapshot.settings; + let first_budget = request.num_retries.unwrap_or(settings.num_retries); + self.host.observe(&RouterEvent::RetriesOpened { + scope: at.scope, + max_retries: first_budget, + model_group_size: snapshot.registry.model_list_len(&target.group), + }); + let first = self + .first_attempt(snapshot, op, request, target, &inputs, first_budget, place) + .await; + let mut failed = match first { + Ok(response) => return Ok(response), + Err(failed) => failed, + }; + self.pick_failed(at, target, &failed.error); + // A guardrail's block and a failed agentic follow-up are re-raised as they are, before + // anything else looks at them (`async_function_with_retries`). + if matches!( + &*failed.error, + Error::Attempt(failure) + if failure.kind == FailureKind::GuardrailIntervention || failure.agentic_followup + ) { + return Err(failed); + } + // The first failure looks the group's retry policy up; a lookup that raises replaces the + // failure with its own error, before any retry (`async_function_with_retries`). + if request.retry_policy_raises.contains(&target.group) { + self.host.observe(&RouterEvent::RetryPolicyRaised { + level: at.level, + model: target.group.clone(), + cause: RetryCause::of(&failed.error), + }); + return Err(Failed { + error: Box::new(Error::Attempt(AttemptError::host_raised(&["Exception"]))), + deployment_id: None, + }); + } + let Some(first_failure) = retried_failure(&failed.error) else { + return Err(failed); + }; + let failure: &AttemptError = &first_failure; + // A request's `num_retries`, a count or not, wins over the deployment's and the router's. + let mut num_retries = if request.uncounted_retries.is_some() { + 0 + } else { + request + .num_retries + .or(failure.num_retries) + .unwrap_or(settings.num_retries) + }; + let group_policies = request + .model_group_retry_policy + .as_ref() + .unwrap_or(&settings.model_group_retry_policy); + let policy_retries = (request.num_retries != Some(0)) + .then(|| { + resolve_retry_policy( + settings.retry_policy.as_ref(), + &target.group, + Some(group_policies), + )? + .retries_for(failure) + }) + .flatten(); + let policy_applies = policy_retries.is_some(); + num_retries = policy_retries.unwrap_or(num_retries); + drop(first_failure); + // `_async_get_healthy_deployments` finds no group for a deployment id, so a call by id + // counts no deployments when deciding whether to retry. + let counted = |c: &Candidates, n: usize| if c.kind.is_single() { 0 } else { n }; + let all = self + .candidates(snapshot, op, request, &target.group, &counting) + .await + .map(|c| counted(&c, c.all.len())) + .unwrap_or(0); + + for retry in 0..=num_retries { + if retry > 0 { + self.log_retry(target, at, retry, &failed.error); + } + let healthy_ids: Vec = match self + .candidates(snapshot, op, request, &target.group, &counting) + .await + { + Ok(c) => { + let ids = c.healthy.iter().map(|d| d.id.clone()); + ids.take(counted(&c, c.healthy.len())).collect() + } + // `_async_get_healthy_deployments` swallows the lookup's error and still reads the + // cooldowns: a Redis round trip the refused pick's failure log runs across. + Err(_) => { + let _ = self.health.active_cooldowns(&snapshot.registry.ids()).await; + Vec::new() + } + }; + let healthy = healthy_ids.len(); + // A pick's error is retried as Python's loop retries any exception: a retry policy + // keeps going, else `should_retry_this_error` decides (it raises when nothing is + // healthy). + let Some(failure) = retried_failure(&failed.error) else { + return Err(failed); + }; + if !policy_applies && !self.should_retry(snapshot, request, &failure, healthy, all) { + return Err(failed); + } + let uncounted = request.uncounted_retries.filter(|_| !policy_applies); + if retry == 0 && uncounted.is_none() { + self.host.observe(&RouterEvent::RetryBudget { + scope: at.scope, + max_retries: num_retries, + }); + } + if uncounted == Some(UncountedRetries::Raises) { + self.host.observe(&RouterEvent::RetryCountRaised { + level: at.level, + model: target.group.clone(), + cause: RetryCause::of(&failed.error), + }); + return Err(Failed { + error: Box::new(Error::Attempt(AttemptError::host_raised(&["Exception"]))), + deployment_id: None, + }); + } + if num_retries == 0 { + return Err(failed); + } + if retry == 0 { + self.log_retry(target, at, 0, &failed.error); + } + let named = failure.failed_deployment_id.as_ref(); + if let (Some(status), Some(id)) = + (failure.status, named.or(failed.deployment_id.as_ref())) + && !should_retry_status(status) + && !inputs.skipped_ids.contains(id) + { + inputs.skipped_ids.push(id.clone()); + } + // An endpoint a deployment does not serve is retried on the others, until every + // one refused (`_every_deployment_refused`). + if failure.is_a("UnsupportedEndpointError") + && healthy_ids.iter().all(|id| inputs.skipped_ids.contains(id)) + { + return Err(failed); + } + let delay = self.time_to_sleep( + snapshot, + failure.retry_after, + num_retries - retry, + num_retries, + healthy, + all, + ); + self.host.sleep(delay).await; + if retry == num_retries { + self.host.observe(&RouterEvent::RetriesExhausted { + model_group: target.group.clone(), + attempted: num_retries, + max: num_retries, + level: at.level, + picked: failed.error.attempt().is_none(), + }); + break; + } + self.host.observe(&RouterEvent::Retry { + model_group: target.group.clone(), + retry: retry + 1, + delay, + }); + match self + .attempt( + snapshot, + op, + request, + target, + &inputs, + (retry + 1, num_retries), + place, + ) + .await + { + Ok(mut response) => { + response.max_retries = Some(num_retries); + return Ok(response); + } + Err(next) if next.error.attempt().is_some_and(|f| f.agentic_followup) => { + return Err(next); + } + Err(next) => { + self.pick_failed(at, target, &next.error); + failed = next; + } + } + } + Err(failed) + } + + /// `error` is a pick's: reported as it happens, whatever the level then does with it. + fn pick_failed(&self, at: FallbackHop, target: &Target, error: &Error) { + if let Some(picked) = PickError::of(error) { + self.host.observe(&RouterEvent::PickFailed { + level: at.level, + model: target.group.clone(), + error: picked, + }); + } + } + + /// The level serving `target` raises a pick's `error` ([`RouterEvent::LevelFailed`]); + /// `handling` when its fallback handling raised it. + fn level_failed(&self, at: FallbackHop, target: &Target, error: &Error, handling: bool) { + if let Some(picked) = PickError::of(error) { + self.host.observe(&RouterEvent::LevelFailed { + level: at.level, + model: target.group.clone(), + error: picked, + handling, + }); + } + } + + /// `_core_log_retry` for a failure in the level's retry loop. + fn log_retry(&self, target: &Target, at: FallbackHop, retry: u32, error: &Error) { + let cause = match RetryCause::of(error) { + RetryCause::Other => RetryCause::Attempt, + cause => cause, + }; + self.host.observe(&RouterEvent::RetryLogged { + scope: at.scope, + level: at.level, + model: target.group.clone(), + attempted_retries: Some(retry), + cause, + }); + } + + /// A `mock_testing_*` failure for the level's group, reported to the host, which raises + /// the matching exception. + fn mock_failed( + &self, + mock: MockFailure, + target: &Target, + at: FallbackHop, + num_retries: Option, + ) -> Error { + self.host.observe(&RouterEvent::MockFailed { + mock, + level: at.level, + model: target.group.clone(), + num_retries, + }); + Error::Attempt(mock.failure(num_retries)) + } + + fn should_retry( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + failure: &AttemptError, + healthy: usize, + all: usize, + ) -> bool { + let lists = Lists::of(snapshot, request); + should_retry( + failure, + RetryContext { + healthy, + all, + has_context_window_fallbacks: lists.context_window.is_some(), + has_content_policy_fallbacks: lists.content_policy.is_some(), + has_regular_fallbacks: lists.has_regular(), + }, + ) + } + + /// `_time_to_sleep_before_retry` and `calculate_retry_after`. + fn time_to_sleep( + &self, + snapshot: &RouterStateSnapshot, + retry_after: Option, + remaining_retries: u32, + num_retries: u32, + healthy: usize, + all: usize, + ) -> Duration { + if all != 1 && healthy > 0 { + return Duration::ZERO; + } + let jitter = JITTER * self.rng.next_f64(); + if let Some(retry_after) = retry_after + && !retry_after.is_zero() + && retry_after <= MAX_HONOURED_RETRY_AFTER + { + return retry_after + Duration::from_secs_f64(jitter); + } + let exponent = num_retries.saturating_sub(remaining_retries); + let backoff = (INITIAL_RETRY_DELAY * 2f64.powi(exponent as i32)) + .max(snapshot.settings.retry_after.as_secs_f64()) + .min(MAX_RETRY_DELAY); + Duration::from_secs_f64(backoff + jitter) + } + + /// One attempt: select a deployment, run it through the host, record the outcome. + #[allow(clippy::too_many_arguments)] + async fn attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + (retry, max_retries): (u32, u32), + (at, attempted): (FallbackHop, &[Tried]), + ) -> Result { + if request.unrouted { + return self + .unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at) + .await; + } + let pick_started = Instant::now(); + let (deployment, candidates) = match self + .picked(snapshot, op, request, target, inputs, retry, at) + .await + { + Ok(picked) => picked, + Err(failed) if request.passthrough_on_no_deployment => { + self.pick_failed(at, target, &failed.error); + return self + .unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at) + .await; + } + Err(failed) => return Err(failed), + }; + self.host.observe(&RouterEvent::Selected { + model_group: candidates.model_group.clone(), + deployment_id: deployment.id.clone(), + pick: pick_started.elapsed(), + }); + if !self.host.supports(op, deployment.provider()) { + return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op { + op: op.clone(), + provider: deployment.provider().to_owned(), + })) + .into()); + } + // A name no deployment serves is routed to the first default fallback + // (`_common_checks_available_deployment`), still stamped as the name. + let defaulted = candidates.model_group != target.group + && matches!( + crate::resolve::resolve_request( + &snapshot.registry, + &target.group, + request.specific_deployment, + crate::resolve::Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + wildcard_request: candidates.wildcard_request.as_deref(), + ..crate::resolve::Caller::default() + }, + None, + snapshot.settings.dynamic.default_ref(), + jiff::Timestamp::now(), + ), + Ok(None) + ); + let attempt = Attempt { + op: op.clone(), + request: request.clone(), + deployment: deployment.clone(), + name: target.group.clone(), + model_group: if defaulted { + target.group.clone() + } else { + candidates.model_group.clone() + }, + target_order: target.order, + candidate_ids: candidates.all.iter().map(|d| d.id.clone()).collect(), + retry, + fallback_depth: at.depth, + fallback_level: at.level, + timeout: snapshot.settings.timeout, + params: target.params.clone(), + scope: at.scope, + max_retries, + model_group_size: snapshot.registry.model_list_len(&target.group), + }; + // Every attempt runs through the host; the core records its outcome itself, as far as the + // request's `OutcomeCallbacks` reach, except a deferred stream's: its request counts now + // (`increment_deployment_usage_for_response`), its success or failure when the host reports + // the stream's end. + match self.host.invoke(&attempt).await { + Ok(output) => { + let tracks = output + .tracks_usage + .unwrap_or_else(|| deployment.tracks_usage()); + let recorded = match (output.deferred, tracks) { + (true, true) => { + let tokens = output.total_tokens as f64; + self.health.add_usage(&deployment, None, tokens, 1.0).await + } + (true, false) => Ok(()), + (false, _) => { + self.health + .record_success( + request.outcome_callbacks, + &deployment, + output.total_tokens, + tracks, + ) + .await + } + }; + if let Err(error) = recorded { + warn!(deployment_id = %deployment.id, error = %error, "usage was not recorded"); + } + self.host.observe(&RouterEvent::Succeeded { + deployment_id: deployment.id.clone(), + }); + // `_should_raise_content_policy_error`: a blocked response is raised only when + // a content-policy fallback would serve it, and counts as no failure. + let raised = + match output.blocked { + Some(Blocked::ContentFilter) => self + .has_content_policy_fallback(snapshot, request, target, at, attempted), + Some(Blocked::Refusal) => self + .refusal_fallback_available(snapshot, request, target, at, attempted), + None => false, + }; + if let Some(blocked) = output.blocked.filter(|_| raised) { + self.host.observe(&RouterEvent::Blocked { + level: at.level, + model: target.group.clone(), + blocked, + }); + let mut failure = + AttemptError::new(FailureKind::ContentPolicyViolation, Some(400)); + failure.classes = crate::retry::content_policy_classes(); + return Err(Failed { + error: Box::new(Error::Attempt(failure)), + deployment_id: Some(deployment.id.clone()), + }); + } + Ok(RouterResponse { + output, + deployment_id: deployment.id.clone(), + model_group: candidates.model_group, + attempted_retries: retry, + max_retries: None, + attempted_fallbacks: at.depth, + }) + } + Err(failure) => { + if !failure.recorded { + self.record_failure(snapshot, request, &candidates, &deployment, &failure) + .await; + } + Err(Failed { + error: Box::new(Error::Attempt(failure)), + deployment_id: Some(deployment.id.clone()), + }) + } + } + } + + /// The attempt's pick: the deployment and the candidates it was picked from. + #[allow(clippy::too_many_arguments)] + async fn picked( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + inputs: &SelectInputs, + retry: u32, + at: FallbackHop, + ) -> Result<(Arc, Candidates), Failed> { + crate::routing_plugins::refuse_sync_pick( + snapshot.settings.routing_plugins, + &snapshot.settings.routing_strategy, + request, + )?; + let routed = self + .pre_routed(PreRoute { + op, + request, + name: &target.group, + fallback_depth: at.depth, + retry, + }) + .await?; + match routed { + PreRouted::Model(name) => { + snapshot.missing_routing_group(request, &name)?; + let candidates = self + .candidates(snapshot, op, request, &name, inputs) + .await?; + if candidates.healthy.is_empty() { + return Err(Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + } + .into()); + } + let deployment = self.pick(snapshot, op, request, &candidates).await?; + Ok((deployment, candidates)) + } + // A custom strategy's pick is served as is, for the requested name. + PreRouted::Deployment(deployment) => { + let candidates = Candidates { + name: target.group.clone(), + model_group: target.group.clone(), + kind: ResolveKind::Group, + all: vec![deployment.clone()], + healthy: vec![deployment.clone()], + wildcard_request: None, + }; + Ok((deployment, candidates)) + } + } + } + + /// An attempt on a name that is no model group: the host serves it under that name, with no + /// deployment picked and no outcome recorded. + async fn unrouted_attempt( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &Arc, + target: &Target, + (retry, max_retries): (u32, u32), + at: FallbackHop, + ) -> Result { + let deployment = + Arc::new(crate::registry::Deployment::unrouted(&target.group).map_err(Error::from)?); + let attempt = Attempt { + op: op.clone(), + request: request.clone(), + deployment, + name: target.group.clone(), + model_group: target.group.clone(), + target_order: target.order, + candidate_ids: Vec::new(), + retry, + fallback_depth: at.depth, + fallback_level: at.level, + timeout: snapshot.settings.timeout, + params: target.params.clone(), + scope: at.scope, + max_retries, + model_group_size: snapshot.registry.model_list_len(&target.group), + }; + match self.host.invoke(&attempt).await { + Ok(output) => Ok(RouterResponse { + output, + deployment_id: String::new(), + model_group: target.group.clone(), + attempted_retries: retry, + max_retries: None, + attempted_fallbacks: at.depth, + }), + Err(failure) => Err(Error::Attempt(failure).into()), + } + } + + /// `_has_content_policy_fallback`: the content-policy chain resolves for the level's + /// lookup groups and holds a target the chain has not tried; with no content-policy list, + /// any router `"*"` entry. + fn has_content_policy_fallback( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> bool { + let lists = Lists::of(snapshot, request); + match lists.content_policy { + Some(list) => unattempted_chain(list, &lists, request, target, at, attempted), + None => snapshot.settings.dynamic.fallbacks.has_star(), + } + } + + /// `_refusal_fallback_available`: the content-policy list decides when there is one; + /// else a router `"*"` entry, else a generic chain with a target the chain has not tried. + fn refusal_fallback_available( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + target: &Target, + at: FallbackHop, + attempted: &[Tried], + ) -> bool { + if request.disable_fallbacks { + return false; + } + let lists = Lists::of(snapshot, request); + if lists.content_policy.is_some() { + return self.has_content_policy_fallback(snapshot, request, target, at, attempted); + } + if snapshot.settings.dynamic.fallbacks.has_star() { + return true; + } + lists + .fallbacks + .is_some_and(|list| unattempted_chain(list, &lists, request, target, at, attempted)) + } + + /// `async_deployment_callback_on_failure`, when it runs for this failure. + async fn count_failed_request( + &self, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) { + if failure.uncounted_request { + return; + } + if let Err(error) = self.health.count_failed_request(deployment, None).await { + warn!(deployment_id = %deployment.id, error = %error, "usage was not recorded"); + } + } + + async fn record_failure( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + candidates: &Candidates, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) { + self.host.observe(&RouterEvent::AttemptFailed { + deployment_id: deployment.id.clone(), + kind: failure.kind, + status: failure.status, + }); + let recorded = request.outcome_callbacks.record( + OutcomeWrite::Failure, + self.attempt_failure(snapshot, request, candidates, deployment, failure), + ); + if let Err(error) = recorded.await { + warn!(deployment_id = %deployment.id, error = %error, "cooldown was not recorded"); + } + } + + /// What the failure callbacks record for a failed attempt. + async fn attempt_failure( + &self, + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + candidates: &Candidates, + deployment: &crate::registry::Deployment, + failure: &AttemptError, + ) -> Result<(), Error> { + // The default deployment's copy is no configured deployment: Python's cooldown check finds + // no group for its id and records nothing, unless the id names a configured deployment, + // which then takes the failure. + let configured; + let deployment = if candidates.kind == ResolveKind::Default { + match snapshot.registry.deployment(&deployment.id) { + Some(found) => { + configured = found; + &*configured + } + None => { + self.count_failed_request(deployment, failure).await; + return Ok(()); + } + } + } else { + deployment + }; + let shape = GroupShape { + // The deployment's own group decides, also when the call named the deployment by id. + single_deployment: snapshot + .registry + .group(&deployment.model_name) + .map_or(candidates.all.len() == 1, |group| group.len() == 1) + // A routing group call with alternatives cools its member down so selection + // moves on (`routing_group_alternatives`). + && !(candidates.kind == ResolveKind::RoutingGroup && candidates.all.len() > 1) + // As does a team deployment with a sibling under its public name + // (`team_model_alternatives`). + && !team_model_has_alternatives(&snapshot.registry, deployment), + has_fallback_group: snapshot + .settings + .dynamic + .fallback_targets(&[candidates.model_group.as_str()]) + .is_some(), + }; + self.count_failed_request(deployment, failure).await; + match self + .health + .record_failure( + &snapshot.settings, + deployment, + shape, + request.cooldown_time, + failure, + ) + .await + { + Ok(Some(cooldown)) => self.host.observe(&RouterEvent::CooldownSet { + deployment_id: deployment.id.clone(), + cooldown, + }), + Ok(None) => {} + Err(error) => return Err(error), + } + Ok(()) + } +} diff --git a/litellm-rust/crates/router-core/src/call/stream.rs b/litellm-rust/crates/router-core/src/call/stream.rs new file mode 100644 index 00000000000..a0fce4372dd --- /dev/null +++ b/litellm-rust/crates/router-core/src/call/stream.rs @@ -0,0 +1,456 @@ +//! Recovering a stream that failed after its attempt returned it. The host wraps every streamed +//! response; when the stream fails before the caller received content, the wrapper asks the core +//! what may still take over and hands the failure back here. Mirrors the decisions of the Python +//! router's stream wrappers: `_stream_can_fall_back`, `_refusal_fallback_available`, +//! `_anthropic_messages_stream_can_retry`, `_stream_retry_budget`, `_astream_retry_same_group`, +//! and the `async_function_with_fallbacks_common_utils` re-entry of the mid-stream fallback. + +use std::sync::Arc; + +use serde::Deserialize; + +use super::{Failed, FallbackChain, FallbackHop, Lists, Target, Tried, group_members}; +use crate::{ + Router, RouterStateSnapshot, + config::Op, + error::{AttemptError, Error, FailureKind}, + fallback::FallbackTarget, + health::should_retry_status, + host::{RouterEvent, RouterRequest, UncountedRetries}, + select::{Candidates, SelectInputs}, +}; + +/// Where a streamed response came from: the attempt that returned it. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct StreamOrigin { + /// The name the attempt served: the requested group, or the fallback target. + pub group: String, + /// The `order` level an order fallback targeted. + pub order: Option, + /// `fallback_depth` of the attempt. + pub depth: u32, + pub level: u32, + /// The deployment the attempt ran on. + pub deployment_id: Option, + /// Same-group stream retries this request already spent (`attempted_retries_for_request`). + pub attempted_retries: u32, + /// The budget the first stream retry committed to (`committed_retry_budget_for_request`). + pub committed_budget: Option, +} + +/// What may still take over a stream that fails before content. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StreamPolicy { + /// `_anthropic_messages_stream_can_retry`. + pub can_retry: bool, + /// `_stream_can_fall_back`. + pub can_fall_back: bool, + /// `_refusal_fallback_available`. + pub refusal_fallback: bool, + /// `_stream_retry_budget` for the failure kind asked about: the budget, and whether a retry + /// policy governs the retry. + pub retry_budget: Option<(u32, bool)>, +} + +impl StreamOrigin { + fn target(&self) -> Target { + Target { + group: self.group.clone(), + order: self.order, + params: None, + excluded: Arc::from([]), + failover_excluded: Arc::from([]), + } + } + + /// `fallback_lookup_groups`: the served group, then the group the request was for. + fn lookup<'a>(&'a self, request: &'a RouterRequest) -> Vec<&'a str> { + let mut lookup = vec![self.group.as_str()]; + if request.model != self.group { + lookup.push(request.model.as_str()); + } + lookup + } + + /// `has_unattempted_fallback_target`. The first attempt carries no attempted targets, so any + /// resolved chain counts; a fallback hop has already tried its own group and the original one. + fn has_unattempted(&self, request: &RouterRequest, chain: Option>) -> bool { + let Some(chain) = chain else { + return false; + }; + if self.depth == 0 { + return true; + } + chain + .iter() + .any(|target| *target != self.group && *target != request.model) + } +} + +/// The group names a chain targets (a dict target counts by its `model`). +fn names(chain: Option>) -> Option> { + chain.map(|targets| { + targets + .iter() + .filter_map(|target| target.model().map(str::to_owned)) + .collect() + }) +} + +/// The `content_policy_fallbacks` chain for the lookup groups: the request's list where it +/// passed one, else the router's (`_has_content_policy_fallback`). +fn content_policy_targets(lists: &Lists<'_>, lookup: &[&str]) -> Option>> { + lists + .content_policy + .map(|list| names(list.lookup_groups(lookup, lists.keys))) +} + +/// The router's `default_fallbacks`, which Python folds into its `fallbacks` as `{"*": [...]}`: a +/// request's own list replaces them. +fn folded_defaults<'a>(snapshot: &'a RouterStateSnapshot, request: &RouterRequest) -> &'a [String] { + match request.fallbacks { + Some(_) => &[], + None => &snapshot.settings.dynamic.default_fallbacks, + } +} + +/// The regular fallback targets for the lookup groups, then the folded `default_fallbacks`. +fn regular_targets( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + lists: &Lists<'_>, + lookup: &[&str], +) -> Option> { + let defaults = folded_defaults(snapshot, request); + names( + lists + .fallbacks + .and_then(|list| list.lookup_groups(lookup, lists.keys)), + ) + .or_else(|| (!defaults.is_empty()).then(|| defaults.to_vec())) +} + +/// `_has_default_fallbacks`: the router's list, whatever the request passed. +fn has_default_fallbacks(snapshot: &RouterStateSnapshot) -> bool { + let dynamic = &snapshot.settings.dynamic; + dynamic.fallbacks.has_star() || !dynamic.default_fallbacks.is_empty() +} + +impl Router { + /// What may still take over a stream from `origin` that fails before content, and the retry + /// budget for a failure of `kind`. + pub fn stream_policy( + &self, + request: &RouterRequest, + origin: &StreamOrigin, + kind: Option, + ) -> StreamPolicy { + let snapshot = self.core.snapshot.load_full(); + StreamPolicy { + can_retry: stream_can_retry(&snapshot, request, origin), + can_fall_back: stream_can_fall_back(&snapshot, request, origin), + refusal_fallback: refusal_fallback_available(&snapshot, request, origin), + retry_budget: kind.map(|kind| stream_retry_budget(&snapshot, request, origin, kind)), + } + } + + /// The fallback chain for a stream from `origin` that failed with `failure` before content: + /// `async_function_with_fallbacks_common_utils` re-entered from the stream wrapper, so the + /// group's own retry loop does not run again. + pub async fn fallback_stream( + &self, + op: Op, + request: RouterRequest, + origin: StreamOrigin, + failure: AttemptError, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let request = Arc::new(request); + let mut chain = FallbackChain::default(); + if request.model != origin.group { + chain + .attempted + .push(Tried::Target(FallbackTarget::Name(request.model.clone()))); + } + self.core + .fall_back( + &snapshot, + &op, + &request, + origin.target(), + FallbackHop { + depth: origin.depth, + level: origin.level, + scope: 0, + }, + &mut chain, + Failed::from(Error::Attempt(failure)), + ) + .await + } + + /// `_astream_retry_same_group`: re-run the attempt in the stream's own group until a retry + /// returns a response or the budget runs out. The failed result carries the newest failure. + pub async fn retry_stream( + &self, + op: Op, + request: RouterRequest, + origin: StreamOrigin, + failure: AttemptError, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let request = Arc::new(request); + let target = origin.target(); + let at = FallbackHop { + depth: origin.depth, + level: origin.level, + scope: 0, + }; + let counting = SelectInputs { + count_only: true, + ..SelectInputs::default() + }; + let counted = |c: &Candidates, n: usize| if c.kind.is_single() { 0 } else { n }; + let (healthy, all) = self + .core + .candidates(&snapshot, &op, &request, &target.group, &counting) + .await + .map(|c| (counted(&c, c.healthy.len()), counted(&c, c.all.len()))) + .unwrap_or((0, 0)); + let (budget, policy_applies) = + stream_retry_budget(&snapshot, &request, &origin, failure.kind); + let inputs = SelectInputs { + target_order: target.order, + requested: Some(target.group.clone()), + fallback_hop: at.level > 0, + ..SelectInputs::default() + }; + let mut last = failure; + for attempt in origin.attempted_retries..budget { + if last.agentic_followup { + break; + } + if !policy_applies + && !self + .core + .should_retry(&snapshot, &request, &last, healthy, all) + { + break; + } + let delay = self.core.time_to_sleep( + &snapshot, + last.retry_after, + budget - attempt, + budget, + healthy, + all, + ); + self.core.host.sleep(delay).await; + self.core.host.observe(&RouterEvent::Retry { + model_group: target.group.clone(), + retry: attempt + 1, + delay, + }); + match self + .core + .attempt( + &snapshot, + &op, + &request, + &target, + &inputs, + (attempt + 1, budget), + (at, &[]), + ) + .await + { + Ok(mut response) => { + response.attempted_retries = attempt + 1; + response.max_retries = Some(budget); + return Ok(response); + } + Err(failed) => match *failed.error { + // `_stream_fallback_error_for_raised`: a status the pre-stream retry rule + // would not retry ends the retries with that failure. + Error::Attempt(failure) => { + let stop = failure.status.is_some_and(|s| !should_retry_status(s)); + last = failure; + if stop { + break; + } + } + other => return Err(other), + }, + } + } + Err(Error::Attempt(last)) + } +} + +/// The request's `num_retries`, then the routed deployment's, then the router's +/// (`_anthropic_messages_plain_retry_budget`). +fn plain_retry_budget( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> u32 { + if let Some(retries) = request.num_retries { + return retries; + } + if request.uncounted_retries == Some(UncountedRetries::Negative) { + return 0; + } + let deployment = origin + .deployment_id + .as_deref() + .and_then(|id| snapshot.registry.deployment(id)); + let configured = deployment.and_then(|d| match d.raw_params.get("num_retries") { + Some(serde_json::Value::Number(n)) => n.as_u64().and_then(|n| u32::try_from(n).ok()), + Some(serde_json::Value::String(s)) if s.chars().all(|c| c.is_ascii_digit()) => { + s.parse().ok() + } + _ => None, + }); + configured.unwrap_or(snapshot.settings.num_retries) +} + +/// The retry policy for the request's model group (`_anthropic_messages_resolved_retry_policy`), +/// unless the request opted out with `num_retries=0`. A policy that does not read as one governs +/// nothing here, with no fall through to the router's `retry_policy`. +fn retry_policy<'a>( + snapshot: &'a RouterStateSnapshot, + request: &'a RouterRequest, + origin: &StreamOrigin, +) -> Option<&'a crate::config::RetryPolicy> { + if request.num_retries == Some(0) || request.retry_policy_raises.contains(&origin.group) { + return None; + } + let settings = &snapshot.settings; + crate::retry::resolve_retry_policy( + settings.retry_policy.as_ref(), + &origin.group, + Some( + request + .model_group_retry_policy + .as_ref() + .unwrap_or(&settings.model_group_retry_policy), + ), + ) +} + +fn stream_retry_budget( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, + kind: FailureKind, +) -> (u32, bool) { + let failure = AttemptError::new(kind, None); + let policy_retries = + retry_policy(snapshot, request, origin).and_then(|p| p.retries_for(&failure)); + if origin.attempted_retries > 0 + && let Some(committed) = origin.committed_budget + { + return (committed, policy_retries.is_some()); + } + match policy_retries { + Some(retries) => (retries, true), + None => (plain_retry_budget(snapshot, request, origin), false), + } +} + +fn stream_can_retry( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + let attempted = origin.attempted_retries; + if attempted > 0 + && let Some(committed) = origin.committed_budget + { + return committed > attempted; + } + let plain = plain_retry_budget(snapshot, request, origin); + // `_retry_policy_ceiling`: every field the policy dumps. + let ceiling = retry_policy(snapshot, request, origin).map_or(plain, |policy| { + let granted = [ + policy.BadRequestErrorRetries, + policy.AuthenticationErrorRetries, + policy.TimeoutErrorRetries, + policy.RateLimitErrorRetries, + policy.ContentPolicyViolationErrorRetries, + policy.InternalServerErrorRetries, + policy.ServiceUnavailableErrorRetries, + policy.NotFoundErrorRetries, + policy.DefaultRetries, + ]; + plain.max(granted.into_iter().flatten().max().unwrap_or(0)) + }); + ceiling > attempted +} + +fn stream_can_fall_back( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + if request.disable_fallbacks { + return false; + } + if snapshot.settings.enable_weighted_failover { + return true; + } + // `_stream_order_levels`, the levels the fallback walk sees for the group. + let listing_team = request.listing_team_id.as_deref(); + let mut orders: Vec = group_members(snapshot, &origin.group, listing_team) + .iter() + .filter(|d| d.listed_for_team(listing_team)) + .filter_map(|d| d.order()) + .collect(); + orders.sort_unstable(); + orders.dedup(); + if orders.len() > 1 { + let current = origin.order.unwrap_or(orders[0]); + if orders.iter().any(|order| *order > current) { + return true; + } + } + let lists = Lists::of(snapshot, request); + let lookup = origin.lookup(request); + if let Some(chain) = content_policy_targets(&lists, &lookup) + && origin.has_unattempted(request, chain) + { + return true; + } + if !lists.has_regular() && folded_defaults(snapshot, request).is_empty() { + return false; + } + if lists + .fallbacks + .is_some_and(|list| list.is_client_side(lists.keys)) + { + return true; + } + origin.has_unattempted(request, regular_targets(snapshot, request, &lists, &lookup)) +} + +fn refusal_fallback_available( + snapshot: &RouterStateSnapshot, + request: &RouterRequest, + origin: &StreamOrigin, +) -> bool { + if request.disable_fallbacks { + return false; + } + let lists = Lists::of(snapshot, request); + let lookup = origin.lookup(request); + if let Some(chain) = content_policy_targets(&lists, &lookup) { + return origin.has_unattempted(request, chain); + } + if has_default_fallbacks(snapshot) { + return true; + } + origin.has_unattempted(request, regular_targets(snapshot, request, &lists, &lookup)) +} diff --git a/litellm-rust/crates/router-core/src/config.rs b/litellm-rust/crates/router-core/src/config.rs new file mode 100644 index 00000000000..fb9b7af215f --- /dev/null +++ b/litellm-rust/crates/router-core/src/config.rs @@ -0,0 +1,355 @@ +//! What the router is told: [`RouterConfig`], fixed for the router's life, and +//! [`RouterDynamicConfig`], a versioned snapshot read once per call. The router never reads +//! process globals. + +use std::{collections::BTreeMap, time::Duration}; + +use serde::Deserialize; + +use crate::{ + error::AttemptError, + registry::{DeploymentSpec, RoutingGroupSpec}, +}; + +/// `DEFAULT_COOLDOWN_TIME_SECONDS` in `litellm/constants.py`. +pub const DEFAULT_COOLDOWN_TIME: Duration = Duration::from_secs(5); +/// `ROUTER_MAX_FALLBACKS` in `litellm/constants.py`. +pub const DEFAULT_MAX_FALLBACKS: u32 = 5; + +/// The router operation a call runs: the Python router's endpoint method name (`acompletion`, +/// `aembedding`, ...). The core runs every op alike; what differs per endpoint is the host's. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize)] +#[serde(transparent)] +pub struct Op(String); + +impl Op { + pub fn new(name: impl Into) -> Self { + Self(name.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From<&str> for Op { + fn from(name: &str) -> Self { + Self::new(name) + } +} + +impl std::fmt::Display for Op { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// `routing_strategy`. Only simple-shuffle is native; the rest go through `Host::select`. +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + strum::EnumString, + strum::Display, + serde_with::DeserializeFromStr, +)] +pub enum RoutingStrategy { + #[default] + #[strum(serialize = "simple-shuffle")] + SimpleShuffle, + #[strum(serialize = "least-busy")] + LeastBusy, + #[strum(serialize = "usage-based-routing")] + UsageBased, + #[strum(serialize = "usage-based-routing-v2")] + UsageBasedV2, + #[strum(serialize = "latency-based-routing")] + LatencyBased, + #[strum(serialize = "cost-based-routing")] + CostBased, + /// `routing_strategy=None` passed explicitly (sent as `"None"`): stored as None, so no + /// strategy arm matches and every pick finds no deployment. + #[strum(serialize = "None")] + Unset, + #[strum(default, transparent)] + Other(String), +} + +impl RoutingStrategy { + pub fn is_native(&self) -> bool { + matches!(self, Self::SimpleShuffle | Self::Unset) + } +} + +/// `PluginKind` in `litellm/router_api.py`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::Display)] +pub enum PluginKind { + #[serde(rename = "auto")] + #[strum(serialize = "auto")] + Auto, + #[serde(rename = "complexity")] + #[strum(serialize = "complexity")] + Complexity, + #[serde(rename = "quality")] + #[strum(serialize = "quality")] + Quality, + #[serde(rename = "adaptive")] + #[strum(serialize = "adaptive")] + Adaptive, + #[serde(rename = "routing")] + #[strum(serialize = "routing")] + Routing, + #[serde(rename = "budget_limiter")] + #[strum(serialize = "budget_limiter")] + BudgetLimiter, + #[serde(rename = "prompt_caching_pins")] + #[strum(serialize = "prompt_caching_pins")] + PromptCachingPins, +} + +/// One registered plugin. Its behaviour lives in the host until it is ported. +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct PluginSpec { + pub kind: PluginKind, + pub name: String, + #[serde(default)] + pub config: serde_json::Value, +} + +/// `RetryPolicy` in `litellm/types/router.py`: retries per exception type. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[allow(non_snake_case)] +pub struct RetryPolicy { + pub BadRequestErrorRetries: Option, + pub AuthenticationErrorRetries: Option, + pub TimeoutErrorRetries: Option, + pub RateLimitErrorRetries: Option, + pub ContentPolicyViolationErrorRetries: Option, + pub InternalServerErrorRetries: Option, + pub ServiceUnavailableErrorRetries: Option, + pub NotFoundErrorRetries: Option, + pub DefaultRetries: Option, +} + +/// `AllowedFailsPolicy` in `litellm/types/router.py`: allowed fails per minute per exception type. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[allow(non_snake_case)] +pub struct AllowedFailsPolicy { + pub BadRequestErrorAllowedFails: Option, + pub AuthenticationErrorAllowedFails: Option, + pub TimeoutErrorAllowedFails: Option, + pub RateLimitErrorAllowedFails: Option, + pub ContentPolicyViolationErrorAllowedFails: Option, + pub InternalServerErrorAllowedFails: Option, + pub ServiceUnavailableErrorAllowedFails: Option, + pub BadGatewayErrorAllowedFails: Option, + pub NotFoundErrorAllowedFails: Option, +} + +impl AllowedFailsPolicy { + /// `_core_get_allowed_fails_from_policy`: the first set field whose exception class the + /// failure is an instance of, in Python's order. + pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option { + [ + ("AuthenticationError", self.AuthenticationErrorAllowedFails), + ("Timeout", self.TimeoutErrorAllowedFails), + ("RateLimitError", self.RateLimitErrorAllowedFails), + ( + "ContentPolicyViolationError", + self.ContentPolicyViolationErrorAllowedFails, + ), + ("BadRequestError", self.BadRequestErrorAllowedFails), + ("InternalServerError", self.InternalServerErrorAllowedFails), + ( + "ServiceUnavailableError", + self.ServiceUnavailableErrorAllowedFails, + ), + ("BadGatewayError", self.BadGatewayErrorAllowedFails), + ("NotFoundError", self.NotFoundErrorAllowedFails), + ] + .into_iter() + .find_map(|(class, allowed)| allowed.filter(|_| failure.is_a(class))) + } +} + +pub use crate::fallback::{FallbackKeys, Fallbacks}; + +/// Static config, fixed for the router's life. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct RouterConfig { + pub model_list: Vec, + pub routing_strategy: RoutingStrategy, + pub routing_groups: Vec, + /// `_model_to_group` as Python last built it (member model group to routing group), set + /// only when a raw `routing_groups` write replaced the groups since: member calls keep + /// routing by it, as the Python router's stale map does. + pub routing_group_members: Option>, + pub plugins: Vec, + /// Router default `cooldown_time`; `None` is `DEFAULT_COOLDOWN_TIME_SECONDS`. + pub cooldown_time: Option, + pub disable_cooldowns: bool, + pub allowed_fails_policy: Option, + pub retry_policy: Option, + /// `model_group_retry_policy`: a group's own policy, which wins over `retry_policy`. + pub model_group_retry_policy: BTreeMap, + /// `retry_after`: the minimum backoff between retries, in seconds. + pub retry_after: f64, + pub timeout: Option, + pub request_timeout: Option, + pub drop_params: Option, + pub enable_pre_call_checks: bool, + /// `Router(plugins=[...])` is set: the host runs the routing-plugin pipeline. + pub routing_plugins: bool, + /// The host's `CustomLogger` deployment filters run on every async candidate list + /// (`async_callback_filter_deployments`). + pub callback_filters: bool, + pub enable_health_check_routing: bool, + /// How old (seconds) a background health check result may be and still count + /// (`health_check_staleness_threshold`, else the default interval times its multiplier). + pub health_check_staleness_threshold: Option, + /// `background_health_check_model_groups`: when set, only these groups are health-filtered. + pub background_health_check_model_groups: Option>, + /// `enable_weighted_failover`: a failed deployment's group is re-picked without it before + /// any cross-group fallback (simple-shuffle only). + pub enable_weighted_failover: bool, + /// `fallback_on_unsupported_params`; `None` is Python's default, `True`. + pub fallback_on_unsupported_params: Option, + /// Deployment ids that are a provider's default and never cool down + /// (`provider_default_deployment_ids`). + pub provider_default_deployment_ids: Vec, + /// The operations this router must serve: `Host::supports` must accept each one for every + /// configured provider, or the router fails to build. + pub served_ops: Vec, +} + +/// Dynamic config: a versioned snapshot read once per call. +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct RouterDynamicConfig { + pub version: u64, + /// `model_group_alias` as Python spells it: each alias maps to a group name, or to + /// `{"model": , "hidden": }`. + #[serde(alias = "model_group_alias", deserialize_with = "alias_targets")] + pub model_alias_map: BTreeMap, + pub fallbacks: Fallbacks, + pub default_fallbacks: Vec, + pub context_window_fallbacks: Option, + pub content_policy_fallbacks: Option, + pub num_retries: Option, + pub max_fallbacks: u32, + /// `allowed_fails`, as the router copied it when it was built. + pub allowed_fails: Option, + /// `allowed_fails` is only the process default: it was not passed to the router and still + /// equals `litellm.allowed_fails`. Anything else switches cooldowns from the failure-ratio + /// rule to the allowed-fails counter (`_is_allowed_fails_set_on_router`). + pub allowed_fails_is_default: bool, + /// Other spellings a fallback key matches. + pub fallback_keys: FallbackKeys, + /// `default_deployment`: what a name no deployment, alias, routing group, team route or + /// wildcard serves is routed to, as a copy whose `litellm_params.model` is the name + /// (`_try_early_resolve_deployments_for_model_not_in_names`). + pub default_deployment: Option, + /// The `default_deployment` is a value Python cannot copy for a name: resolving to it + /// raises the copy's error ([`crate::Error::DefaultDeploymentRaises`]). + pub default_deployment_raises: bool, +} + +/// `model_group_alias` as `resolve_model_group_alias` reads it: an alias maps to a non-empty +/// group name, or to `{"model": , ...}`; any other value rewrites nothing. +fn alias_targets<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let aliases = BTreeMap::::deserialize(deserializer)?; + Ok(aliases + .into_iter() + .filter_map(|(alias, target)| { + let group = match &target { + serde_json::Value::Object(entry) => entry.get("model")?, + other => other, + }; + group + .as_str() + .filter(|group| !group.is_empty()) + .map(|group| (alias, group.to_owned())) + }) + .collect()) +} + +impl Default for RouterDynamicConfig { + fn default() -> Self { + Self { + version: 0, + model_alias_map: BTreeMap::new(), + fallbacks: Fallbacks::default(), + default_fallbacks: Vec::new(), + context_window_fallbacks: None, + content_policy_fallbacks: None, + num_retries: None, + max_fallbacks: DEFAULT_MAX_FALLBACKS, + allowed_fails: None, + allowed_fails_is_default: false, + fallback_keys: FallbackKeys::default(), + default_deployment: None, + default_deployment_raises: false, + } + } +} + +impl RouterDynamicConfig { + /// The `default_deployment` as resolution serves it. + pub fn default_ref(&self) -> Option> { + if self.default_deployment_raises { + return Some(crate::resolve::DefaultDeployment::Raises); + } + self.default_deployment + .as_ref() + .map(crate::resolve::DefaultDeployment::Spec) + } + + /// The regular fallback targets for the lookup groups, then `default_fallbacks`, which + /// Python folds into `fallbacks` as `{"*": [...]}`: what `_has_fallback_group` reads. + pub fn fallback_targets(&self, groups: &[&str]) -> Option> { + if groups + .iter() + .any(|group| self.fallbacks.has_keyed_chain(group, &self.fallback_keys)) + { + return Some(Vec::new()); + } + (!self.default_fallbacks.is_empty()).then(|| self.default_fallbacks.clone()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::group_name(json!({"model_group_alias": {"gpt": "gpt-4"}}))] + #[case::hidden_entry(json!({"model_group_alias": {"gpt": {"model": "gpt-4", "hidden": true}}}))] + #[case::native_name(json!({"model_alias_map": {"gpt": "gpt-4"}}))] + fn model_group_alias_reads_both_python_spellings(#[case] wire: serde_json::Value) { + let dynamic: RouterDynamicConfig = serde_json::from_value(wire).unwrap(); + assert_eq!( + dynamic.model_alias_map, + BTreeMap::from([("gpt".to_owned(), "gpt-4".to_owned())]) + ); + } + + #[rstest] + #[case::native("simple-shuffle", RoutingStrategy::SimpleShuffle)] + #[case::unknown("my-strategy", RoutingStrategy::Other("my-strategy".to_owned()))] + fn routing_strategy_parses_its_wire_name( + #[case] wire: &str, + #[case] expected: RoutingStrategy, + ) { + assert_eq!(wire.parse::().unwrap(), expected); + assert_eq!(expected.to_string(), wire); + } +} diff --git a/litellm-rust/crates/router-core/src/eligibility.rs b/litellm-rust/crates/router-core/src/eligibility.rs new file mode 100644 index 00000000000..5cedf218b9a --- /dev/null +++ b/litellm-rust/crates/router-core/src/eligibility.rs @@ -0,0 +1,240 @@ +//! Which deployments may serve a request's endpoint: a port of +//! `litellm/router_utils/endpoint_eligibility.py`. A deployment that sets +//! `enforce_supported_endpoints` and lists none of the chat-shaped endpoints serves none of them; +//! the host names the endpoint for the calls `endpoint_selection` wraps. + +use std::{collections::BTreeSet, sync::Arc}; + +use serde_json::{Map, Value}; + +use crate::{error::Error, registry::Deployment, resolve::Caller}; + +/// `CHAT_ENDPOINTS`: `_endpoint_path` of each value of `CHAT_ENDPOINT_BY_CALL`. +const CHAT_ENDPOINTS: [&str; 8] = [ + "/chat/completions", + "/completions", + "/messages", + "/responses", + "/generatecontent", + "/streamgeneratecontent", + "/interactions", + "/responses/compact", +]; + +/// `_endpoint_path`. +fn endpoint_path(endpoint: &str) -> String { + let rooted = format!("/{}", endpoint.trim().trim_matches('/').to_lowercase()); + let path = rooted.strip_prefix("/openai").unwrap_or(&rooted); + let path = path.strip_prefix("/v1beta").unwrap_or(path); + let path = path.strip_prefix("/v1").unwrap_or(path); + match path.rsplit_once(':') { + Some((_, action)) => format!("/{action}"), + None => path.to_owned(), + } +} + +/// The flag as pydantic's lax `bool | None` reads it: `None` for a value it refuses, which fails +/// the whole declaration. +fn enforce_flag(value: Option<&Value>) -> Option { + match value { + None | Some(Value::Null) => Some(false), + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Number(number)) => match number.as_f64() { + Some(0.0) => Some(false), + Some(1.0) => Some(true), + _ => None, + }, + Some(Value::String(text)) => match text.to_lowercase().as_str() { + "0" | "off" | "f" | "false" | "n" | "no" => Some(false), + "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), + _ => None, + }, + Some(_) => None, + } +} + +/// `_enforced_endpoints`: the declared endpoints of a deployment that enforces them. One endpoint +/// may be written as a string; a declaration that does not validate enforces nothing. +fn enforced_endpoints(model_info: &Map) -> Option> { + if !enforce_flag(model_info.get("enforce_supported_endpoints"))? { + return None; + } + match model_info.get("supported_endpoints")? { + Value::String(endpoint) => Some(vec![endpoint.clone()]), + Value::Array(endpoints) => endpoints + .iter() + .map(|endpoint| endpoint.as_str().map(str::to_owned)) + .collect(), + _ => None, + } +} + +/// `non_chat_endpoints`: what a deployment serves when it serves no chat-shaped endpoint, else +/// `None`. +fn non_chat_endpoints(deployment: &Deployment) -> Option> { + let declared = enforced_endpoints(&deployment.model_info.raw)?; + let chat = declared + .iter() + .any(|endpoint| CHAT_ENDPOINTS.contains(&endpoint_path(endpoint).as_str())); + (!chat).then(|| declared.into_iter().collect()) +} + +/// `eligible_deployments`: the deployments that may serve the caller's endpoint. When none may, +/// the refusal names what they do serve. +pub(crate) fn eligible_deployments( + model: &str, + caller: Caller<'_>, + deployments: Vec>, +) -> Result>, Error> { + let Some(endpoint) = caller.endpoint.filter(|_| !deployments.is_empty()) else { + return Ok(deployments); + }; + let declared: Vec>> = deployments + .iter() + .map(|deployment| non_chat_endpoints(deployment)) + .collect(); + let eligible: Vec> = deployments + .into_iter() + .zip(&declared) + .filter(|(_, endpoints)| endpoints.is_none()) + .map(|(deployment, _)| deployment) + .collect(); + if !eligible.is_empty() { + return Ok(eligible); + } + let supported: BTreeSet = declared.into_iter().flatten().flatten().collect(); + let supported = if supported.is_empty() { + "none".to_owned() + } else { + Vec::from_iter(supported).join(", ") + }; + let requested = if caller.requested.is_empty() { + model + } else { + caller.requested + }; + Err(Error::EndpointNotSupported { + model: model.to_owned(), + message: format!( + "{endpoint} is not supported by model={requested}. Supported endpoints: {supported}" + ), + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, info: Value) -> Arc { + let mut model_info = info; + model_info["id"] = json!(id); + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": model_info, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + fn chat<'a>() -> Caller<'a> { + Caller { + endpoint: Some("/v1/chat/completions"), + requested: "g", + ..Caller::default() + } + } + + #[rstest] + #[case::v1_prefix("/v1/chat/completions", "/chat/completions")] + #[case::openai_prefix("/openai/v1/responses/", "/responses")] + #[case::bare_gemini_action("generateContent", "/generatecontent")] + #[case::beta_prefix("/v1beta/interactions", "/interactions")] + #[case::gemini_model_path( + "/v1beta/models/gemini:streamGenerateContent", + "/streamgeneratecontent" + )] + fn endpoint_paths_normalize(#[case] endpoint: &str, #[case] expected: &str) { + assert_eq!(endpoint_path(endpoint), expected); + } + + #[rstest] + #[case::embeddings_only( + json!({"enforce_supported_endpoints": true, "supported_endpoints": ["/v1/embeddings"]}), + false + )] + #[case::one_endpoint_as_a_string( + json!({"enforce_supported_endpoints": true, "supported_endpoints": "/v1/embeddings"}), + false + )] + #[case::string_flag( + json!({"enforce_supported_endpoints": "yes", "supported_endpoints": ["/v1/embeddings"]}), + false + )] + #[case::an_empty_list_serves_nothing( + json!({"enforce_supported_endpoints": true, "supported_endpoints": []}), + false + )] + #[case::any_chat_shaped_endpoint_serves_every_one( + json!({"enforce_supported_endpoints": true, "supported_endpoints": ["/v1/responses"]}), + true + )] + #[case::not_enforced( + json!({"enforce_supported_endpoints": false, "supported_endpoints": ["/v1/embeddings"]}), + true + )] + #[case::unset_flag(json!({"supported_endpoints": ["/v1/embeddings"]}), true)] + #[case::enforced_without_a_list(json!({"enforce_supported_endpoints": true}), true)] + #[case::an_invalid_flag_enforces_nothing( + json!({"enforce_supported_endpoints": "maybe", "supported_endpoints": ["/v1/embeddings"]}), + true + )] + #[case::an_invalid_list_enforces_nothing( + json!({"enforce_supported_endpoints": true, "supported_endpoints": [1]}), + true + )] + fn a_declaration_keeps_or_drops_a_deployment(#[case] info: Value, #[case] kept: bool) { + let open = deployment("open", json!({})); + let declared = deployment("declared", info); + let eligible = eligible_deployments("g", chat(), vec![declared, open]).unwrap(); + assert_eq!(eligible.len(), if kept { 2 } else { 1 }); + } + + #[test] + fn no_endpoint_keeps_everything() { + let declared = deployment( + "d", + json!({"enforce_supported_endpoints": true, "supported_endpoints": []}), + ); + let eligible = eligible_deployments("g", Caller::default(), vec![declared]).unwrap(); + assert_eq!(eligible.len(), 1); + } + + #[rstest] + #[case::lists_what_they_serve( + json!(["/v1/rerank", "/v1/embeddings"]), + "/v1/chat/completions is not supported by model=g. \ + Supported endpoints: /v1/embeddings, /v1/rerank" + )] + #[case::nothing_declared( + json!([]), + "/v1/chat/completions is not supported by model=g. Supported endpoints: none" + )] + fn every_deployment_refusing_names_the_endpoint( + #[case] endpoints: Value, + #[case] expected: &str, + ) { + let declared = deployment( + "d", + json!({"enforce_supported_endpoints": true, "supported_endpoints": endpoints}), + ); + let refused = eligible_deployments("resolved", chat(), vec![declared]); + assert!(matches!( + refused, + Err(Error::EndpointNotSupported { model, message }) + if model == "resolved" && message == expected + )); + } +} diff --git a/litellm-rust/crates/router-core/src/error.rs b/litellm-rust/crates/router-core/src/error.rs new file mode 100644 index 00000000000..b4112501c36 --- /dev/null +++ b/litellm-rust/crates/router-core/src/error.rs @@ -0,0 +1,473 @@ +//! The router's errors: [`ConfigError`] when a router is built or changed, [`Error`] when a call +//! or a selection fails, and [`AttemptError`], the typed facts of one failed attempt. + +use std::{any::Any, fmt, sync::Arc, time::Duration}; + +use crate::config::{Op, RoutingStrategy}; + +/// What a host cannot serve, found when the router is built rather than partway through a call. +#[derive(Clone, Debug, PartialEq, thiserror::Error)] +pub enum Unsupported { + #[error("routing strategy `{0}` needs a host that can select")] + Strategy(RoutingStrategy), + #[error("routing plugin `{0}` needs a host that can select")] + Plugin(String), + #[error("the host cannot run operation `{op}` on provider `{provider}`")] + Op { op: Op, provider: String }, +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("the router config asks for something this host cannot serve")] + Unsupported(#[from] Unsupported), + #[error("deployment `{model_name}` has invalid litellm_params")] + InvalidDeployment { + model_name: String, + #[source] + source: serde_json::Error, + }, + #[error("deployment id `{id}` is configured more than once")] + DuplicateId { id: String }, + /// A removal the host's indexes could not finish ([`crate::host_index::RemovalError`]). + #[error("{0}")] + Removal(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("the router config is invalid")] + Config(#[from] ConfigError), + #[error("no deployment serves model `{model}`")] + UnknownModel { model: String }, + #[error("no deployment of `{model}` is available")] + NoHealthyDeployments { model: String }, + #[error("the attempt failed")] + Attempt(#[from] AttemptError), + #[error("the routing cache failed")] + Cache(#[from] litellm_cache::Error), + #[error("the router was retired")] + Retired, + /// A blocked deployment named by its id (`model` is its litellm model). + #[error("deployment model `{model}` is paused")] + Paused { model: String }, + /// A pass-through pick named a deployment id without `use_in_pass_through`. + #[error("deployment `{id}` does not support pass-through")] + NotPassThrough { id: String }, + /// A pass-through pick of a group with healthy deployments, none of them pass-through. + #[error("model `{model}` has no pass-through deployment")] + NoPassThroughDeployments { model: String }, + /// Every deployment `model` resolved to is a pre-routing strategy marker (`auto_router/...`). + #[error("every deployment of `{model}` is a strategy router marker")] + OnlyMarkers { model: String }, + /// Every deployment `model` resolved to refuses the request's endpoint + /// (`eligible_deployments`); `message` names what they serve. + #[error("{message}")] + EndpointNotSupported { model: String, message: String }, + /// Every deployment `model` resolved to is split into PTU shares the caller holds none of. + #[error("deployment `{model}` is reserved for the teams holding a PTU share of it")] + PtuReserved { model: String }, + /// A proxy admin without a team named a public model name several teams use. + #[error("model name `{model}` matches deployments from multiple teams")] + MultipleTeams { model: String }, + /// Every deployment `model` resolved to is reserved for other teams by an active access + /// window, the first of which ends at `until` (`HH:MM`) in `timezone`. + #[error("deployment `{model}` is reserved for another team until {until} {timezone}")] + Reserved { + model: String, + until: String, + timezone: String, + }, + /// The pre-call checks found no deployment whose context window fits the prompt; `detail` + /// names each one dropped for it. + #[error("no deployment of `{model}` has a context window large enough for this call")] + ContextWindowExceeded { model: String, detail: String }, + /// The pre-call checks found no deployment of `model` that takes the requested `params`. + #[error("no deployment of `{model}` supports {params:?}")] + UnsupportedParams { model: String, params: Vec }, + /// The routing plugins left none of the healthy deployments' models. + #[error("no deployments left after routing-plugin filtering: {models:?}")] + RoutingPluginsLeftNothing { models: Vec }, + /// The request's `_router_weights` failed `validate_router_weights` when simple-shuffle picked. + #[error("the request's router weights are invalid")] + InvalidRouterWeights, + /// A deployment's `litellm_params` `metric` (weight, rpm or tpm) is not a number. + #[error("deployment {metric} must be numeric")] + NonNumericWeight { metric: &'static str }, + /// A router with routing plugins was asked for a pick on the sync path, which never runs them. + #[error("routing plugins only run on the async selection path")] + RoutingPluginsNeedAsync, + /// The stale routing-group member map (`_model_to_group`, kept as it was when a raw + /// `routing_groups` write replaced the groups) names a group that is no longer configured: + /// Python's `_get_routing_context` raises `KeyError(group)` for every pick of the name. + #[error("routing group `{group}` is no longer configured")] + MissingRoutingGroup { group: String }, + /// The router's `default_deployment` is not a value Python can copy for `model` + /// (`_try_early_resolve_deployments_for_model_not_in_names` raises the copy's error). + #[error("the default deployment cannot be served for `{model}`")] + DefaultDeploymentRaises { model: String }, + /// A written model-name or team index names a position past the end of the model list: + /// Python's lookup of `model` raises `IndexError`. + #[error("a written index of `{model}` names a position past the model list")] + IndexOutOfRange { model: String }, + /// A model-info query raised where the Python router raises: `exception` names the Python + /// exception class (`KeyError`, `TypeError`, `AttributeError`, `ValueError`) and `detail` its + /// argument. + #[error("the model-info query raised {exception}({detail:?})")] + ModelInfo { + exception: &'static str, + detail: String, + }, +} + +impl Error { + /// The failed attempt behind this error, when a deployment was called. + pub fn attempt(&self) -> Option<&AttemptError> { + match self { + Self::Attempt(attempt) => Some(attempt), + Self::Config(_) + | Self::UnknownModel { .. } + | Self::NoHealthyDeployments { .. } + | Self::Cache(_) + | Self::Retired + | Self::Paused { .. } + | Self::NotPassThrough { .. } + | Self::NoPassThroughDeployments { .. } + | Self::OnlyMarkers { .. } + | Self::EndpointNotSupported { .. } + | Self::PtuReserved { .. } + | Self::MultipleTeams { .. } + | Self::Reserved { .. } + | Self::ContextWindowExceeded { .. } + | Self::UnsupportedParams { .. } + | Self::RoutingPluginsLeftNothing { .. } + | Self::InvalidRouterWeights + | Self::NonNumericWeight { .. } + | Self::RoutingPluginsNeedAsync + | Self::MissingRoutingGroup { .. } + | Self::DefaultDeploymentRaises { .. } + | Self::IndexOutOfRange { .. } + | Self::ModelInfo { .. } => None, + } + } + + /// The Python exception a pick that failed with this error raises, as the retry loop + /// decides by it (its class names, most specific first, and its status): `None` for an + /// attempt's failure and for errors no pick raises. + pub fn pick_failure(&self) -> Option { + const BAD_REQUEST: &[&str] = &["BadRequestError"]; + let (classes, status): (&[&str], Option) = match self { + Self::UnknownModel { .. } + | Self::OnlyMarkers { .. } + | Self::PtuReserved { .. } + | Self::NotPassThrough { .. } + | Self::NoPassThroughDeployments { .. } + | Self::MultipleTeams { .. } + | Self::Reserved { .. } => (BAD_REQUEST, Some(400)), + Self::EndpointNotSupported { .. } => ( + &["EndpointNotSupportedByModelError", "BadRequestError"], + Some(400), + ), + Self::ContextWindowExceeded { .. } => ( + &["ContextWindowExceededError", "BadRequestError"], + Some(400), + ), + Self::UnsupportedParams { .. } => { + (&["UnsupportedParamsError", "BadRequestError"], Some(400)) + } + Self::Paused { .. } => (&["ServiceUnavailableError"], Some(503)), + Self::NoHealthyDeployments { .. } => (&["RouterRateLimitError", "ValueError"], None), + Self::RoutingPluginsLeftNothing { .. } | Self::RoutingPluginsNeedAsync => { + (&["ValueError"], None) + } + Self::InvalidRouterWeights => (&["ValidationError", "ValueError"], None), + Self::NonNumericWeight { .. } => (&["TypeError"], None), + Self::MissingRoutingGroup { .. } => (&["KeyError", "LookupError"], None), + Self::IndexOutOfRange { .. } => (&["IndexError", "LookupError"], None), + Self::DefaultDeploymentRaises { .. } => (&["Exception"], None), + Self::Attempt(_) + | Self::Config(_) + | Self::Cache(_) + | Self::Retired + | Self::ModelInfo { .. } => return None, + }; + let mut failure = AttemptError::host_raised(classes); + failure.status = status; + Some(failure) + } + + /// A copy of a pick's error, for the events that name it; `None` for any other error. + pub fn pick_copy(&self) -> Option { + Some(match self { + Self::UnknownModel { model } => Self::UnknownModel { + model: model.clone(), + }, + Self::NoHealthyDeployments { model } => Self::NoHealthyDeployments { + model: model.clone(), + }, + Self::Paused { model } => Self::Paused { + model: model.clone(), + }, + Self::NotPassThrough { id } => Self::NotPassThrough { id: id.clone() }, + Self::NoPassThroughDeployments { model } => Self::NoPassThroughDeployments { + model: model.clone(), + }, + Self::OnlyMarkers { model } => Self::OnlyMarkers { + model: model.clone(), + }, + Self::EndpointNotSupported { model, message } => Self::EndpointNotSupported { + model: model.clone(), + message: message.clone(), + }, + Self::PtuReserved { model } => Self::PtuReserved { + model: model.clone(), + }, + Self::MultipleTeams { model } => Self::MultipleTeams { + model: model.clone(), + }, + Self::Reserved { + model, + until, + timezone, + } => Self::Reserved { + model: model.clone(), + until: until.clone(), + timezone: timezone.clone(), + }, + Self::ContextWindowExceeded { model, detail } => Self::ContextWindowExceeded { + model: model.clone(), + detail: detail.clone(), + }, + Self::UnsupportedParams { model, params } => Self::UnsupportedParams { + model: model.clone(), + params: params.clone(), + }, + Self::RoutingPluginsLeftNothing { models } => Self::RoutingPluginsLeftNothing { + models: models.clone(), + }, + Self::InvalidRouterWeights => Self::InvalidRouterWeights, + Self::NonNumericWeight { metric } => Self::NonNumericWeight { metric }, + Self::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync, + Self::MissingRoutingGroup { group } => Self::MissingRoutingGroup { + group: group.clone(), + }, + Self::DefaultDeploymentRaises { model } => Self::DefaultDeploymentRaises { + model: model.clone(), + }, + Self::IndexOutOfRange { model } => Self::IndexOutOfRange { + model: model.clone(), + }, + Self::Attempt(_) + | Self::Config(_) + | Self::Cache(_) + | Self::Retired + | Self::ModelInfo { .. } => return None, + }) + } +} + +/// The class of a failed attempt, as litellm's exception mapping names it. +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + strum::Display, + strum::EnumString, + strum::IntoStaticStr, + strum::VariantArray, +)] +pub enum FailureKind { + #[strum(serialize = "RateLimitError")] + RateLimit, + #[strum(serialize = "ContextWindowExceededError")] + ContextWindowExceeded, + #[strum(serialize = "ContentPolicyViolationError")] + ContentPolicyViolation, + #[strum(serialize = "AuthenticationError")] + Authentication, + #[strum(serialize = "PermissionDeniedError")] + PermissionDenied, + #[strum(serialize = "NotFoundError")] + NotFound, + #[strum(serialize = "BadRequestError")] + BadRequest, + #[strum(serialize = "Timeout")] + Timeout, + #[strum(serialize = "APIConnectionError")] + ApiConnection, + #[strum(serialize = "InternalServerError")] + InternalServer, + #[strum(serialize = "ServiceUnavailableError")] + ServiceUnavailable, + #[strum(serialize = "BadGatewayError")] + BadGateway, + /// A guardrail blocked the request: never retried, never a fallback. + #[strum(serialize = "GuardrailIntervention")] + GuardrailIntervention, + /// litellm rejected the request before calling the deployment (`MalformedRequestFieldError`, + /// `UnsupportedEndpointError`): never cools the deployment down. + #[strum(serialize = "Declined")] + Declined, + /// The caller's own stored credential was rejected, not the deployment's + /// (`CallerCredentialAuthenticationError`): never cools the deployment down. + #[strum(serialize = "CallerCredentialAuthenticationError")] + CallerCredentialAuthentication, + /// A per-user credential exchange was rate limited (`CallerCredentialRateLimitError`): never + /// cools the deployment down. + #[strum(serialize = "CallerCredentialRateLimitError")] + CallerCredentialRateLimit, + #[strum(serialize = "APIError")] + Other, +} + +/// An opaque value the host hands through the router unchanged, such as the original Python +/// exception or response object. +#[derive(Clone)] +pub struct Payload(pub Arc); + +impl fmt::Debug for Payload { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("Payload(..)") + } +} + +/// One failed attempt, as typed facts the retry, fallback and cooldown rules decide by. +#[derive(Clone, Debug, thiserror::Error)] +#[error("the deployment call failed with {kind}")] +pub struct AttemptError { + pub kind: FailureKind, + pub status: Option, + /// The provider's `retry-after` header (`_get_retry_after_from_exception_header`). + pub retry_after: Option, + /// A deployment-level `num_retries` the failure carries (Python's `exception.num_retries`). + pub num_retries: Option, + /// The deployment the exception itself names (`_deployment_ids_to_skip_on_retry` reads its + /// `retry_skip_deployment_id`, then `failed_deployment_id`): an exception raised again keeps + /// the id its first failure stamped, whichever deployment raised it now. + pub failed_deployment_id: Option, + /// The provider's error text, kept only for the cooldown record Python writes + /// (`exception_received`, masked past 50 characters). + pub detail: Option>, + /// The exception's class names, most specific first (its `__mro__`), for the rules Python + /// decides by `isinstance`. Empty when the host reports none; the kind then stands in. + pub classes: Vec>, + /// The host already recorded what this failure does to the deployment's health (a routing + /// pre-call check's failure, which Python's attempt handles itself: a rate limit cools the + /// deployment down, anything else records nothing), or no failure callback runs for it (it + /// was raised outside the litellm call, or a shared logging object already logged a failure), + /// so the core records nothing. + pub recorded: bool, + /// The exception names its provider (`llm_provider` is set), which marks a provider's own + /// 400 for `fallback_on_unsupported_params`. + pub names_provider: bool, + /// An agentic follow-up call failed after the first call was served + /// (`is_agentic_followup_failure`): never retried, since a retry reruns every paid round. The + /// failure keeps its own class, which decides its fallbacks and cooldown as for any failure. + pub agentic_followup: bool, + /// One of `deployment_callback_on_failure`'s early exits holds: the failure says nothing about + /// the deployment, so it is neither counted nor cooled down (a failed request still counts + /// against the minute's requests, which `async_deployment_callback_on_failure` does apart). + pub outside_deployment: bool, + /// One of `_should_run_cooldown_logic`'s per-caller exits holds (`_is_per_user_provider_request` + /// or a per-user Copilot deployment): the failure is counted but never cools the deployment. + pub caller_scoped: bool, + /// The exception has no `status_code` (Python's status `""`): `_is_cooldown_required` holds + /// no cooldown for it, unlike a status the host could not read, which cools down as a 500. + pub status_absent: bool, + /// The failure counts against no per-minute requests: `async_deployment_callback_on_failure` + /// does not run for a sync call, and returns at once for a batch retrieve. + pub uncounted_request: bool, + pub payload: Option, +} + +impl AttemptError { + pub fn new(kind: FailureKind, status: Option) -> Self { + Self { + kind, + status, + retry_after: None, + num_retries: None, + failed_deployment_id: None, + detail: None, + classes: Vec::new(), + recorded: false, + names_provider: false, + agentic_followup: false, + outside_deployment: false, + caller_scoped: false, + status_absent: false, + uncounted_request: false, + payload: None, + } + } + + /// The exception's class names, most specific first: the host's, else the kind's. + pub fn class_names(&self) -> Vec<&str> { + if self.classes.is_empty() { + return crate::retry::kind_classes(self.kind).to_vec(); + } + self.classes.iter().map(AsRef::as_ref).collect() + } + + /// `isinstance(error, )`. + pub fn is_a(&self, name: &str) -> bool { + self.class_names().contains(&name) + } + + #[must_use] + pub fn with_retry_after(self, retry_after: Duration) -> Self { + Self { + retry_after: Some(retry_after), + ..self + } + } + + /// A plain Python exception the host raised outside any attempt (a `KeyError` from a stale + /// routing-group lookup, an error a retry policy lookup raised): no status, no provider, + /// nothing recorded against a deployment. `classes` are its class names, most specific first. + pub fn host_raised(classes: &[&str]) -> Self { + Self { + classes: classes.iter().map(|class| (*class).into()).collect(), + recorded: true, + ..Self::new(FailureKind::Other, None) + } + } + + /// The status Python's cooldown rules read: the provider's, else 500. + pub fn cooldown_status(&self) -> u16 { + self.status.unwrap_or(500) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::rate_limit("RateLimitError", FailureKind::RateLimit)] + #[case::context_window("ContextWindowExceededError", FailureKind::ContextWindowExceeded)] + #[case::caller_credential_auth( + "CallerCredentialAuthenticationError", + FailureKind::CallerCredentialAuthentication + )] + #[case::caller_credential_rate_limit( + "CallerCredentialRateLimitError", + FailureKind::CallerCredentialRateLimit + )] + #[case::other("APIError", FailureKind::Other)] + fn failure_kind_round_trips_its_exception_name(#[case] wire: &str, #[case] kind: FailureKind) { + assert_eq!(wire.parse::().unwrap(), kind); + assert_eq!(<&'static str>::from(kind), wire); + } + + #[rstest] + #[case::empty("")] + #[case::lowercase("ratelimiterror")] + fn failure_kind_rejects_unknown_names(#[case] wire: &str) { + assert!(wire.parse::().is_err()); + } +} diff --git a/litellm-rust/crates/router-core/src/fallback.rs b/litellm-rust/crates/router-core/src/fallback.rs new file mode 100644 index 00000000000..a1755bd6ad5 --- /dev/null +++ b/litellm-rust/crates/router-core/src/fallback.rs @@ -0,0 +1,330 @@ +//! Fallback lists as Python spells them, and how a failed group's chain is found in one. +//! Mirrors `get_fallback_model_group`, `get_fallback_model_group_for_lookup_groups`, +//! `check_non_standard_fallback_format` and `fallback_attempt_key` in +//! `router_utils/fallback_event_handlers.py`. +//! +//! A list holds keyed entries (`{"gpt-4": ["gpt-3.5", {"model": "x", "temperature": 0}]}`), +//! bare names (`"claude"`, a chain for every group) and, in a client-side list, dict targets +//! (`{"model": "x", "messages": [...]}`), which override request params for their attempt. + +use serde::Deserialize; +use serde_json::{Map, Value}; + +use std::collections::BTreeMap; + +/// One item of a fallback list, kept as Python holds it. +#[derive(Clone, Debug, PartialEq)] +enum Entry { + Name(String), + Dict(Map), + /// Anything else: Python skips it when it looks a chain up. + Other, +} + +/// A fallback list, in order. `"*"` keys the default chain. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(from = "Vec")] +pub struct Fallbacks(Vec); + +impl From> for Fallbacks { + fn from(entries: Vec) -> Self { + Self( + entries + .into_iter() + .map(|entry| match entry { + Value::String(name) => Entry::Name(name), + Value::Object(dict) => Entry::Dict(dict), + _ => Entry::Other, + }) + .collect(), + ) + } +} + +/// One target of a chain: a group name, or a dict target that also overrides request params. +#[derive(Clone, Debug, PartialEq)] +pub enum FallbackTarget { + Name(String), + Params(Map), +} + +impl FallbackTarget { + fn of(value: &Value) -> Option { + match value { + Value::String(name) => Some(Self::Name(name.clone())), + Value::Object(dict) => Some(Self::Params(dict.clone())), + _ => None, + } + } + + /// `_get_fallback_target_model_group`: the group the target names, if any. + pub fn model(&self) -> Option<&str> { + match self { + Self::Name(name) => Some(name), + Self::Params(dict) => dict.get("model").and_then(Value::as_str), + } + } + + /// The `model` of a dict target that sets a non-string one: the hop routes that value, which + /// no group answers (`_get_fallback_target_model_group` finds no group to check, but + /// `kwargs.update(target)` still rebinds the request's model). + pub fn unroutable_model(&self) -> Option<&Value> { + match self { + Self::Params(dict) => dict.get("model").filter(|model| !model.is_string()), + Self::Name(_) => None, + } + } + + /// `fallback_attempt_key`: a bare name and `{"model": name}` are the same attempt; a dict + /// carrying anything else is its own. + pub fn identity(&self) -> FallbackTarget { + match self { + Self::Params(dict) if dict.len() == 1 => match dict.get("model") { + Some(Value::String(name)) => Self::Name(name.clone()), + _ => self.clone(), + }, + _ => self.clone(), + } + } + + /// The params a dict target merges into the request (`kwargs.update(target)`). + pub fn params(&self) -> Option<&Map> { + match self { + Self::Params(dict) => Some(dict), + Self::Name(_) => None, + } + } +} + +/// What fallback keys match besides the exact group name: Python's `provider_list`, for +/// `"openai/gpt-4o"` to use a `"gpt-4o"` entry, the provider-prefixed spelling of each bare +/// group name (`_provider_prefixed_model_group`), for `"gpt-4o"` to use an `"openai/gpt-4o"` +/// entry, and the `LiteLLMParamsTypedDict` keys that mark a client-side dict target. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default)] +pub struct FallbackKeys { + pub providers: Vec, + pub provider_prefixed: BTreeMap, + pub param_keys: Vec, +} + +impl FallbackKeys { + /// `_check_stripped_model_group`. + fn is_stripped(&self, group: &str, key: &str) -> bool { + self.providers.iter().any(|provider| { + let prefix = format!("{provider}/"); + group.starts_with(&prefix) && group.replace(&prefix, "") == key + }) + } +} + +/// A chain found for a group, and whether it is the `"*"` chain itself. +struct Found { + targets: Vec, + generic: bool, +} + +/// The targets of a keyed entry's value: a list of names and dicts. A bare string is walked +/// character by character, as Python iterates it. +fn chain_of(value: &Value) -> Vec { + match value { + Value::Array(items) => items.iter().filter_map(FallbackTarget::of).collect(), + Value::String(text) => text + .chars() + .map(|c| FallbackTarget::Name(c.to_string())) + .collect(), + _ => Vec::new(), + } +} + +impl Fallbacks { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// `check_non_standard_fallback_format`: a client-side list of names, or of dicts that set + /// a litellm param, is one chain for every group. + pub fn is_client_side(&self, keys: &FallbackKeys) -> bool { + if self.0.is_empty() { + return false; + } + if self.0.iter().all(|entry| matches!(entry, Entry::Name(_))) { + return true; + } + let dicts: Vec<_> = self + .0 + .iter() + .filter_map(|entry| match entry { + Entry::Dict(dict) => Some(dict), + _ => None, + }) + .collect(); + dicts.len() == self.0.len() + && dicts.iter().any(|dict| { + keys.param_keys + .iter() + .any(|key| dict.get(key).is_some_and(|value| !value.is_array())) + }) + } + + /// The whole list as one chain (a client-side list). + pub fn as_chain(&self) -> Vec { + self.0 + .iter() + .filter_map(|entry| match entry { + Entry::Name(name) => Some(FallbackTarget::Name(name.clone())), + Entry::Dict(dict) => Some(FallbackTarget::Params(dict.clone())), + Entry::Other => None, + }) + .collect() + } + + /// `_get_first_default_fallback`: the first target of the first `"*"` entry. + pub fn default_target(&self) -> Option<&str> { + self.0.iter().find_map(|entry| match entry { + Entry::Dict(dict) => match dict.get("*") { + Some(Value::Array(targets)) => targets.first().and_then(Value::as_str), + _ => None, + }, + _ => None, + }) + } + + /// `_has_default_fallbacks`: some entry keys `"*"`. + pub fn has_star(&self) -> bool { + self.0 + .iter() + .any(|entry| matches!(entry, Entry::Dict(dict) if dict.contains_key("*"))) + } + + /// `get_fallback_model_group`. + fn lookup(&self, group: &str, keys: &FallbackKeys) -> Option { + let prefixed = keys.provider_prefixed.get(group).map(String::as_str); + let mut generic = None; + let mut stripped = None; + let mut found = None; + for entry in &self.0 { + match entry { + Entry::Dict(dict) => { + let Some((key, value)) = dict.iter().next() else { + continue; + }; + if key == group { + found = Some(chain_of(value)); + break; + } else if Some(key.as_str()) == prefixed || keys.is_stripped(group, key) { + stripped = Some(chain_of(value)); + } else if key == "*" { + generic = Some(value); + } + } + Entry::Name(name) => found = Some(vec![FallbackTarget::Name(name.clone())]), + Entry::Other => {} + } + } + found + .or(stripped) + .map(|targets| Found { + targets, + generic: false, + }) + .or_else(|| { + generic.map(|value| Found { + targets: chain_of(value), + generic: true, + }) + }) + } + + /// `get_fallback_model_group_for_lookup_groups`: the first lookup group with its own chain + /// wins; `"*"` applies only after every group missed. + pub fn lookup_groups( + &self, + groups: &[&str], + keys: &FallbackKeys, + ) -> Option> { + let results: Vec = groups + .iter() + .filter_map(|group| self.lookup(group, keys)) + .collect(); + let index = results + .iter() + .position(|found| !found.generic) + .or_else(|| (!results.is_empty()).then_some(0))?; + results.into_iter().nth(index).map(|found| found.targets) + } + + /// `_has_fallback_group`'s view: only keyed entries whose targets are names or dicts count. + pub fn has_keyed_chain(&self, group: &str, keys: &FallbackKeys) -> bool { + let keyed = Fallbacks( + self.0 + .iter() + .filter(|entry| matches!(entry, Entry::Dict(_))) + .cloned() + .collect(), + ); + keyed + .lookup(group, keys) + .is_some_and(|found| !found.targets.is_empty()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::an_int(json!({"model": 5}), true)] + #[case::none(json!({"model": null}), true)] + #[case::a_string(json!({"model": "b"}), false)] + #[case::no_model(json!({"temperature": 0}), false)] + fn only_a_set_non_string_model_is_unroutable(#[case] target: Value, #[case] unroutable: bool) { + let target = FallbackTarget::of(&target).unwrap(); + assert_eq!(target.unroutable_model().is_some(), unroutable); + } + + fn names(targets: Option>) -> Option> { + targets.map(|targets| { + targets + .iter() + .map(|t| t.model().unwrap_or_default().to_owned()) + .collect() + }) + } + + #[rstest] + #[case::own_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "a", Some(vec!["b"]))] + #[case::star_entry(json!([{"a": ["b"]}, {"*": ["d"]}]), "x", Some(vec!["d"]))] + #[case::string_is_every_groups_chain(json!(["b", {"x": ["c"]}]), "a", Some(vec!["b"]))] + #[case::keyed_entry_wins(json!(["c", {"a": ["b"]}]), "a", Some(vec!["b"]))] + #[case::string_value_is_iterated(json!([{"a": "bc"}]), "a", Some(vec!["b", "c"]))] + #[case::none(json!([{"x": ["c"]}]), "a", None)] + fn chains_follow_python( + #[case] list: Value, + #[case] group: &str, + #[case] expected: Option>, + ) { + let fallbacks: Fallbacks = serde_json::from_value(list).unwrap(); + assert_eq!( + names(fallbacks.lookup_groups(&[group], &FallbackKeys::default())), + expected.map(|e| e.into_iter().map(str::to_owned).collect()) + ); + } + + #[rstest] + #[case::names(json!(["a", "b"]), true)] + #[case::param_dicts(json!([{"model": "a", "temperature": 1}]), true)] + #[case::keyed(json!([{"a": ["b"]}]), false)] + #[case::mixed(json!(["a", {"a": ["b"]}]), false)] + fn client_side_lists_follow_python(#[case] list: Value, #[case] expected: bool) { + let fallbacks: Fallbacks = serde_json::from_value(list).unwrap(); + let keys = FallbackKeys { + param_keys: vec!["model".to_owned(), "temperature".to_owned()], + ..FallbackKeys::default() + }; + assert_eq!(fallbacks.is_client_side(&keys), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/group_info.rs b/litellm-rust/crates/router-core/src/group_info.rs new file mode 100644 index 00000000000..9a49858ca18 --- /dev/null +++ b/litellm-rust/crates/router-core/src/group_info.rs @@ -0,0 +1,646 @@ +//! Model group info and the configured-value reads: what the router reports a model group, or a +//! concrete deployment behind a listed name, can do. +//! +//! The rules are the Python router core's (`_set_model_group_info`, `get_model_listing_info`, +//! `get_configured_token_limits`, `get_configured_mode`, `get_configured_display_name`, +//! `get_configured_service_tiers`). The host picks the deployments (the registry lookups) and +//! answers what needs its objects as [`Host::model_info`] lookups asked while the query runs: a +//! deployment's validated params, its merged model info and provider, its capabilities, and the +//! group's first entry as its model validates it. The core decides which deployments count and +//! how their values combine. + +use serde_json::{Map, Number, Value}; + +use crate::{ + error::Error, + host::{Host, ModelInfoLookup}, + model_info::{ + discovered, is_opaque, lookup, malformed, python_float, python_int, raised, truthy, + without_digit_underscores, + }, +}; + +/// One `_set_model_group_info` query. +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(default)] +pub struct GroupInfoQuery { + /// The model group the deployments serve (an alias already resolved). + pub model_group: String, + /// The name the group is reported as. + pub user_facing_model_group_name: String, + /// The group is a team's public model name the team's deployments serve. + pub serves_team_deployments: bool, + /// A wildcard pattern routes `model_group` (asked only when an entry needs it). + pub pattern_routes: bool, + /// The first string `model_info.description` among the deployments. + pub description: Value, + /// The deployments the group routes to, as stored. + pub deployments: Vec, +} + +/// The group info: `None` when no deployment matched, else the fields the merge assigned after +/// the host built the group from its first deployment ([`ModelInfoLookup::GroupSeed`]). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct GroupInfo { + pub assignments: Map, +} + +/// The type name Python reports for a value, in its error messages. +pub(crate) fn type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) if is_opaque(value) => "object", + Value::Object(_) => "dict", + } +} + +/// A number as Python's arithmetic reads it (a bool is 0 or 1), or `None`. +fn arithmetic(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(Number::from(u8::from(*flag))), + Value::Number(number) => Some(number.clone()), + _ => None, + } +} + +/// `total += value`, with `total` starting at `0`; adding a non-number raises `TypeError`. +fn add(total: Option, value: &Value) -> Result { + let total = total.unwrap_or_else(|| Value::Number(Number::from(0))); + let (Some(left), Some(right)) = (arithmetic(&total), arithmetic(value)) else { + return Err(raised( + "TypeError", + format!( + "unsupported operand type(s) for +=: '{}' and '{}'", + type_name(&total), + type_name(value) + ), + )); + }; + if let (Some(left), Some(right)) = (left.as_i64(), right.as_i64()) + && let Some(sum) = left.checked_add(right) + { + return Ok(Value::Number(Number::from(sum))); + } + let sum = left.as_f64().unwrap_or(0.0) + right.as_f64().unwrap_or(0.0); + Ok(Number::from_f64(sum).map_or(Value::Null, Value::Number)) +} + +/// `left > right`, for numbers or strings; any other pair raises `TypeError`. +fn greater(left: &Value, right: &Value) -> Result { + if let (Some(left), Some(right)) = (arithmetic(left), arithmetic(right)) { + return Ok(left.as_f64().unwrap_or(0.0) > right.as_f64().unwrap_or(0.0)); + } + if let (Value::String(left), Value::String(right)) = (left, right) { + return Ok(left > right); + } + Err(raised( + "TypeError", + format!( + "'>' not supported between instances of '{}' and '{}'", + type_name(left), + type_name(right) + ), + )) +} + +/// `_cost_value_as_float`: `float(value)`, or `None` when it is unset or not a number. +fn cost_as_float(value: Option<&Value>) -> Option { + match value? { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + Value::String(text) => { + without_digit_underscores(text.trim()).and_then(|digits| python_float(&digits)) + } + _ => None, + } +} + +fn float(value: Option) -> Value { + value + .and_then(Number::from_f64) + .map_or(Value::Null, Value::Number) +} + +/// `value.get(key)` on a dict; a value that is no dict raises `AttributeError`, as `.get` does. +fn get<'a>(value: &'a Value, key: &str) -> Result, Error> { + match value { + Value::Object(map) if !is_opaque(value) => Ok(map.get(key)), + other => Err(raised( + "AttributeError", + format!("'{}' object has no attribute 'get'", type_name(other)), + )), + } +} + +/// `model.get(key)`, then `litellm_params.get(key)`, then `model_info.get(key)`: the first that is +/// not `None`. +fn first_set( + model: &Value, + params: &Value, + info: &Value, + key: &str, +) -> Result, Error> { + for layer in [model, params, info] { + if let Some(value) = get(layer, key)?.filter(|value| !value.is_null()) { + return Ok(Some(value.clone())); + } + } + Ok(None) +} + +fn object(answer: Value, kind: &str) -> Result, Error> { + match answer { + Value::Object(map) => Ok(map), + _ => Err(malformed(kind)), + } +} + +/// The group being merged: the host's validated first entry, and what the merge assigned since. +struct Merged { + current: Map, + assignments: Map, +} + +impl Merged { + fn get(&self, field: &str) -> &Value { + self.current.get(field).unwrap_or(&Value::Null) + } + + fn set(&mut self, field: &str, value: Value) { + self.current.insert(field.to_owned(), value.clone()); + self.assignments.insert(field.to_owned(), value); + } + + /// `if info.get(field) is not None and info[field] > group.field (or group.field is None)`. + fn widen(&mut self, field: &str, info: &Map) -> Result<(), Error> { + let Some(value) = info.get(field).filter(|value| !value.is_null()) else { + return Ok(()); + }; + let current = self.get(field); + if current.is_null() || greater(value, current)? { + self.set(field, value.clone()); + } + Ok(()) + } + + /// The cost `field` widened to `float(info[field])` when that exceeds `group.field or 0.0`. + fn widen_cost(&mut self, field: &str, info: &Map) { + let Some(cost) = cost_as_float(info.get(field)) else { + return; + }; + let current = self.get(field); + let floor = if truthy(current) { + current.as_f64().unwrap_or(0.0) + } else { + 0.0 + }; + if current.is_null() || cost > floor { + self.set(field, float(Some(cost))); + } + } + + /// `if info.get(field) is True: group.field = True`. + fn any(&mut self, field: &str, info: &Map) { + if info.get(field) == Some(&Value::Bool(true)) { + self.set(field, Value::Bool(true)); + } + } +} + +/// `tuple(dict.fromkeys((*current, *added))) or None`. +fn union(current: &Value, added: &[Value]) -> Value { + let mut kept: Vec = Vec::new(); + let existing = current.as_array().map(Vec::as_slice).unwrap_or_default(); + for value in existing.iter().chain(added) { + if !kept.contains(value) { + kept.push(value.clone()); + } + } + if kept.is_empty() { + Value::Null + } else { + Value::Array(kept) + } +} + +/// `_set_model_group_info`: the group's info merged across the deployments that serve it. +pub async fn model_group_info( + host: &dyn Host, + query: &GroupInfoQuery, +) -> Result, Error> { + let mut merged: Option = None; + let mut totals: [(&str, Option); 4] = + [("tpm", None), ("rpm", None), ("itpm", None), ("otpm", None)]; + let mut clientside_auth_params = Value::Null; + let mut efforts_initialized = false; + let mut efforts_unknown = false; + let empty = Value::Object(Map::new()); + + for (index, model) in query.deployments.iter().enumerate() { + let named = get(model, "model_name")?; + let is_match = query.serves_team_deployments + || named.is_some_and(|name| name.as_str() == Some(query.model_group.as_str())) + || (named.is_some() && query.pattern_routes); + if !is_match { + continue; + } + let params = object( + lookup(host, ModelInfoLookup::GroupParams { index }).await?, + "group_params", + )?; + clientside_auth_params = params + .get("configurable_clientside_auth_params") + .cloned() + .unwrap_or(Value::Null); + let model_params = get(model, "litellm_params")?.unwrap_or(&empty); + let model_info_dict = get(model, "model_info")?.unwrap_or(&empty); + let mut deployment = [None, None, None, None]; + for (slot, (metric, _)) in deployment.iter_mut().zip(totals.iter()) { + *slot = first_set(model, model_params, model_info_dict, metric)?; + } + + let facts = object( + lookup( + host, + ModelInfoLookup::GroupDeployment { + index, + model: params.get("model").cloned().unwrap_or(Value::Null), + }, + ) + .await?, + "group_deployment", + )?; + let mapped = facts.get("mapped") == Some(&Value::Bool(true)); + let litellm_model = facts.get("litellm_model").cloned().unwrap_or(Value::Null); + let llm_provider = facts.get("llm_provider").cloned().unwrap_or(Value::Null); + let model_info = match facts.get("model_info") { + Some(Value::Object(info)) if !is_opaque(&Value::Object(info.clone())) => info.clone(), + Some(Value::Null) | None => { + // No model info: what the deployment's own (database) model_info says. + let db_info = get(model, "model_info")?.unwrap_or(&empty); + let mut info = Map::new(); + info.insert("key".to_owned(), Value::String(query.model_group.clone())); + for key in ["max_tokens", "max_input_tokens", "max_output_tokens"] { + info.insert(key.to_owned(), Value::Null); + } + for key in ["input_cost_per_token", "output_cost_per_token"] { + info.insert(key.to_owned(), float(cost_as_float(get(db_info, key)?))); + } + info.insert("litellm_provider".to_owned(), llm_provider.clone()); + info.insert( + "mode".to_owned(), + match get(db_info, "mode")? { + Some(mode) => mode.clone(), + None => Value::String("chat".to_owned()), + }, + ); + info.insert( + "supported_openai_params".to_owned(), + match facts.get("supported_openai_params") { + Some(Value::Null) | None => Value::Array(Vec::new()), + Some(params) => params.clone(), + }, + ); + info.insert("supports_system_messages".to_owned(), Value::Null); + info + } + Some(_) => return Err(malformed("group_deployment")), + }; + + match merged.as_mut() { + None => { + let mut seed = Map::new(); + seed.insert( + "model_group".to_owned(), + Value::String(query.user_facing_model_group_name.clone()), + ); + seed.insert( + "providers".to_owned(), + Value::Array(vec![llm_provider.clone()]), + ); + seed.insert("description".to_owned(), query.description.clone()); + seed.extend(model_info.clone()); + seed.insert( + "supports_continuation".to_owned(), + Value::Bool( + model_info.get("supports_continuation") == Some(&Value::Bool(true)), + ), + ); + seed.insert("supports_fast_mode".to_owned(), Value::Bool(true)); + seed.insert("supported_reasoning_efforts".to_owned(), Value::Null); + seed.insert("supported_endpoints".to_owned(), Value::Null); + let current = object( + lookup(host, ModelInfoLookup::GroupSeed { info: seed }).await?, + "group_seed", + )?; + merged = Some(Merged { + current, + assignments: Map::new(), + }); + } + Some(group) => { + let providers = group + .get("providers") + .as_array() + .cloned() + .unwrap_or_default(); + if !providers.contains(&llm_provider) { + let mut providers = providers; + providers.push(llm_provider.clone()); + group.set("providers", Value::Array(providers)); + } + group.widen("max_input_tokens", &model_info)?; + group.widen("max_output_tokens", &model_info)?; + group.widen_cost("input_cost_per_token", &model_info); + group.widen_cost("output_cost_per_token", &model_info); + for field in [ + "supports_parallel_function_calling", + "supports_vision", + "supports_function_calling", + "supports_web_search", + "supports_url_context", + "supports_continuation", + "supports_reasoning", + ] { + group.any(field, &model_info); + } + if let Some(params) = model_info + .get("supported_openai_params") + .filter(|params| !params.is_null()) + { + group.set("supported_openai_params", params.clone()); + } + for (slot, metric) in deployment.iter_mut().zip(["tpm", "rpm"]) { + if slot.is_none() + && let Some(value) = model_info.get(metric).filter(|v| !v.is_null()) + { + *slot = Some(value.clone()); + } + } + } + } + let Some(group) = merged.as_mut() else { + unreachable!("the group was just built"); + }; + + let capabilities = object( + lookup( + host, + ModelInfoLookup::GroupCapabilities { + index, + model_info: Value::Object(model_info), + litellm_model, + llm_provider, + mapped, + }, + ) + .await?, + "group_capabilities", + )?; + let endpoints = capabilities + .get("supported_endpoints") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let endpoints = union(group.get("supported_endpoints"), &endpoints); + group.set("supported_endpoints", endpoints); + let fast_mode = truthy(group.get("supports_fast_mode")) + && capabilities.get("supports_fast_mode") == Some(&Value::Bool(true)); + group.set("supports_fast_mode", Value::Bool(fast_mode)); + match capabilities + .get("reasoning_efforts") + .cloned() + .unwrap_or(Value::Null) + { + Value::Null => { + efforts_unknown = true; + group.set("supported_reasoning_efforts", Value::Null); + } + resolved if !efforts_initialized => { + efforts_initialized = true; + if !efforts_unknown { + group.set("supported_reasoning_efforts", resolved); + } + } + resolved if !efforts_unknown => { + let intersected = lookup( + host, + ModelInfoLookup::ReasoningIntersect { + current: group.get("supported_reasoning_efforts").clone(), + resolved, + }, + ) + .await?; + group.set("supported_reasoning_efforts", intersected); + } + _ => {} + } + + for ((_, total), value) in totals.iter_mut().zip(deployment) { + if let Some(value) = value { + *total = Some(add(total.take(), &value)?); + } + } + } + + let Some(mut group) = merged else { + return Ok(None); + }; + for (metric, total) in totals { + if let Some(total) = total { + group.set(metric, total); + } + } + if !clientside_auth_params.is_null() { + group.set( + "configurable_clientside_auth_params", + clientside_auth_params, + ); + } + Ok(Some(GroupInfo { + assignments: group.assignments, + })) +} + +/// `coerce_token_limit`: a token limit as an int, `None` when it is unset or not a usable number +/// (a bool never is). +pub fn coerce_token_limit(value: Option<&Value>) -> Option { + match value? { + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .filter(|n| n.is_finite()) + .map(|n| n.trunc() as i64) + }), + Value::String(text) => without_digit_underscores(text.trim()) + .and_then(|digits| python_int(&digits)) + .and_then(|number| number.as_i64()), + _ => None, + } +} + +/// The deployment's discovered limits under its configured `model_info` values that are set. +async fn effective_model_info( + host: &dyn Host, + id: Value, + configured: &Map, +) -> Result, Error> { + let mut info = discovered(host, id).await?; + info.extend( + configured + .iter() + .filter(|(_, value)| !value.is_null()) + .map(|(key, value)| (key.clone(), value.clone())), + ); + Ok(info) +} + +/// `get_configured_token_limits`: the (max input, max output) tokens a concrete deployment +/// configures, or its refresh discovered, from its `model_info` (`exclude_none` dump) and id. +pub async fn configured_token_limits( + host: &dyn Host, + id: Value, + configured: &Map, +) -> Result<(Option, Option), Error> { + let info = effective_model_info(host, id, configured).await?; + Ok(( + coerce_token_limit(info.get("max_input_tokens")), + coerce_token_limit(info.get("max_output_tokens")), + )) +} + +/// `get_configured_mode` / `get_configured_display_name`: a configured string that is not blank. +pub fn configured_text(value: Option<&Value>) -> Option { + value + .and_then(Value::as_str) + .filter(|text| !text.trim().is_empty()) + .map(str::to_owned) +} + +/// `get_configured_service_tiers`: each deployment's `model_info.service_tiers`, unvalidated. +pub fn configured_service_tiers(model_infos: &[Map]) -> Vec { + model_infos + .iter() + .map(|info| info.get("service_tiers").cloned().unwrap_or(Value::Null)) + .collect() +} + +/// What a listed name's deployments contribute to its `/v1/models` entry. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ListingInfo { + pub cost_map_keys: Vec, + pub max_input_tokens: Option, + pub max_output_tokens: Option, +} + +fn named_text(value: Option<&Value>) -> Option<&str> { + value + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) +} + +/// `deployment_pricing_models(deployment)[0]`: `model_info.base_model`, then +/// `litellm_params.base_model`, then `litellm_params.model`; malformed blocks count as unset. +fn pricing_key(deployment: &Value) -> Option { + let block = |key: &str| { + deployment + .get(key) + .and_then(Value::as_object) + .filter(|_| !is_opaque(deployment.get(key).unwrap_or(&Value::Null))) + }; + let params = block("litellm_params"); + let info = block("model_info"); + let base_model = info + .and_then(|info| info.get("base_model")) + .filter(|base| truthy(base)) + .or_else(|| params.and_then(|params| params.get("base_model"))); + named_text(base_model) + .or_else(|| named_text(params.and_then(|params| params.get("model")))) + .map(str::to_owned) +} + +/// `get_model_listing_info` over the deployments a listed name names: the cost-map keys of their +/// models, deduplicated, and the widest token limits they configure or discovered. +pub async fn model_listing_info( + host: &dyn Host, + deployments: &[Value], +) -> Result { + let mut listing = ListingInfo::default(); + let empty = Value::Object(Map::new()); + for deployment in deployments { + let info = match get(deployment, "model_info")? { + Some(info) if truthy(info) => info, + _ => &empty, + }; + let id = get(info, "id")?.cloned().unwrap_or(Value::Null); + let configured = match info { + Value::Object(map) => map.clone(), + _ => Map::new(), + }; + let effective = effective_model_info(host, id, &configured).await?; + for (field, widest) in [ + ("max_input_tokens", &mut listing.max_input_tokens), + ("max_output_tokens", &mut listing.max_output_tokens), + ] { + if let Some(limit) = coerce_token_limit(effective.get(field)) { + *widest = Some(widest.map_or(limit, |current| current.max(limit))); + } + } + } + for key in deployments.iter().filter_map(pricing_key) { + if !listing.cost_map_keys.contains(&key) { + listing.cost_map_keys.push(key); + } + } + Ok(listing) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn coerce_token_limit_reads_python_ints() { + assert_eq!(coerce_token_limit(Some(&json!(128))), Some(128)); + assert_eq!(coerce_token_limit(Some(&json!(12.9))), Some(12)); + assert_eq!(coerce_token_limit(Some(&json!(" 1_000 "))), Some(1000)); + assert_eq!(coerce_token_limit(Some(&json!("128,000"))), None); + assert_eq!(coerce_token_limit(Some(&json!(true))), None); + assert_eq!(coerce_token_limit(None), None); + } + + #[test] + fn totals_add_like_python() { + assert_eq!(add(None, &json!(5)).unwrap(), json!(5)); + assert_eq!(add(Some(json!(5)), &json!(true)).unwrap(), json!(6)); + assert_eq!(add(Some(json!(5)), &json!(0.5)).unwrap(), json!(5.5)); + assert!(add(Some(json!(5)), &json!("1")).is_err()); + } + + #[test] + fn union_keeps_first_seen_order_and_empty_reads_as_none() { + assert_eq!( + union(&json!(["a", "b"]), &[json!("b"), json!("c")]), + json!(["a", "b", "c"]) + ); + assert_eq!(union(&Value::Null, &[]), Value::Null); + } + + #[test] + fn pricing_key_prefers_the_base_model() { + let deployment = json!({ + "litellm_params": {"model": "azure/x", "base_model": "params-base"}, + "model_info": {"base_model": "info-base"}, + }); + assert_eq!(pricing_key(&deployment).as_deref(), Some("info-base")); + let deployment = json!({"litellm_params": {"model": "azure/x"}, "model_info": "junk"}); + assert_eq!(pricing_key(&deployment).as_deref(), Some("azure/x")); + } +} diff --git a/litellm-rust/crates/router-core/src/health.rs b/litellm-rust/crates/router-core/src/health.rs new file mode 100644 index 00000000000..020a1a0f47d --- /dev/null +++ b/litellm-rust/crates/router-core/src/health.rs @@ -0,0 +1,960 @@ +//! [`Health`]: cooldowns, allowed-fails counters and per-minute usage, kept in the shared +//! [`RouterCache`] under the Python router's keys, encodings and TTLs: +//! +//! | state | key | value | TTL | +//! |---|---|---|---| +//! | cooldown | `deployment:{id}:cooldown` | `{exception_received, status_code, timestamp, cooldown_time}` | cooldown time | +//! | allowed fails | `deployment:{id}:allowed_fails[:{suffix}]` | int counter | cooldown time | +//! | usage | `global_router:{id}:{model}:tpm:{HH-MM}` and `:rpm:` | int counter | 60s | +//! | outcomes (local only) | `{id}:successes`, `{id}:fails` | int counter | 60s | +//! +//! From `router_utils/cooldown_cache.py`, `router_utils/cooldown_handlers.py`, +//! `router_utils/router_callbacks/track_deployment_metrics.py`, `RouterCacheEnum` in +//! `types/router.py` and `Router._increment_usage` / `deployment_callback_on_failure` in +//! `router.py`. + +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use serde::{Deserialize, Serialize}; +use tracing::debug; + +use crate::{ + cache::RouterCache, + error::{AttemptError, Error, FailureKind}, + outcome::{OutcomeCallbacks, OutcomeWrite}, + registry::{Deployment, Registry}, + settings::Settings, +}; + +/// `RoutingArgs.ttl`: per-minute usage keys expire after a minute. +pub const USAGE_TTL: Duration = Duration::from_secs(60); +/// `DEFAULT_FAILURE_THRESHOLD_PERCENT`. +pub const FAILURE_THRESHOLD_PERCENT: f64 = 0.5; +/// `DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS`. +pub const FAILURE_THRESHOLD_MINIMUM_REQUESTS: u64 = 5; +/// `SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD`. +pub const SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: u64 = 1000; +/// `exception_masker` in `CooldownCache`: the visible prefix of `exception_received`. +const EXCEPTION_VISIBLE_PREFIX: usize = 50; + +pub trait Clock: Send + Sync { + fn now(&self) -> SystemTime; +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> SystemTime { + SystemTime::now() + } +} + +fn unix_seconds(time: SystemTime) -> f64 { + time.duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs_f64()) + .unwrap_or_default() +} + +/// `get_utc_datetime().strftime("%H-%M")`. +pub fn current_minute(time: SystemTime) -> String { + let seconds = time + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs()) + .unwrap_or_default() + % 86_400; + format!("{:02}-{:02}", seconds / 3600, (seconds % 3600) / 60) +} + +pub fn cooldown_key(id: &str) -> String { + format!("deployment:{id}:cooldown") +} + +pub fn allowed_fails_key(id: &str, suffix: Option<&str>) -> String { + match suffix { + Some(suffix) => format!("deployment:{id}:allowed_fails:{suffix}"), + None => format!("deployment:{id}:allowed_fails"), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::EnumString)] +pub enum UsageMetric { + #[strum(serialize = "tpm")] + Tpm, + #[strum(serialize = "rpm")] + Rpm, + /// Input tokens per minute, which the io-token rate limit check counts. + #[strum(serialize = "itpm")] + Itpm, + /// Output tokens per minute. + #[strum(serialize = "otpm")] + Otpm, +} + +/// `RouterCacheEnum.TPM` / `RPM` / `ITPM` / `OTPM`: `model` is the deployment's +/// `litellm_params.model`. +pub fn usage_key(id: &str, model: &str, metric: UsageMetric, minute: &str) -> String { + format!("global_router:{id}:{model}:{metric}:{minute}") +} + +/// `CooldownCacheValue`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CooldownValue { + pub exception_received: String, + pub status_code: String, + pub timestamp: f64, + /// Whole seconds are written as an int, as Python's default `cooldown_time` is, so the + /// "Try again in N seconds" text reads the same. + #[serde(serialize_with = "whole_as_int")] + pub cooldown_time: f64, +} + +fn whole_as_int(value: &f64, serializer: S) -> Result { + if value.fract() == 0.0 && value.abs() < i64::MAX as f64 { + serializer.serialize_i64(*value as i64) + } else { + serializer.serialize_f64(*value) + } +} + +/// `litellm.should_retry`. +pub fn should_retry_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429) || status >= 500 +} + +/// `_is_cooldown_required`: a 4xx cools down only for 429, 401, 402, 404 and 408, every other +/// status does, and a connection error or an empty status never does. +pub fn is_cooldown_required(failure: &AttemptError) -> bool { + if failure.kind == FailureKind::ApiConnection || failure.status_absent { + return false; + } + match failure.cooldown_status() { + status @ 400..500 => matches!(status, 429 | 401 | 402 | 408 | 404), + _ => true, + } +} + +/// `SensitiveDataMasker(visible_prefix=50, visible_suffix=0, mask_short_values=False)`. +fn mask_exception(text: &str) -> String { + let length = text.chars().count(); + if length <= EXCEPTION_VISIBLE_PREFIX { + return text.to_owned(); + } + let mut masked: String = text.chars().take(EXCEPTION_VISIBLE_PREFIX).collect(); + masked.extend(std::iter::repeat_n('*', length - EXCEPTION_VISIBLE_PREFIX)); + masked +} + +/// `_EXCEPTION_POLICY_FIELDS`: a deployment policy's fields in the order Python matches them +/// (a content-policy violation is a bad request, so it comes first). +const DEPLOYMENT_POLICY_FIELDS: [(&str, &str); 9] = [ + ( + "ContentPolicyViolationError", + "ContentPolicyViolationErrorAllowedFails", + ), + ("BadRequestError", "BadRequestErrorAllowedFails"), + ("AuthenticationError", "AuthenticationErrorAllowedFails"), + ("Timeout", "TimeoutErrorAllowedFails"), + ("RateLimitError", "RateLimitErrorAllowedFails"), + ("InternalServerError", "InternalServerErrorAllowedFails"), + ( + "ServiceUnavailableError", + "ServiceUnavailableErrorAllowedFails", + ), + ("BadGatewayError", "BadGatewayErrorAllowedFails"), + ("NotFoundError", "NotFoundErrorAllowedFails"), +]; + +/// `_resolve_allowed_fails_from_policy` over a deployment's `model_info.allowed_fails_policy`: +/// the first field set for a class the failure is an instance of. +pub fn deployment_allowed_fails( + policy: Option<&serde_json::Map>, + failure: &AttemptError, +) -> Option { + let policy = policy?; + DEPLOYMENT_POLICY_FIELDS + .iter() + .filter(|(class, _)| failure.is_a(class)) + .find_map(|(_, field)| policy.get(*field).and_then(serde_json::Value::as_f64)) +} + +/// What the cooldown rules need to know about the deployment's group. +#[derive(Clone, Copy, Debug, Default)] +pub struct GroupShape { + /// The deployment is the only one in its model group. + pub single_deployment: bool, + /// The requested group has a fallback configured. + pub has_fallback_group: bool, +} + +/// `_core_team_model_has_alternatives`: another routable deployment serves the deployment's team +/// public name. +pub(crate) fn team_model_has_alternatives(registry: &Registry, deployment: &Deployment) -> bool { + let info = &deployment.model_info; + let (Some(team_id), Some(public_name)) = (&info.team_id, &info.team_public_model_name) else { + return false; + }; + registry + .team_named(team_id, public_name) + .is_some_and(|siblings| { + siblings + .iter() + .filter(|sibling| !sibling.model_info.blocked) + .count() + > 1 + }) +} + +/// `_core_routing_group_has_alternatives`: `model_group` names a callable routing group whose +/// members span more than one deployment. +pub(crate) fn routing_group_has_alternatives(registry: &Registry, model_group: &str) -> bool { + let resolved = registry.alias_of(model_group).unwrap_or(model_group); + registry.routing_group_named(resolved).is_some_and(|group| { + group + .models + .iter() + .map(|member| registry.named(member).map_or(0, |members| members.len())) + .sum::() + > 1 + }) +} + +/// An outcome the router did not see itself (`Health.report`). +#[derive(Clone, Debug)] +pub enum Outcome { + Success { total_tokens: u64 }, + Failure(AttemptError), +} + +#[derive(Clone)] +pub struct Health { + cache: Arc, + clock: Arc, +} + +impl Health { + pub fn new(cache: Arc, clock: Arc) -> Self { + Self { cache, clock } + } + + pub fn cache(&self) -> &Arc { + &self.cache + } + + pub(crate) fn now_time(&self) -> SystemTime { + self.clock.now() + } + + fn now(&self) -> f64 { + unix_seconds(self.clock.now()) + } + + /// The ids among `ids` with an unexpired cooldown (`async_get_active_cooldowns`). + pub async fn active_cooldowns( + &self, + ids: &[String], + ) -> Result, Error> { + let keys: Vec = ids.iter().map(|id| cooldown_key(id)).collect(); + let values = self.cache.async_batch_get_cache(&keys).await?; + let now = self.now(); + Ok(ids + .iter() + .zip(values) + .filter_map(|(id, value)| { + let value: CooldownValue = serde_json::from_value(value?).ok()?; + (value.timestamp + value.cooldown_time - now > 0.0).then(|| (id.clone(), value)) + }) + .collect()) + } + + /// `add_deployment_to_cooldown`. + pub async fn add_cooldown( + &self, + id: &str, + failure: &AttemptError, + cooldown: Duration, + ) -> Result<(), Error> { + let value = CooldownValue { + exception_received: mask_exception(failure.detail.as_deref().unwrap_or_default()), + status_code: failure.cooldown_status().to_string(), + timestamp: self.now(), + cooldown_time: cooldown.as_secs_f64(), + }; + let value = serde_json::to_value(value).unwrap_or_default(); + self.cache + .async_set_cache(&cooldown_key(id), value, Some(cooldown)) + .await?; + Ok(()) + } + + async fn local_count(&self, key: &str) -> Result { + Ok(self + .cache + .async_get_local(key) + .await? + .and_then(|value| value.as_f64()) + .map_or(0, |count| count as u64)) + } + + /// The per-minute (successes, fails) this worker saw (`{id}:successes`, `{id}:fails`, kept + /// `local_only`). + pub async fn outcomes_this_minute(&self, id: &str) -> Result<(u64, u64), Error> { + Ok(( + self.local_count(&format!("{id}:successes")).await?, + self.local_count(&format!("{id}:fails")).await?, + )) + } + + /// `increment_deployment_successes_for_current_minute`: returns the key it counted under. + pub async fn count_success(&self, id: &str) -> Result { + let key = format!("{id}:successes"); + self.cache + .async_increment_local(&key, 1.0, Some(USAGE_TTL)) + .await?; + Ok(key) + } + + /// `increment_deployment_failures_for_current_minute`. + pub async fn count_failure(&self, id: &str) -> Result<(), Error> { + self.cache + .async_increment_local(&format!("{id}:fails"), 1.0, Some(USAGE_TTL)) + .await?; + Ok(()) + } + + /// Cooldown duration precedence from `deployment_callback_on_failure`: the deployment's + /// `cooldown_time`, then the provider's retry-after, then the router default. + pub fn cooldown_duration( + settings: &Settings, + deployment: &Deployment, + failure: &AttemptError, + ) -> Duration { + Self::cooldown_for_call(settings, deployment, None, failure) + } + + /// `cooldown_duration` for a call whose own params carry `cooldown_time` (a request's + /// value wins over the deployment's `litellm_params`, not over its `model_info`). + pub fn cooldown_for_call( + settings: &Settings, + deployment: &Deployment, + call_cooldown: Option, + failure: &AttemptError, + ) -> Duration { + let configured = deployment + .model_info + .cooldown_time + .or(call_cooldown) + .or_else(|| { + deployment + .raw_params + .get("cooldown_time") + .and_then(|v| v.as_f64()) + }) + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()); + configured + .or(failure.retry_after) + .unwrap_or(settings.cooldown_time) + } + + /// The deployment's own `cooldown_time`: `model_info` first, then `litellm_params` + /// (`_first_present(model_info, litellm_params, key="cooldown_time")`). It is both the + /// cooldown and the TTL of the deployment's `allowed_fails:generic` counter. + fn configured_cooldown(deployment: &Deployment) -> Option { + deployment + .model_info + .cooldown_time + .or_else(|| { + deployment + .raw_params + .get("cooldown_time") + .and_then(|v| v.as_f64()) + }) + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + } + + /// Record a failed attempt and cool the deployment down when the rules say so. Returns the + /// cooldown applied, if any. + pub async fn record_failure( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + call_cooldown: Option, + failure: &AttemptError, + ) -> Result, Error> { + if failure.outside_deployment { + return Ok(None); + } + self.count_failure(&deployment.id).await?; + let cooldown = Self::cooldown_for_call(settings, deployment, call_cooldown, failure); + self.cool_down_after(settings, deployment, shape, failure, cooldown) + .await + } + + /// `set_cooldown_deployments` for a failure already counted: cool the deployment down for + /// `cooldown` when the rules say so. Returns the cooldown applied, if any. + pub async fn cool_down_after( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + failure: &AttemptError, + cooldown: Duration, + ) -> Result, Error> { + if !Self::should_run_cooldown_logic(settings, deployment, failure, cooldown) { + return Ok(None); + } + if !self + .should_cooldown(settings, deployment, shape, failure) + .await? + { + return Ok(None); + } + debug!(deployment_id = %deployment.id, cooldown_seconds = cooldown.as_secs_f64(), "cooling deployment down"); + self.add_cooldown(&deployment.id, failure, cooldown).await?; + Ok(Some(cooldown)) + } + + /// `_should_run_cooldown_logic`. + fn should_run_cooldown_logic( + settings: &Settings, + deployment: &Deployment, + failure: &AttemptError, + cooldown: Duration, + ) -> bool { + !matches!( + failure.kind, + FailureKind::Declined + | FailureKind::CallerCredentialAuthentication + | FailureKind::CallerCredentialRateLimit + ) && !failure.caller_scoped + && !cooldown.is_zero() + && !settings.disable_cooldowns + && (is_cooldown_required(failure) + || deployment_allowed_fails( + deployment.model_info.allowed_fails_policy.as_ref(), + failure, + ) + .is_some()) + && !settings + .provider_default_deployment_ids + .contains(&deployment.id) + } + + /// `_should_cooldown_deployment`. + async fn should_cooldown( + &self, + settings: &Settings, + deployment: &Deployment, + shape: GroupShape, + failure: &AttemptError, + ) -> Result { + let status = failure.cooldown_status(); + let policy_fails = settings.allowed_fails_for(failure); + let lone = shape.single_deployment && !shape.has_fallback_group; + let deployment_policy = deployment.model_info.allowed_fails_policy.as_ref(); + let deployment_fails = deployment_allowed_fails(deployment_policy, failure); + if shape.single_deployment + && status == 402 + && deployment_fails.is_none() + && policy_fails.is_none() + { + return Ok(false); + } + let deployment_allowed = deployment.model_info.allowed_fails; + if deployment_policy.is_some() || deployment_allowed.is_some() { + // `_should_cooldown_based_on_deployment_policy`: a deployment-wide `allowed_fails` + // keeps the lone-deployment safety net; a policy entry naming the error does not. + if deployment_fails.is_none() + && lone + && (deployment_allowed.is_some() || policy_fails.is_none()) + { + return Ok(false); + } + let (allowed, suffix) = match deployment_fails { + Some(allowed) => ( + Some(allowed), + failure.class_names().first().map(|name| (*name).to_owned()), + ), + None => ( + deployment_allowed.map(f64::from), + deployment_allowed.map(|_| "generic".to_owned()), + ), + }; + let allowed = allowed + .or_else(|| policy_fails.map(f64::from)) + .unwrap_or_else(|| f64::from(settings.effective_allowed_fails())); + let ttl = + Self::configured_cooldown(deployment).unwrap_or_else(|| settings.fails_window()); + return self + .exceeds_allowed_fails(&deployment.id, suffix.as_deref(), allowed, ttl) + .await; + } + if settings.allowed_fails_policy.is_none() && !settings.allowed_fails_set() { + let (successes, fails) = self.outcomes_this_minute(&deployment.id).await?; + let total = successes + fails; + let percent_fails = if total > 0 { + fails as f64 / total as f64 + } else { + 0.0 + }; + return Ok((status == 429 && !shape.single_deployment) + || (percent_fails == 1.0 && total >= SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD) + || (percent_fails > FAILURE_THRESHOLD_PERCENT + && total >= FAILURE_THRESHOLD_MINIMUM_REQUESTS + && !shape.single_deployment) + || !should_retry_status(status)); + } + if lone && policy_fails.is_none() { + return Ok(false); + } + let allowed = policy_fails.unwrap_or_else(|| settings.effective_allowed_fails()); + self.exceeds_allowed_fails( + &deployment.id, + None, + f64::from(allowed), + settings.fails_window(), + ) + .await + } + + /// `should_cooldown_based_on_allowed_fails_policy`: count this failure, then compare. + async fn exceeds_allowed_fails( + &self, + id: &str, + suffix: Option<&str>, + allowed: f64, + ttl: Duration, + ) -> Result { + let fails = self + .cache + .async_increment(&allowed_fails_key(id, suffix), 1.0, Some(ttl)) + .await?; + Ok(fails > allowed) + } + + /// Count a success, and the tokens and request against the deployment's per-minute usage + /// when `tracks` says it has limits to track. The count is the success callback's; the usage + /// is `make_call`'s, which runs whatever the callbacks do. + pub async fn record_success( + &self, + callbacks: OutcomeCallbacks, + deployment: &Deployment, + total_tokens: u64, + tracks: bool, + ) -> Result<(), Error> { + callbacks + .record(OutcomeWrite::Success, self.count_success(&deployment.id)) + .await?; + if !tracks { + return Ok(()); + } + callbacks + .record( + OutcomeWrite::AttemptUsage, + self.add_usage(deployment, None, total_tokens as f64, 1.0), + ) + .await + } + + /// `_increment_deployment_usage`, for a deployment that tracks usage: tokens and requests + /// under `model` (metadata's `deployment`, which a wildcard route fills; else the + /// deployment's own), unless there is nothing to add. + pub async fn add_usage( + &self, + deployment: &Deployment, + model: Option<&str>, + total_tokens: f64, + requests: f64, + ) -> Result<(), Error> { + if total_tokens <= 0.0 && requests <= 0.0 { + return Ok(()); + } + let minute = current_minute(self.clock.now()); + let model = model.unwrap_or(&deployment.params.model); + self.cache + .async_increment_usage( + &[ + ( + usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), + total_tokens, + ), + ( + usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), + requests, + ), + ], + Some(USAGE_TTL), + ) + .await?; + Ok(()) + } + + /// `async_deployment_callback_on_failure`: a failed request counts against the deployment's + /// per-minute requests under `model` (see [`Self::add_usage`]), whether or not it tracks usage. + pub async fn count_failed_request( + &self, + deployment: &Deployment, + model: Option<&str>, + ) -> Result<(), Error> { + let minute = current_minute(self.clock.now()); + self.cache + .async_increment( + &usage_key( + &deployment.id, + model.unwrap_or(&deployment.params.model), + UsageMetric::Rpm, + &minute, + ), + 1.0, + Some(USAGE_TTL), + ) + .await?; + Ok(()) + } + + /// This minute's (tpm, rpm) for a deployment. + pub async fn usage(&self, deployment: &Deployment) -> Result<(u64, u64), Error> { + let minute = current_minute(self.clock.now()); + let model = &deployment.params.model; + let keys = [UsageMetric::Tpm, UsageMetric::Rpm] + .map(|metric| usage_key(&deployment.id, model, metric, &minute)); + let values = self.cache.async_batch_get_cache(&keys).await?; + let count = |index: usize| { + values + .get(index) + .cloned() + .flatten() + .and_then(|value| value.as_f64()) + .map_or(0, |count| count as u64) + }; + Ok((count(0), count(1))) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + + use super::*; + use crate::{ + cache::InMemoryRouterCache, + config::{RouterConfig, RouterDynamicConfig}, + registry::{DeploymentSpec, RoutingGroupSpec}, + }; + + #[rstest] + #[case::rate_limit(FailureKind::RateLimit, Some(429), true)] + #[case::auth(FailureKind::Authentication, Some(401), true)] + #[case::payment(FailureKind::Other, Some(402), true)] + #[case::not_found(FailureKind::NotFound, Some(404), true)] + #[case::timeout(FailureKind::Timeout, Some(408), true)] + #[case::bad_request(FailureKind::BadRequest, Some(400), false)] + #[case::forbidden(FailureKind::PermissionDenied, Some(403), false)] + #[case::server(FailureKind::InternalServer, Some(500), true)] + #[case::unreadable_status(FailureKind::Other, None, true)] + #[case::connection(FailureKind::ApiConnection, Some(500), false)] + fn cooldown_required_follows_python( + #[case] kind: FailureKind, + #[case] status: Option, + #[case] expected: bool, + ) { + assert_eq!( + is_cooldown_required(&AttemptError::new(kind, status)), + expected + ); + } + + /// An exception with no `status_code` is Python's status `""`, which no rule cools down. + #[rstest] + fn an_empty_status_requires_no_cooldown() { + let mut failure = AttemptError::new(FailureKind::Other, None); + failure.status_absent = true; + assert!(!is_cooldown_required(&failure)); + } + + #[rstest] + #[case::midnight(0, "00-00")] + #[case::afternoon(13 * 3600 + 7 * 60 + 59, "13-07")] + fn current_minute_is_utc_hours_and_minutes(#[case] seconds: u64, #[case] expected: &str) { + let time = UNIX_EPOCH + Duration::from_secs(86_400 * 3 + seconds); + assert_eq!(current_minute(time), expected); + } + + #[rstest] + #[case::content_policy_first(&["ContentPolicyViolationError", "BadRequestError"], Some(1.0))] + #[case::bad_request(&["BadRequestError"], Some(2.0))] + #[case::unnamed(&["Timeout"], None)] + fn a_deployment_policy_matches_in_python_order( + #[case] classes: &[&str], + #[case] expected: Option, + ) { + let policy = serde_json::json!({ + "BadRequestErrorAllowedFails": 2, + "ContentPolicyViolationErrorAllowedFails": 1, + }); + let mut failure = AttemptError::new(FailureKind::BadRequest, Some(400)); + failure.classes = classes.iter().map(|class| (*class).into()).collect(); + assert_eq!( + deployment_allowed_fails(policy.as_object(), &failure), + expected + ); + } + + fn deployment(id: &str, info: serde_json::Value) -> DeploymentSpec { + let mut model_info = serde_json::json!({"id": id}); + model_info + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + serde_json::from_value(serde_json::json!({ + "model_name": id, + "litellm_params": {"model": format!("openai/{id}")}, + "model_info": model_info, + })) + .unwrap() + } + + fn health() -> Health { + Health::new( + Arc::new(InMemoryRouterCache::default()), + Arc::new(SystemClock), + ) + } + + fn settings(dynamic: RouterDynamicConfig) -> Settings { + Settings::new(&RouterConfig::default(), Arc::new(dynamic)) + } + + #[rstest] + #[case::held(false, false, 1, true)] + #[case::caller_scoped(false, true, 1, false)] + #[case::outside_the_deployment(true, false, 0, false)] + #[tokio::test] + async fn the_failure_exits_decide_what_is_counted_and_cooled( + #[case] outside_deployment: bool, + #[case] caller_scoped: bool, + #[case] counted: u64, + #[case] cooled: bool, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let mut failure = AttemptError::new(FailureKind::RateLimit, Some(429)); + failure.outside_deployment = outside_deployment; + failure.caller_scoped = caller_scoped; + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, counted); + } + + #[rstest] + #[case::authentication( + FailureKind::CallerCredentialAuthentication, + 401, + "AuthenticationError" + )] + #[case::rate_limit(FailureKind::CallerCredentialRateLimit, 429, "RateLimitError")] + #[tokio::test] + async fn a_caller_credential_failure_counts_without_cooling( + #[case] kind: FailureKind, + #[case] status: u16, + #[case] base_class: &str, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let failure = AttemptError::new(kind, Some(status)); + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown, None); + assert!(failure.is_a(base_class)); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, 1); + } + + /// `litellm.allowed_fails` written after the build: the copied 3 starts counting instead of + /// the failure-ratio rule, which cools a 429 down at once. + #[rstest] + #[case::still_the_process_default(true, true)] + #[case::the_process_default_moved(false, false)] + #[tokio::test] + async fn the_copied_allowed_fails_counts_once_it_differs_from_the_process_default( + #[case] allowed_fails_is_default: bool, + #[case] cooled: bool, + ) { + let settings = settings(RouterDynamicConfig { + allowed_fails: Some(3), + allowed_fails_is_default, + ..RouterDynamicConfig::default() + }); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + let failure = AttemptError::new(FailureKind::RateLimit, Some(429)); + + let cooldown = health() + .record_failure( + &settings, + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + } + + fn team(id: &str, team_id: &str, blocked: bool) -> DeploymentSpec { + deployment( + id, + serde_json::json!({"team_id": team_id, "team_public_model_name": "tp", "blocked": blocked}), + ) + } + + #[rstest] + #[case::a_sibling(team("b", "t1", false), true)] + #[case::a_blocked_sibling(team("b", "t1", true), false)] + #[case::another_teams_deployment(team("b", "t2", false), false)] + #[case::no_team(deployment("b", serde_json::json!({})), false)] + fn a_team_deployment_has_alternatives_under_its_public_name( + #[case] other: DeploymentSpec, + #[case] expected: bool, + ) { + let registry = Registry::build( + vec![team("a", "t1", false), other], + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let first = registry.deployment("a").unwrap(); + assert_eq!(team_model_has_alternatives(®istry, &first), expected); + } + + #[rstest] + #[case::spans_two_deployments(&["a", "b"], "rg", true)] + #[case::one_deployment(&["a"], "rg", false)] + #[case::through_its_alias(&["a", "b"], "alias", true)] + #[case::a_member_called_directly(&["a", "b"], "a", false)] + fn a_routing_group_has_alternatives_past_one_deployment( + #[case] members: &[&str], + #[case] requested: &str, + #[case] expected: bool, + ) { + let group: RoutingGroupSpec = serde_json::from_value(serde_json::json!({ + "group_name": "rg", "models": members, "routing_strategy": "simple-shuffle", + })) + .unwrap(); + let registry = Registry::build( + vec![ + deployment("a", serde_json::json!({})), + deployment("b", serde_json::json!({})), + ], + BTreeMap::from([("alias".to_owned(), "rg".to_owned())]), + vec![group], + ) + .unwrap(); + assert_eq!( + routing_group_has_alternatives(®istry, requested), + expected + ); + } + + /// An empty status is counted and cools nothing down, unless the deployment's own policy + /// names the error. + #[rstest] + #[case::no_policy(serde_json::json!({}), false)] + #[case::policy_names_the_error( + serde_json::json!({"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}}), + true + )] + #[tokio::test] + async fn an_empty_status_on_the_attempt_path_needs_the_deployments_policy_to_cool( + #[case] info: serde_json::Value, + #[case] cooled: bool, + ) { + let health = health(); + let deployment = Deployment::from_spec(deployment("a", info)).unwrap(); + let mut failure = AttemptError::new(FailureKind::BadRequest, None); + failure.status_absent = true; + + let cooldown = health + .record_failure( + &settings(RouterDynamicConfig::default()), + &deployment, + GroupShape::default(), + None, + &failure, + ) + .await + .unwrap(); + + assert_eq!(cooldown.is_some(), cooled); + assert_eq!(health.outcomes_this_minute("a").await.unwrap().1, 1); + } + + #[rstest] + #[case::the_deployments_own_model(None, "openai/a")] + #[case::a_filled_wildcard(Some("openai/gpt-4o"), "openai/gpt-4o")] + #[tokio::test] + async fn usage_is_keyed_by_the_model_served(#[case] model: Option<&str>, #[case] keyed: &str) { + let cache = InMemoryRouterCache::default(); + let clock = Arc::new(SystemClock); + let health = Health::new(Arc::new(cache.clone()), clock.clone()); + let deployment = Deployment::from_spec(deployment("a", serde_json::json!({}))).unwrap(); + + health + .add_usage(&deployment, model, 15.0, 1.0) + .await + .unwrap(); + + let minute = current_minute(clock.now()); + assert_eq!( + ( + cache.get(&usage_key("a", keyed, UsageMetric::Tpm, &minute)), + cache.get(&usage_key("a", keyed, UsageMetric::Rpm, &minute)), + ), + (Some(serde_json::json!(15)), Some(serde_json::json!(1))), + ); + } + + #[rstest] + fn keys_match_python() { + assert_eq!( + ( + cooldown_key("abc"), + allowed_fails_key("abc", Some("generic")), + usage_key("abc", "openai/gpt-4o", UsageMetric::Tpm, "09-05"), + ), + ( + "deployment:abc:cooldown".to_owned(), + "deployment:abc:allowed_fails:generic".to_owned(), + "global_router:abc:openai/gpt-4o:tpm:09-05".to_owned(), + ) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/health_routing.rs b/litellm-rust/crates/router-core/src/health_routing.rs new file mode 100644 index 00000000000..bc3e43a6e92 --- /dev/null +++ b/litellm-rust/crates/router-core/src/health_routing.rs @@ -0,0 +1,126 @@ +//! Health-check-driven routing: the background health checks' results, kept in the shared +//! [`RouterCache`](crate::cache::RouterCache) under one key as Python's `DeploymentHealthCache` +//! writes them, and the filter `_async_filter_health_check_unhealthy_deployments` / +//! `_filter_health_check_unhealthy_deployments` run before the cooldown filter. + +use std::{collections::BTreeSet, sync::Arc, time::UNIX_EPOCH}; + +use serde_json::Value; + +use crate::{health::Health, registry::Deployment, settings::Settings}; + +/// `DeploymentHealthCache.CACHE_KEY`. +pub const HEALTH_STATE_KEY: &str = "litellm:health_check:deployment_health_state"; + +/// `_extract_unhealthy_ids`: the ids marked unhealthy whose result is younger than `staleness`. +pub fn unhealthy_ids(raw: Option<&Value>, now: f64, staleness: f64) -> BTreeSet { + let Some(Value::Object(states)) = raw else { + return BTreeSet::new(); + }; + states + .iter() + .filter(|(_, state)| { + let Value::Object(state) = state else { + return false; + }; + // `not state.get("is_healthy", True)`: any falsy value is unhealthy. + let unhealthy = match state.get("is_healthy") { + None => false, + Some(Value::Null) => true, + Some(Value::Bool(value)) => !value, + Some(Value::Number(value)) => value.as_f64() == Some(0.0), + Some(Value::String(value)) => value.is_empty(), + Some(Value::Array(value)) => value.is_empty(), + Some(Value::Object(value)) => value.is_empty(), + }; + let timestamp = state + .get("timestamp") + .and_then(Value::as_f64) + .unwrap_or(0.0); + unhealthy && now - timestamp < staleness + }) + .map(|(id, _)| id.clone()) + .collect() +} + +/// What the filter does when it would leave nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WhenAllUnhealthy { + /// The safety net: every candidate is kept. + KeepAll, + /// A health check probe sees no deployment. + Nothing, +} + +impl Health { + /// The deployments the background health checks have not marked unhealthy, as the + /// router's health-check routing settings say. + pub async fn health_check_filter( + &self, + settings: &Settings, + deployments: Vec>, + when_all_unhealthy: WhenAllUnhealthy, + ) -> Vec> { + if !settings.enable_health_check_routing { + return deployments; + } + let scoped = settings.background_health_check_model_groups.as_deref(); + // With an allowed_fails_policy cooldowns are the only exclusion, unless the filter is + // scoped to listed groups. + if settings.allowed_fails_policy.is_some() && scoped.is_none() { + return deployments; + } + // A read that fails counts as no health state. + let raw = self + .cache() + .async_get_cache(HEALTH_STATE_KEY) + .await + .ok() + .flatten(); + let now = self + .now_time() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs_f64()) + .unwrap_or_default(); + let unhealthy = unhealthy_ids( + raw.as_ref(), + now, + settings.health_check_staleness.as_secs_f64(), + ); + if unhealthy.is_empty() { + return deployments; + } + let filtered: Vec> = deployments + .iter() + .filter(|d| { + !unhealthy.contains(&d.id) + || scoped.is_some_and(|groups| !groups.contains(&d.model_name)) + }) + .cloned() + .collect(); + match (filtered.is_empty(), when_all_unhealthy) { + (false, _) => filtered, + (true, WhenAllUnhealthy::KeepAll) => deployments, + (true, WhenAllUnhealthy::Nothing) => Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::fresh_unhealthy(json!({"a": {"is_healthy": false, "timestamp": 95.0}}), vec!["a"])] + #[case::stale(json!({"a": {"is_healthy": false, "timestamp": 50.0}}), vec![])] + #[case::healthy(json!({"a": {"is_healthy": true, "timestamp": 95.0}}), vec![])] + #[case::no_flag(json!({"a": {"timestamp": 95.0}}), vec![])] + #[case::not_a_dict(json!({"a": 1}), vec![])] + fn unhealthy_ids_follow_python(#[case] raw: Value, #[case] expected: Vec<&str>) { + let ids = unhealthy_ids(Some(&raw), 100.0, 10.0); + assert_eq!(ids.iter().map(String::as_str).collect::>(), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/host.rs b/litellm-rust/crates/router-core/src/host.rs new file mode 100644 index 00000000000..0a72be8f980 --- /dev/null +++ b/litellm-rust/crates/router-core/src/host.rs @@ -0,0 +1,726 @@ +//! [`Host`]: the router core's only boundary. Everything the core needs from the embedding +//! process goes through it, and every attempt runs through [`Host::invoke`]: the core never runs +//! an operation itself. `PythonHost` serves it in a Python process (attempts call `litellm.`, +//! and the deployment build asks it for secrets, validation, providers and side effects). + +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::{ + cache::RouterCache, + config::{Op, RetryPolicy, RoutingStrategy}, + error::{AttemptError, Error, FailureKind, Payload}, + fallback::Fallbacks, + health::Health, + mock::MockFailure, + outcome::OutcomeCallbacks, + precall::PreCallFacts, + prerouting::{PreRoute, PreRouted}, + registry::{Deployment, Registry}, + settings::Settings, + strategy::StrategyScope, +}; + +/// One call into the router. +#[derive(Clone, Debug, Default)] +pub struct RouterRequest { + /// The requested name: a model group, alias, routing group, deployment id or wildcard match. + pub model: String, + /// The request params, for the plugins and host steps that read them. + pub body: Value, + /// `user_api_key_team_id`, from the first metadata bucket that holds one + /// (`get_request_team_id`). + pub team_id: Option, + /// The `metadata` bucket's `user_api_key_team_id` alone: the team a failed level lists the + /// group's deployments for (`async_function_with_fallbacks_common_utils`, + /// `_stream_order_levels`). `None` lists every team's. + pub listing_team_id: Option, + /// The caller's `user_role` is `proxy_admin`. + pub proxy_admin: bool, + /// The caller's `models` plus `team_models` (`user_api_key_auth`), which narrow a caller + /// allowed only through access groups to those groups' deployments. + pub allowed_models: Vec, + /// For a proxy admin, the model the request names (`model`, else the metadata + /// `model_group`), which keeps one team's deployments listed under it (`filter_team_based_models`). + pub admin_model: Option, + pub specific_deployment: bool, + /// A per-request `num_retries`, which wins over the router's. + pub num_retries: Option, + /// A per-request `num_retries` that is no count, which replaces the router's as well. + pub uncounted_retries: Option, + pub disable_fallbacks: bool, + /// `_router_weights` as the request passed it: model group to deployment id to weight, + /// validated when simple-shuffle picks ([`crate::weights::validate`]). + pub router_weights: Option, + /// The deployment a call on an existing resource (a video id) names: picked whenever the + /// filters keep it (`_narrow_to_video_deployment`). + pub pinned_deployment_id: Option, + /// A per-request `routing_strategy` override, as Python normalizes it. + pub routing_strategy: Option, + /// Whether tag routing may apply to this request (router, request or model setting), so the + /// host's tag stage runs. + pub filter_tags: bool, + /// A pass-through pick: only `use_in_pass_through` deployments may serve it. + pub pass_through: Option, + /// `_target_order`: the order level to pick from instead of the lowest. + pub target_order: Option, + /// `_excluded_deployment_ids`: never picked, even when that leaves nothing. + pub excluded_ids: Vec, + /// `_retry_skipped_deployment_ids`: skipped unless that leaves nothing. + pub retry_skipped_ids: Vec, + /// The endpoint `endpoint_selection` names around the call, which deployments that declare + /// `enforce_supported_endpoints` may refuse ([`crate::eligibility`]). + pub endpoint: Option, + /// A web search request on an async pick: `filter_web_search_deployments` drops the + /// deployments that declare no support. The sync pick never runs that filter. + pub web_search: bool, + /// The handler name of a call only some providers serve (`_ROUTE_CAPABILITY_RESOLVERS`): + /// the deployments whose provider cannot serve it are dropped unless that leaves nothing. + pub route_handler: Option, + /// A Decisions call carrying audio or videos: the deployments that refuse them are skipped + /// unless that leaves nothing (`_without_decisions_deployments_refusing_media`). + pub decisions_media: bool, + /// `f"{provider}/{model}"` for a lookup that cannot ask the host + /// (`_core_get_routable_deployments`); a pick asks [`Host::provider_request`]. + pub wildcard_request: Option, + /// The request carries an image (`_request_carries_image`), so deployments that declare no + /// vision are skipped while any other remains. + pub has_image: bool, + /// An async pick: the host's pre-routing step runs before each pick resolves its name. + pub pre_routes: bool, + /// Which of the router's outcome callbacks run for this request. + pub outcome_callbacks: OutcomeCallbacks, + /// The requested name is no model group (a search tool name): nothing is selected, every + /// attempt is handed to the host as is, and no outcome is recorded for a deployment + /// (`_asearch_with_fallbacks`, whose helper picks among the router's search tools). + pub unrouted: bool, + /// `passthrough_on_no_deployment`: a pick that fails hands the attempt to the host under the + /// requested name, inside the level's retries and fallbacks, with no outcome recorded for a + /// deployment (`_ageneric_api_call_with_fallbacks_helper`). + pub passthrough_on_no_deployment: bool, + /// The endpoint's attempt stamps no `failed_deployment_id` on its error (file and batch + /// creation, whose result belongs to the deployment that served it), so weighted failover + /// never runs for it. + pub skip_weighted_failover: bool, + /// The request creates or names a resource only one provider holds (a file upload, a batch + /// on an uploaded file): it falls back within its own model group only + /// (`run_async_fallback`'s `same_model_group_only`). + pub same_group_fallbacks_only: bool, + /// Fallback lists the request passed (`Some(None)` when it passed `None`), which replace + /// the router's. + pub fallbacks: Option>, + pub context_window_fallbacks: Option>, + pub content_policy_fallbacks: Option>, + /// A per-request `max_fallbacks`, which replaces the router's. + pub max_fallbacks: Option, + /// A per-request `cooldown_time`: the call's own params carry it, so it cools a failed + /// deployment down for that long unless its `model_info` sets one. + pub cooldown_time: Option, + /// A `mock_testing_*` flag the request set. + pub mock_failure: Option, + /// A per-request `model_group_retry_policy`, which replaces the router's. + pub model_group_retry_policy: Option>, + /// The model groups whose `model_group_retry_policy` lookup raises (the policy is no mapping + /// of policies, or the group's entry is not one): the first failure of a hop serving one ends + /// the hop with the host's error instead ([`RouterEvent::RetryPolicyRaised`]), no retry. + pub retry_policy_raises: Vec, + /// `health_check_probe`: a health check's own pick, which sees no deployment when every + /// one is marked unhealthy (the safety net keeps them all for any other request). + pub health_check_probe: bool, + /// What the host needs to run the call itself, such as the Python kwargs. + pub payload: Option, +} + +/// A per-request `num_retries` that is no count of retries. `async_function_with_retries` only +/// reads it once a hop's first failure was not taken over by a retry policy: a call that succeeds +/// first time never does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UncountedRetries { + /// A negative int: `num_retries > 0` is false, so the failure stands, and a stream's plain + /// budget is spent (`isinstance(num_retries, int)`). + Negative, + /// Any other value `num_retries > 0` is false for (a negative float, a NaN): the failure + /// stands, and a stream falls through to the deployment's budget, then the router's. + NotPositive, + /// `num_retries > 0` or `range(num_retries)` raises (a string, a positive float): the hop ends + /// with that error ([`RouterEvent::RetryCountRaised`]). + Raises, +} + +/// One `os.environ/` reference the deployment build found ([`crate::build`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SecretRef { + /// The value as written, `os.environ/`. + pub name: String, + /// Where the value sits in the deployment. + pub at: SecretAt, +} + +/// Where a [`SecretRef`] sits in a deployment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SecretAt { + /// A top-level `litellm_params` value: resolved through the host's secret manager. + Param(String), + /// `litellm_params.dataSources[source].parameters[key]` (`endpoint` or `key`): read from the + /// environment alone, empty when unset. + DataSource { source: usize, key: String }, +} + +/// Which of the router's adds a build runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum BuildPath { + /// `set_model_list` (and the constructor's list): config entries, one deployment per + /// organization, ignored when invalid if the router ignores invalid deployments. + Create, + /// `add_deployment` (and upserts): one validated deployment. + Add, +} + +/// What the build asks the host about one deployment. +#[derive(Clone, Debug)] +pub struct BuildAsk<'a> { + pub path: BuildPath, + /// The entry's position in the list the build was given. + pub index: usize, + /// The entry as the build holds it: `model_name`, `litellm_params` (secrets resolved, the + /// organization of this deployment set), `model_info` (with its `id`) and any other keys. + pub entry: &'a Map, + /// The `model_info.id` the entry declared, before the build assigned one. + pub declared_id: Option<&'a str>, + /// Another entry of the same list declares the same id. + pub duplicate_id: bool, +} + +/// The host's answer about one deployment. +#[derive(Clone, Debug, PartialEq)] +pub enum Built { + Entry(Map), + /// Left out without an error (`model_info.supported_environments` excludes this one, or an + /// `add_deployment` id that already exists). + Skip, + /// The host failed the deployment and the router ignores invalid deployments. + Invalid, +} + +/// The host's provider answer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Provided { + Provider(Option), + Invalid, +} + +/// What the model-info merge ([`crate::model_info`]) asks the host while it answers one query. +/// Asked at query time, so a price reload or a registration the host made since the router was +/// built is what the next query reads. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(tag = "lookup", rename_all = "snake_case")] +pub enum ModelInfoLookup { + /// The process cost map's row for `model` (`get_model_info(model=...)`), the host choosing + /// the keys it tries. Answered with the row, or `None`. A lookup that raises propagates, + /// unless `tolerate` is set: then the host answers `{"failed": True}`. `exact` also asks + /// whether the row came from a fallback-generalization rule (`{"row": ..., "generalized": + /// bool}`). + ModelInfo { + model: Value, + tolerate: bool, + exact: bool, + }, + /// The raw cost-map entry under `key` (`model_cost.get(key)`): answered with + /// `{"entry": value}`, or `None` when the key is absent. `process` reads the process global + /// (`litellm.model_cost`) rather than the host's live settings, where the Python router does. + CostEntry { key: String, process: bool }, + /// `get_llm_provider` for the queried deployment's `model` and validated params: answered + /// with `[model, provider]`. + Provider { model: String }, + /// The deployments a wildcard pattern routes `name` to: answered with `[{id, model}]`, or + /// `None` when no pattern matches. + PatternModels { name: String }, + /// The base model a bedrock model id names (`get_bedrock_base_model`). + BedrockBase { model_id: String }, + /// The limits the background model-info refresh discovered for deployment `id`. + Discovered { id: Value }, + /// The queried deployment's own `model_info`, read when the merge reaches it (a malformed one + /// raises there). + UserModelInfo, + /// A log record, emitted where the Python router logs it. + Log { level: LogLevel, message: String }, + /// Group info: deployment `index`'s validated params (`LiteLLM_Params(**litellm_params)`, which + /// raises on bad params): answered with `{"model", "configurable_clientside_auth_params"}`. + GroupParams { index: usize }, + /// Group info: what the host knows of deployment `index` serving `model`: answered with + /// `{"model_info"}` (its merged model info, `None` when that fails), `"mapped"` + /// (`deployment_is_catalog_mapped`), `"litellm_model"` and `"llm_provider"`, and, when it has + /// no model info, `"supported_openai_params"`. + GroupDeployment { index: usize, model: Value }, + /// Group info: deployment `index`'s capabilities under `model_info`: answered with + /// `{"supported_endpoints"}` (empty when the team may not use it), `"supports_fast_mode"` and + /// `"reasoning_efforts"` (`resolve_supported_reasoning_efforts`, or `None`). + GroupCapabilities { + index: usize, + model_info: Value, + litellm_model: Value, + llm_provider: Value, + mapped: bool, + }, + /// Group info: the group's first deployment as the host builds it (`ModelGroupInfo(**info)`, + /// whose validation may raise): answered with the fields the merge compares, as validated. + GroupSeed { info: Map }, + /// `intersect_supported_reasoning_efforts(current, resolved)`, in the host's advertisement order. + ReasoningIntersect { current: Value, resolved: Value }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LogLevel { + Debug, + Error, +} + +/// One attempt against one deployment. +#[derive(Clone, Debug)] +pub struct Attempt { + pub op: Op, + pub request: Arc, + pub deployment: Arc, + /// The name this attempt resolved: the requested name, or the fallback target. + pub name: String, + /// The model group being served, after fallbacks. + pub model_group: String, + /// The `order` level an order fallback targets in the same group. + pub target_order: Option, + /// Every deployment `name` resolved to for this caller, the picked one included. + pub candidate_ids: Vec, + /// 0 for the first attempt in this group, then the retry number. + pub retry: u32, + pub fallback_depth: u32, + /// How many fallbacks deep the chain is: 0 for the requested name. + pub fallback_level: u32, + pub timeout: Option, + /// Request params the fallback chain's dict targets merged in (`kwargs.update(target)`). + pub params: Option>>, + /// The chain level whose request metadata this attempt carries (see [`RouterEvent::Fallback`]). + pub scope: u32, + /// The retry budget the level's loop stamps on the request before this attempt. + pub max_retries: u32, + /// How many deployments `get_model_list` lists for the level's group (`model_group_size`). + pub model_group_size: usize, +} + +/// What a successful attempt returns. +#[derive(Clone, Debug)] +pub struct Output { + pub value: Payload, + /// Prompt plus completion tokens, counted against the deployment's per-minute usage. + pub total_tokens: u64, + /// A response the provider blocked, which the router may raise to fall back. + pub blocked: Option, + /// A stream the host reports the outcome of when it ends ([`crate::Router::report_success`], + /// [`crate::Router::report_failure`]), or a realtime session, whose success the host counts + /// only when it ended cleanly: the attempt counts its request and `total_tokens` now, and no + /// success. + pub deferred: bool, + /// Whether the attempt counts against the deployment's per-minute usage, when the host knows + /// better than the deployment's own limits ([`Deployment::tracks_usage`]): a limit only the + /// model catalog sets (`_deployment_tracks_usage`), or a realtime session, whose `None` + /// response counts nothing (`increment_deployment_usage_for_response`). + pub tracks_usage: Option, +} + +/// How a provider blocked a response the call otherwise served. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum Blocked { + /// A chat response whose first choice was content-filtered (or that has no choices). + ContentFilter, + /// An Anthropic /v1/messages safeguard refusal carrying `stop_details`. + Refusal, +} + +/// What a plugin or strategy sees: never the router itself. +pub struct RoutingContext<'a> { + pub registry: &'a Registry, + pub settings: &'a Settings, + pub health: &'a Health, + pub request: &'a RouterRequest, + pub op: &'a Op, + pub model_group: &'a str, + /// The name the request resolved from (Python's `model`), which strategies key state by. + pub name: &'a str, + /// Whose selector a strategy stage reads. + pub scope: StrategyScope, + /// The deployments the name resolved to for this caller, before health filters + /// (`request_deployments`), for the tag stage. + pub pool: &'a [Arc], + /// `_excluded_deployment_ids` a weighted failover hop put on the request, which the callback + /// filters read (`deployment_ids_failed_in_request`). + pub excluded_ids: &'a [String], + /// `_retry_skipped_deployment_ids` the retry loop put on the request, as `excluded_ids`. + pub skipped_ids: &'a [String], + /// `_target_order` as the filters read it: the level an order fallback targets, else the one + /// the request named. + pub target_order: Option, +} + +/// Which selection step the host is asked to run. +#[derive(Clone, Debug, PartialEq)] +pub enum SelectStage { + /// Pre-call checks and sub-routers registered as plugins: return the deployments that remain. + Plugins, + /// A strategy not yet ported: return the candidates with the pick first. + Strategy(RoutingStrategy), + /// Tag routing (`get_deployments_for_tag`), not yet ported: return the deployments that remain. + Tags, + /// `_deployment_declares_no_vision`: return the candidates that declare no vision support. + TextOnly, + /// `_deployment_serves_route`: return the candidates whose provider cannot serve the call + /// `handler` names (a Decisions call on a provider with no Decisions config). + RouteIncapable { handler: String }, + /// `_decisions_deployment_refuses_media`: return the candidates that refuse the audio or + /// videos a Decisions call carries. + MediaRefusing, +} + +/// Routing events, delivered through [`Host::observe`]. +#[derive(Clone, Debug, PartialEq)] +pub enum RouterEvent { + Selected { + model_group: String, + deployment_id: String, + /// How long the pick took (`_acompletion`'s `async_get_available_deployment` service event). + pick: Duration, + }, + Succeeded { + deployment_id: String, + }, + AttemptFailed { + deployment_id: String, + kind: FailureKind, + status: Option, + }, + CooldownSet { + deployment_id: String, + cooldown: Duration, + }, + Retry { + model_group: String, + retry: u32, + delay: Duration, + }, + Fallback { + from: String, + to: String, + /// The order level of `to` a same-group hop walks to. + order: Option, + depth: u32, + /// The fallback level `to` is tried at: 1 for the request's own fallbacks. + level: u32, + /// The new level's request metadata scope, copied from `parent`'s at the hop. + scope: u32, + parent: u32, + /// The `max_fallbacks` the chain reads, stamped on the hop's request. + max_fallbacks: u32, + /// The non-string `model` of the dict target this hop routes + /// ([`crate::fallback::FallbackTarget::unroutable_model`]): the hop fails on it before any + /// attempt, and the host raises what Python's level raises for that value. + unroutable: Option, + }, + /// A hop of the fallback chain returned: `log_success_fallback_event` or + /// `log_failure_fallback_event` for the level at `parent`. + FallbackDone { + from: String, + to: String, + level: u32, + scope: u32, + parent: u32, + succeeded: bool, + }, + /// The router raised a `ContentPolicyViolationError` for a blocked response, since a + /// fallback would serve it (`_should_raise_content_policy_error`, + /// `_should_raise_anthropic_refusal_error`). + Blocked { + level: u32, + model: String, + blocked: Blocked, + }, + /// `log_retry`: the router recorded a failure in `scope`'s request metadata + /// (`previous_models`, `request_retry_count`) before retrying or falling back. + RetryLogged { + scope: u32, + level: u32, + /// The group the failed level served (`kwargs["model"]`). + model: String, + /// The retry number stamped before the failed try; `None` keeps the scope's own. + attempted_retries: Option, + cause: RetryCause, + }, + /// `async_function_with_retries` opened a level's retry loop, before any pick: `scope`'s + /// request metadata tracks no retry yet, `max_retries` of them, and the group's size. + RetriesOpened { + scope: u32, + max_retries: u32, + model_group_size: usize, + }, + /// The level's first failure settled its overrides: the budget `scope` then tracks. + RetryBudget { + scope: u32, + max_retries: u32, + }, + /// A pick of the async selection or the sync one failed on `error`: the async one's `except` + /// logs every refusal to the request's logging object (`_async_get_available_deployment`). + PickFailed { + level: u32, + model: String, + error: PickError, + }, + /// A level of the fallback chain serving `model` ended on an error a pick raised (no + /// deployment for the caller, none available, a stale routing group, ...): Python raises + /// that error for the level, whatever its earlier attempts raised. + LevelFailed { + level: u32, + model: String, + error: PickError, + /// Raised by the level's fallback handling, while the level's own error was handled + /// (so it chains to that error), rather than by the level's retry loop. + handling: bool, + }, + /// The router raised a `mock_testing_*` failure for `model` instead of calling it. + MockFailed { + mock: MockFailure, + level: u32, + model: String, + num_retries: Option, + }, + /// The first failure of a hop serving `model` looked up a `model_group_retry_policy` that + /// raises for it: the hop ends with the error the lookup raised (the host raises it, from the + /// failure `cause` names), without retrying, and falls back as from any plain exception. + RetryPolicyRaised { + level: u32, + model: String, + cause: RetryCause, + }, + /// The first failure of a hop serving `model` read a request `num_retries` that raises + /// ([`UncountedRetries::Raises`]): the hop ends with that error (the host raises it, from the + /// failure `cause` names), and falls back as from any plain exception. + RetryCountRaised { + level: u32, + model: String, + cause: RetryCause, + }, + /// The level serving `model` reached the fallback handling that appends the context window + /// and content policy hints to its error, before any hop runs + /// (`async_function_with_fallbacks_common_utils`). The order path never gets there. + FallingBack { + level: u32, + model: String, + }, + /// The order fallback chain of the level at `level` served nothing, so the level raises from + /// `async_function_with_fallbacks_common_utils`' order path, with no outcome message: what the + /// chain's last hop raised when `last_hop`, else its own error (a context window error, or a + /// chain no hop of which ran). Weighted failover raising what it does not catch + /// (`_maybe_run_weighted_failover`) ends the level the same way. + OrderChainFailed { + level: u32, + last_hop: bool, + }, + /// The regular fallback handling of the level at `level` served nothing, where + /// `async_function_with_fallbacks_common_utils` reaches its `except` and logs the failure. + /// `chain` when the level had targets to walk: the error it logs is what the last hop raised + /// when `last_hop`, else its own. With no `chain`, the host decides: only the host holds + /// whether a level with no match had a fallback list at all, which Python does not log. + /// The order path and weighted failover are outside that `try` and never report this. + FallbacksFailed { + level: u32, + chain: bool, + last_hop: bool, + }, + /// A group's retry loop ran every retry and the last one failed: Python stamps + /// `num_retries` and `max_retries` on that error. + RetriesExhausted { + model_group: String, + attempted: u32, + max: u32, + /// The level the retry loop served. + level: u32, + /// The last failure was a pick's error, which the level then raises + /// ([`RouterEvent::LevelFailed`]), rather than an attempt's. + picked: bool, + }, +} + +/// A pick's error, as the events that name it carry it. +#[derive(Clone, Debug)] +pub struct PickError(pub Arc); + +impl PickError { + /// The copy of `error` events carry, when a pick raised it. + pub fn of(error: &Error) -> Option { + error.pick_copy().map(|copy| Self(Arc::new(copy))) + } +} + +impl PartialEq for PickError { + fn eq(&self, other: &Self) -> bool { + format!("{:?}", self.0) == format!("{:?}", other.0) + } +} + +/// Which error a [`RouterEvent::RetryLogged`] records. +#[derive(Clone, Debug, PartialEq)] +pub enum RetryCause { + /// The last attempt's failure. + Attempt, + /// The error the level ended on (`run_async_fallback`'s `original_exception`). + Level, + /// A pick raised this error: the group was unknown or had none available, the stale + /// routing-group member map named a group no longer configured, ... + Pick(PickError), + /// Any other error no attempt raised. + Other, +} + +impl RetryCause { + /// The cause that names `error`. + pub fn of(error: &Error) -> Self { + match error { + Error::Attempt(_) => Self::Attempt, + _ => PickError::of(error).map_or(Self::Other, Self::Pick), + } + } +} + +#[async_trait] +pub trait Host: Send + Sync { + /// Run one attempt against the picked deployment. The core records its outcome (cooldowns, + /// allowed-fails counters, usage) itself; the host only runs it. + async fn invoke(&self, attempt: &Attempt) -> Result; + + /// Run a selection stage the core does not implement. + async fn select( + &self, + stage: &SelectStage, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error>; + + /// Pre-routing before a pick resolves `ask.name`: the model group to resolve instead, a + /// custom strategy's deployment to serve, or `None` to resolve the name itself. Asked only + /// for picks that pre-route (`pre_routes`). + async fn pre_route(&self, ask: &PreRoute<'_>) -> Result, Error> { + let _ = ask; + Ok(None) + } + + /// Access and budget checks before a cross-group fallback target. + /// `Err` when a check raised: the chain ends there, outside any target's `try`. + async fn allow_fallback( + &self, + request: &RouterRequest, + target: &str, + ) -> Result; + + /// Whether `run_async_fallback` refuses a chain for one of its dict `targets`, a server-owned + /// federation field or an `oidc/` api key. The chain is walked by `level`. + async fn refuses_fallbacks(&self, _level: u32, _targets: &[Map]) -> bool { + false + } + + async fn sleep(&self, duration: Duration); + + fn observe(&self, event: &RouterEvent); + + /// The shared store routing state lives in. + fn cache(&self) -> Arc; + + /// Whether [`Self::invoke`] can run `op` on `provider`. Checked for every served op and + /// deployment when the router is built or changed, so an unsupported pair fails the config, + /// never a request. `PythonHost` runs every op. + fn supports(&self, _op: &Op, _provider: &str) -> bool { + true + } + + /// Whether [`Self::select`] can run strategies and plugins. + fn can_select(&self) -> bool { + true + } + + /// Run the routing plugins for `ctx.name` when `run` (the async pick's pre-routing hook), and + /// report the candidate models they left on the request, if any. + async fn routing_plugin_models( + &self, + _ctx: &RoutingContext<'_>, + _run: bool, + ) -> Result>, Error> { + Ok(None) + } + + /// The facts the `enable_pre_call_checks` filter decides by (token count, context windows, + /// param support), for `candidates` and the deployments in `ctx.pool`. A host without them + /// reports no input, so nothing is checked. + async fn pre_call_facts( + &self, + _ctx: &RoutingContext<'_>, + _candidates: &[Arc], + ) -> Result { + Ok(PreCallFacts::default()) + } + + /// `f"{provider}/{model}"` for `ctx.name`, which `PatternMatchRouter.get_pattern` tries when + /// no pattern matches the name itself: only the host can infer a provider. `None` from a + /// host that infers none. + async fn provider_request(&self, _ctx: &RoutingContext<'_>) -> Result, Error> { + Ok(None) + } + + /// `async_callback_filter_deployments`: the host's routing callbacks narrow `candidates` in + /// turn. A filter that raises fails the try with [`Error::Attempt`], which the call retries and + /// falls back from like any attempt failure. A host without filters keeps every candidate. + async fn callback_filters( + &self, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + /// The values of `refs`, one per ref, in order: what the deployment build substitutes for + /// each `os.environ/` reference it found in entry `index` ([`crate::build`]). Asked once per + /// deployment, as the build reaches it. A host without secrets leaves every reference as + /// written. + async fn secrets(&self, index: usize, refs: &[SecretRef]) -> Result, Error> { + let _ = index; + Ok(refs.iter().map(|r| Value::String(r.name.clone())).collect()) + } + + /// Check one deployment before it is added, and answer the entry as the host normalized it, + /// a skip (the deployment is left out), or [`Built::Invalid`] (it failed and the router + /// ignores invalid deployments). A host without checks accepts the entry as given. + async fn validate(&self, ask: &BuildAsk<'_>) -> Result { + Ok(Built::Entry(ask.entry.clone())) + } + + /// The provider that serves one validated deployment (`get_llm_provider`), `None` for one + /// served without a provider (a prompt-management model), or [`Provided::Invalid`]. A host + /// without provider resolution answers `None`, so [`Deployment::provider`] reads the model. + async fn provider(&self, ask: &BuildAsk<'_>) -> Result { + let _ = ask; + Ok(Provided::Provider(None)) + } + + /// Run the host's side effects for an accepted deployment and answer the entry as the + /// registry stores it. A host without side effects stores the entry it validated. + async fn added(&self, ask: &BuildAsk<'_>, provider: Option<&str>) -> Result { + let _ = provider; + Ok(Built::Entry(ask.entry.clone())) + } + + /// Answer one model-info lookup ([`ModelInfoLookup`]). A host without a cost map answers + /// `None` to everything, so no deployment has model info. + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + let _ = lookup; + Ok(Value::Null) + } +} diff --git a/litellm-rust/crates/router-core/src/host_index.rs b/litellm-rust/crates/router-core/src/host_index.rs new file mode 100644 index 00000000000..3748f7b4d19 --- /dev/null +++ b/litellm-rust/crates/router-core/src/host_index.rs @@ -0,0 +1,727 @@ +//! [`HostIndexes`]: the registry indexes the Python router hands out, kept the way it keeps them. +//! +//! The Python router keeps its indexes by history rather than deriving them from the deployment +//! list: `model_names` is the set of names `set_model_list` was given (invalid entries included), +//! `deployment_names` only ever grows, assigning `model_list` reindexes nothing, a removal shifts +//! the positions every index holds (a caller-written one included), and the wildcard pattern +//! routers keep the copy of a deployment they were given. The registry keeps one of these per +//! version and applies those rules on its own writes; the host reads them back as the live +//! objects `registry().index()`, `names()` and `patterns()` return, and hands a caller's write to +//! any of them back as a changeset ([`IndexWrite`]). +//! +//! Values are JSON so whatever a caller wrote reads back as written, junk included. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// One wildcard pattern router (`PatternMatchRouter`): regex to the deployments it routes to, +/// most specific pattern first. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct PatternRouter { + pub patterns: Vec<(String, Vec)>, +} + +/// The indexes, as the host reads and writes them. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct HostIndexes { + /// `model_id_to_deployment_index_map`: id to position, in insertion order. + pub id_map: Map, + /// `model_name_to_deployment_indices`: name to its positions. + pub model_name: Map, + /// `team_model_to_deployment_indices`: `[team_id, public name, positions]`. + pub team_model: Vec<(String, String, Value)>, + /// `model_names`: a set, its members in the order they joined. + pub model_names: Vec, + pub deployment_names: Vec, + pub team_public_model_names: Vec, + pub provider_default_ids: Vec, + /// `pattern_router`. + pub patterns: PatternRouter, + /// `team_pattern_routers`: team id to its router. + pub team_patterns: Vec<(String, PatternRouter)>, +} + +/// A host write to the indexes: each field it names replaces the one held. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default)] +pub struct IndexWrite { + pub id_map: Option>, + pub model_name: Option>, + pub team_model: Option>, + pub model_names: Option>, + pub deployment_names: Option>, + pub team_public_model_names: Option>, + pub provider_default_ids: Option>, + pub patterns: Option, + pub team_patterns: Option>, +} + +/// A removal the Python router could not finish: an index held a position it cannot compare +/// (`_without_indices` raised `TypeError`). The indexes keep what it shifted before that. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("'>' not supported between instances of '{found}' and 'int'")] +pub struct RemovalError { + pub found: &'static str, +} + +impl HostIndexes { + /// Apply a host write. + pub fn write(&mut self, write: IndexWrite) { + let IndexWrite { + id_map, + model_name, + team_model, + model_names, + deployment_names, + team_public_model_names, + provider_default_ids, + patterns, + team_patterns, + } = write; + if let Some(value) = id_map { + self.id_map = value; + } + if let Some(value) = model_name { + self.model_name = value; + } + if let Some(value) = team_model { + self.team_model = value; + } + if let Some(value) = model_names { + self.model_names = unique(value); + } + if let Some(value) = deployment_names { + self.deployment_names = value; + } + if let Some(value) = team_public_model_names { + self.team_public_model_names = value; + } + if let Some(value) = provider_default_ids { + self.provider_default_ids = value; + } + if let Some(value) = patterns { + self.patterns = value; + } + if let Some(value) = team_patterns { + self.team_patterns = value; + } + } + + /// `set_model_list` starting over: the id, name and team indexes and the team routers go; + /// the deployment names, the wildcard router and the provider default ids stay. + pub fn reset(&mut self) { + self.id_map.clear(); + self.model_name.clear(); + self.team_model.clear(); + self.team_patterns.clear(); + self.team_public_model_names.clear(); + } + + /// `set_model_list`'s `model_names`: the names it was given. + pub fn set_model_names(&mut self, names: Vec) { + self.model_names = unique(names.into_iter().map(Value::String).collect()); + } + + /// A deployment joined the list at `position` (`_add_deployment` then + /// `_add_model_to_list_and_index_map`); `add` is `add_deployment`, which also names its group. + pub fn added(&mut self, entry: &Map, position: usize, add: bool) { + let params = entry.get("litellm_params").and_then(Value::as_object); + let info = entry.get("model_info").and_then(Value::as_object); + let model_name = entry.get("model_name").and_then(Value::as_str); + let id = info + .and_then(|info| info.get("id")) + .filter(|id| !id.is_null()); + self.deployment_names.push( + params + .and_then(|params| params.get("model")) + .cloned() + .unwrap_or(Value::Null), + ); + if let Some(name) = model_name.filter(|name| name.contains('*')) { + self.patterns.add(name, Value::Object(entry.clone())); + if let Some(id) = id.and_then(Value::as_str).filter(|id| !id.is_empty()) { + self.provider_default_ids.push(id.to_owned()); + } + } + let team_id = info + .and_then(|info| info.get("team_id")) + .filter(|v| !v.is_null()); + let public = info + .and_then(|info| info.get("team_public_model_name")) + .filter(|v| !v.is_null()); + if let (Some(team_id), Some(Value::String(public))) = (team_id, public) + && public.contains('*') + { + let team = python_str(team_id); + let router = match self.team_patterns.iter_mut().find(|(t, _)| *t == team) { + Some((_, router)) => router, + None => { + self.team_patterns.push((team, PatternRouter::default())); + &mut self.team_patterns.last_mut().expect("just pushed").1 + } + }; + router.add(public, Value::Object(entry.clone())); + } + let position = Value::from(position); + if let Some(id) = id { + self.id_map.insert(python_str(id), position.clone()); + } + if let Some(name) = model_name.filter(|name| !name.is_empty()) { + push_position( + self.model_name + .entry(name.to_owned()) + .or_insert_with(|| Value::Array(Vec::new())), + &position, + ); + } + if add { + let name = Value::String(model_name.unwrap_or_default().to_owned()); + if !self.model_names.contains(&name) { + self.model_names.push(name); + } + } + // `_update_team_model_index`: a truthy team id and public name. + if let (Some(team_id), Some(public)) = (team_id, public) + && truthy(team_id) + && truthy(public) + { + let (team, public) = (python_str(team_id), python_str(public)); + if !self.team_public_model_names.contains(&public) { + self.team_public_model_names.push(public.clone()); + } + let held = match self + .team_model + .iter_mut() + .find(|(t, p, _)| *t == team && *p == public) + { + Some((_, _, held)) => held, + None => { + self.team_model + .push((team, public, Value::Array(Vec::new()))); + &mut self.team_model.last_mut().expect("just pushed").2 + } + }; + if let Value::Array(positions) = held + && !positions.contains(&position) + { + positions.push(position); + } + } + } + + /// The deployment `id` left the list from `position` (`_update_deployment_indices_after_removal`). + pub fn removed(&mut self, id: &str, position: usize) -> Result<(), RemovalError> { + let mut id_map = Map::new(); + for (key, held) in std::mem::take(&mut self.id_map) { + if key == id { + continue; + } + let shifted = shift(&held, position)?.unwrap_or(held); + id_map.insert(key, shifted); + } + self.id_map = id_map; + let names: Vec = self.model_name.keys().cloned().collect(); + for name in names { + let kept = without(&self.model_name[&name], position)?; + if kept.is_empty() { + self.model_name.remove(&name); + self.model_names + .retain(|held| held.as_str() != Some(name.as_str())); + } else { + self.model_name.insert(name, Value::Array(kept)); + } + } + let mut team_model = Vec::new(); + let held = std::mem::take(&mut self.team_model); + let mut rest = held.into_iter(); + for (team, public, positions) in rest.by_ref() { + match without(&positions, position) { + Ok(kept) if kept.is_empty() => {} + Ok(kept) => team_model.push((team, public, Value::Array(kept))), + Err(error) => { + team_model.push((team, public, positions)); + team_model.extend(rest); + self.team_model = team_model; + return Err(error); + } + } + } + self.team_model = team_model; + let mut publics: Vec = Vec::new(); + for (_, public, _) in &self.team_model { + if !publics.contains(public) { + publics.push(public.clone()); + } + } + self.team_public_model_names = publics; + self.patterns.remove(id); + for (_, router) in &mut self.team_patterns { + router.remove(id); + } + self.team_patterns + .retain(|(_, router)| !router.patterns.is_empty()); + Ok(()) + } + + /// Whether the id map holds `id` (`_has_model_id`). + pub fn has_id(&self, id: &str) -> bool { + self.id_map.contains_key(id) + } + + /// What the id map holds for `id`, as written (`model_id_to_deployment_index_map.get(id)`). + pub fn id_entry(&self, id: &str) -> Option<&Value> { + self.id_map.get(id) + } + + /// What the name index holds for `name`, as written. + pub fn named(&self, name: &str) -> Option<&Value> { + self.model_name.get(name) + } + + /// What the team index holds for `(team, public)`, as written. + pub fn team_named(&self, team: &str, public: &str) -> Option<&Value> { + self.team_model + .iter() + .find(|(t, p, _)| t == team && p == public) + .map(|(_, _, positions)| positions) + } + + /// `team_public_model_names_of`: the public names the team index holds for `team`. + pub fn team_public_names_of(&self, team: &str) -> Vec { + let mut names: Vec = Vec::new(); + for (t, public, _) in &self.team_model { + if t == team && !names.contains(public) { + names.push(public.clone()); + } + } + names + } + + /// The position the id map holds for `id`, when it is a position. + pub fn position_of(&self, id: &str) -> Option { + self.id_map + .get(id) + .and_then(Value::as_u64) + .and_then(|position| usize::try_from(position).ok()) + } +} + +impl PatternRouter { + /// `add_pattern`: a new regex joins in specificity order (a stable sort, most specific + /// first); a known one gains the deployment. + pub fn add(&mut self, pattern: &str, deployment: Value) { + let regex = pattern_to_regex(pattern); + if let Some((_, deployments)) = self.patterns.iter_mut().find(|(held, _)| *held == regex) { + deployments.push(deployment); + return; + } + self.patterns.push((regex, vec![deployment])); + self.patterns + .sort_by_key(|(regex, _)| std::cmp::Reverse(regex_specificity(regex))); + } + + /// `remove_deployment`: every deployment with this id goes, and so does a pattern left empty. + pub fn remove(&mut self, id: &str) { + for (_, deployments) in &mut self.patterns { + deployments.retain(|deployment| { + deployment + .get("model_info") + .and_then(|info| info.get("id")) + .and_then(Value::as_str) + != Some(id) + }); + } + self.patterns + .retain(|(_, deployments)| !deployments.is_empty()); + } + + /// `route`: the deployments of the first pattern `request` matches, each a copy whose + /// `litellm_params.model` takes the parts the wildcards matched; `filtered` keeps to the + /// patterns of those names. + pub fn route(&self, request: &str, filtered: Option<&[String]>) -> Option> { + let filtered: Option> = + filtered.map(|names| names.iter().map(|name| pattern_to_regex(name)).collect()); + for (regex, deployments) in &self.patterns { + if filtered + .as_ref() + .is_some_and(|names| !names.contains(regex)) + { + continue; + } + let Some(groups) = regex_match(regex, request) else { + continue; + }; + let mut routed = Vec::with_capacity(deployments.len()); + for deployment in deployments { + let mut copy = deployment.clone(); + let model = deployment + .get("litellm_params") + .and_then(|params| params.get("model")); + let Some(Value::String(model)) = model else { + // `deployment["litellm_params"]["model"]` raising ends the route with None. + return None; + }; + let named = deployment_model_name(&groups, request, model); + if model.contains('*') + && let Some(params) = + deployment.get("litellm_params").and_then(Value::as_object) + && let Some(info) = copy.get_mut("model_info").and_then(Value::as_object_mut) + { + for flag in cache_flags_left_to_the_catalog(params) { + info.remove(*flag); + } + } + if let Some(params) = copy + .get_mut("litellm_params") + .and_then(Value::as_object_mut) + { + params.insert("model".to_owned(), Value::String(named)); + } + routed.push(copy); + } + return Some(routed); + } + None + } + + /// `get_models_by_pattern`'s pick: the first (request, pattern) pair that matches, trying + /// `model` against every pattern, then `provider/model`. + pub fn models_by_pattern(&self, requests: &[String]) -> Option> { + for request in requests { + for (regex, deployments) in &self.patterns { + if let Some(groups) = regex_match(regex, request) { + return Some( + deployments + .iter() + .filter_map(|deployment| { + deployment + .get("litellm_params") + .and_then(|params| params.get("model")) + .and_then(Value::as_str) + .map(|model| deployment_model_name(&groups, request, model)) + }) + .collect(), + ); + } + } + } + None + } +} + +/// `_cache_flags_left_to_the_catalog`: the `model_info` flags a wildcard deployment does not speak +/// for on the models it matches. An azure_ai wildcard's `supports_prompt_cache_breakpoint` does. +/// `inferred_provider` names `azure_ai` only for a model carrying that prefix. +pub(crate) fn cache_flags_left_to_the_catalog( + params: &Map, +) -> &'static [&'static str] { + let provider = params + .get("custom_llm_provider") + .filter(|provider| truthy(provider)) + .and_then(Value::as_str) + .or_else(|| { + params + .get("model") + .and_then(Value::as_str) + .and_then(|model| model.split_once('/')) + .map(|(prefix, _)| prefix) + }); + if provider == Some("azure_ai") { + &["supports_prompt_caching"] + } else { + &[ + "supports_prompt_cache_breakpoint", + "supports_prompt_caching", + ] + } +} + +/// `re.escape(pattern).replace(r"\*", "(.*)")`. +pub fn pattern_to_regex(pattern: &str) -> String { + let mut regex = String::new(); + for c in pattern.chars() { + match c { + '*' => regex.push_str("(.*)"), + '(' | ')' | '[' | ']' | '{' | '}' | '?' | '+' | '-' | '|' | '^' | '$' | '\\' | '.' + | '&' | '~' | '#' | ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c' => { + regex.push('\\'); + regex.push(c); + } + _ => regex.push(c), + } + } + regex +} + +/// `PatternUtils.calculate_pattern_specificity`: the regex's length, then how much regex syntax +/// it has. +pub fn regex_specificity(regex: &str) -> (usize, usize) { + let complexity = regex + .chars() + .filter(|c| matches!(c, '*' | '+' | '?' | '\\' | '^' | '$' | '|' | '(' | ')')) + .count(); + (regex.chars().count(), complexity) +} + +/// A pattern regex read back as its literal runs between wildcards, or None when it holds regex +/// syntax `pattern_to_regex` never writes (a caller wrote it). +fn literal_runs(regex: &str) -> Option> { + let mut runs = vec![String::new()]; + let mut chars = regex.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => runs.last_mut()?.push(chars.next()?), + '(' => { + let rest: String = chars.by_ref().take(3).collect(); + if rest != ".*)" { + return None; + } + runs.push(String::new()); + } + '.' | '*' | '+' | '?' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | ')' => return None, + other => runs.last_mut()?.push(other), + } + } + Some(runs) +} + +/// `re.match(regex, request)` for a pattern regex: anchored at the start only, each `(.*)` +/// greedy. The groups it captured, or None when it does not match. +pub fn regex_match(regex: &str, request: &str) -> Option> { + let runs = literal_runs(regex)?; + let rest = request.strip_prefix(runs[0].as_str())?; + let mut groups = Vec::new(); + greedy(&runs[1..], rest, &mut groups).then_some(groups) +} + +/// Match `runs` (each a literal that follows a group) against `rest`, longest group first. +fn greedy(runs: &[String], rest: &str, groups: &mut Vec) -> bool { + let Some((literal, later)) = runs.split_first() else { + return true; + }; + let mut ends: Vec = rest.char_indices().map(|(at, _)| at).collect(); + ends.push(rest.len()); + for &end in ends.iter().rev() { + let (group, after) = rest.split_at(end); + let Some(after) = after.strip_prefix(literal.as_str()) else { + continue; + }; + groups.push(group.to_owned()); + if greedy(later, after, groups) { + return true; + } + groups.pop(); + } + false +} + +/// `set_deployment_model_name`: the deployment's model with each wildcard replaced by the group +/// matched for it, the request itself when more groups matched than it has wildcards. +pub fn deployment_model_name(groups: &[String], request: &str, model: &str) -> String { + if !model.contains('*') { + return model.to_owned(); + } + if groups.len() > model.matches('*').count() { + return request.to_owned(); + } + let mut named = model.to_owned(); + for group in groups { + named = named.replacen('*', group, 1); + } + named +} + +fn unique(values: Vec) -> Vec { + let mut kept: Vec = Vec::with_capacity(values.len()); + for value in values { + if !kept.contains(&value) { + kept.push(value); + } + } + kept +} + +fn push_position(held: &mut Value, position: &Value) { + if let Value::Array(positions) = held { + positions.push(position.clone()); + } +} + +/// Python's truthiness for a JSON value. +pub(crate) fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(items) => !items.is_empty(), + } +} + +/// Python's `str()` for an id or team id. +pub(crate) fn python_str(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => "True".to_owned(), + Value::Bool(false) => "False".to_owned(), + Value::Null => "None".to_owned(), + other => other.to_string(), + } +} + +fn type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +/// `idx - 1 if idx > removal_idx else idx`: None keeps `held` as is. +fn shift(held: &Value, position: usize) -> Result, RemovalError> { + let removal = position as f64; + let number = match held { + Value::Bool(flag) => f64::from(u8::from(*flag)), + Value::Number(number) => number.as_f64().unwrap_or_default(), + other => { + return Err(RemovalError { + found: type_name(other), + }); + } + }; + if number <= removal { + return Ok(None); + } + Ok(Some(match held { + Value::Number(n) if n.is_i64() || n.is_u64() => Value::from(n.as_i64().unwrap_or(0) - 1), + _ => Value::from(number - 1.0), + })) +} + +/// `_without_indices`: the positions without `position`, the later ones shifted down. +fn without(held: &Value, position: usize) -> Result, RemovalError> { + let Value::Array(positions) = held else { + return Ok(Vec::new()); + }; + let removal = Value::from(position); + let mut kept = Vec::with_capacity(positions.len()); + for index in positions { + if *index == removal { + continue; + } + kept.push(shift(index, position)?.unwrap_or_else(|| index.clone())); + } + Ok(kept) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn entry(name: &str, id: &str, model: &str) -> Map { + serde_json::from_value(json!({ + "model_name": name, + "litellm_params": {"model": model}, + "model_info": {"id": id}, + })) + .unwrap() + } + + #[test] + fn removal_shifts_every_index_and_drops_emptied_names() { + let mut indexes = HostIndexes::default(); + indexes.added(&entry("gpt", "a", "openai/a"), 0, false); + indexes.added(&entry("gpt", "b", "openai/b"), 1, false); + indexes.added(&entry("claude", "c", "anthropic/c"), 2, false); + indexes.set_model_names(vec!["gpt".into(), "claude".into(), "broken".into()]); + indexes.removed("a", 0).unwrap(); + assert_eq!( + indexes.id_map, + json!({"b": 0, "c": 1}).as_object().unwrap().clone() + ); + assert_eq!(indexes.model_name["gpt"], json!([0])); + indexes.removed("c", 1).unwrap(); + assert!(!indexes.model_name.contains_key("claude")); + assert_eq!(indexes.model_names, vec![json!("gpt"), json!("broken")]); + assert_eq!(indexes.deployment_names.len(), 3); + } + + #[test] + fn a_pattern_routes_with_the_matched_parts() { + let mut router = PatternRouter::default(); + router.add( + "llmengine/*", + json!({"litellm_params": {"model": "openai/*"}, "model_info": {"id": "w"}}), + ); + router.add( + "*meta.llama3*", + json!({"litellm_params": {"model": "bedrock/meta.llama3*"}, "model_info": {"id": "m"}}), + ); + let routed = router.route("llmengine/foo", None).unwrap(); + assert_eq!(routed[0]["litellm_params"]["model"], "openai/foo"); + let routed = router.route("hello-world-meta.llama3-70b", None).unwrap(); + assert_eq!( + routed[0]["litellm_params"]["model"], + "hello-world-meta.llama3-70b" + ); + assert_eq!( + regex_match(r"a(.*)b(.*)", "axbyb").unwrap(), + vec!["xby", ""] + ); + router.remove("w"); + assert!(router.route("llmengine/foo", None).is_none()); + } + + #[rstest] + #[case::any_wildcard( + json!({"model": "openai/*"}), + json!({"id": "w", "keep": 1}), + )] + #[case::azure_ai_keeps_the_breakpoint_flag( + json!({"model": "azure_ai/*"}), + json!({"id": "w", "keep": 1, "supports_prompt_cache_breakpoint": true}), + )] + #[case::azure_ai_named_by_custom_llm_provider( + json!({"model": "*", "custom_llm_provider": "azure_ai"}), + json!({"id": "w", "keep": 1, "supports_prompt_cache_breakpoint": true}), + )] + #[case::a_fixed_model_speaks_for_itself( + json!({"model": "openai/gpt-4o"}), + json!({ + "id": "w", + "keep": 1, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + }), + )] + fn a_wildcard_leaves_the_cache_flags_to_the_catalog( + #[case] params: Value, + #[case] expected: Value, + ) { + let mut router = PatternRouter::default(); + router.add( + "any/*", + json!({ + "litellm_params": params, + "model_info": { + "id": "w", + "keep": 1, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + }, + }), + ); + let routed = router.route("any/gpt-4o", None).unwrap(); + assert_eq!(routed[0]["model_info"], expected); + } +} diff --git a/litellm-rust/crates/router-core/src/lib.rs b/litellm-rust/crates/router-core/src/lib.rs index 05d95d02c84..5f15b6b8df0 100644 --- a/litellm-rust/crates/router-core/src/lib.rs +++ b/litellm-rust/crates/router-core/src/lib.rs @@ -43,21 +43,706 @@ //! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the //! policy is exhausted. //! -//! # Execution -//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs -//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A -//! started attempt is never replayed on a different executor. +//! # Hosts: the only boundary +//! The core reaches outside itself only through [`Host`], and every attempt runs through +//! `Host::invoke`: the core picks the deployment, the host runs the call, the core records the +//! outcome. There is no second, "native" attempt path in the core. Which code runs an attempt is +//! the host's business: +//! - `PythonHost` (inside a Python process) builds the request for the picked deployment and calls +//! `litellm.`. It never calls back into a Python router. //! -//! # Hosts -//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select` -//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks, -//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all -//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost` -//! serves them from the existing Python code when the router runs inside Python; `NativeHost` -//! serves them in the Rust server, where an operation without a Rust implementation makes the -//! config fail at build time, never partway through a request. +//! An op or provider a host cannot run fails `Host::supports` when the router is built or +//! changed, never partway through a request. +//! +//! The core also builds its deployments through the host ([`build`]): for each configured entry +//! it finds the secret references and asks `secrets`, then `validate`, `provider` and `added`. +//! +//! Besides `invoke` and `supports`, a host provides `select` for strategies and plugins not yet +//! ported, `allow_fallback` for access and budget checks, `refuses_fallbacks` for a chain's dict +//! targets, `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that +//! holds all routing state under the same keys, encodings and TTLs the Python router uses. The +//! core writes cooldowns, allowed-fails counters and usage itself, through that cache. //! //! # Plugins //! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive) //! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and //! the current request), never the router itself. +//! +//! # Modules +//! - [`registry`]: deployments and their indices, `deployment_id`, `ChangeSet`. +//! - [`config`] / [`settings`]: static and dynamic config, and the frozen `Settings` view. +//! - [`cache`] / [`health`]: the shared routing cache, cooldowns and usage. +//! - [`select`]: resolve, filter chain, simple-shuffle. +//! - [`call`]: retries and fallbacks. +//! - [`eligibility`]: the endpoints a deployment refuses to serve. +//! - [`precall`]: the `enable_pre_call_checks` filter. +//! - [`model_info`]: model info merged from the cost map, asked of the host at query time. +//! - [`host`]: the `Host` boundary. + +pub mod access; +pub mod build; +pub mod cache; +pub mod call; +pub mod config; +pub mod eligibility; +pub mod error; +pub mod fallback; +pub mod group_info; +pub mod health; +pub mod health_routing; +pub mod host; +pub mod host_index; +pub mod lookup; +pub mod mock; +pub mod model_info; +pub mod outcome; +pub mod precall; +pub mod prerouting; +pub mod ptu; +pub mod registry; +pub mod resolve; +pub mod retry; +pub mod routing_groups; +pub mod routing_plugins; +pub mod select; +pub mod settings; +pub mod strategy; +pub mod weights; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use arc_swap::ArcSwap; +use parking_lot::Mutex; + +pub use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + call::RouterResponse, + config::{Op, RouterConfig, RouterDynamicConfig, RoutingStrategy}, + error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported}, + health::{Clock, Health, Outcome, SystemClock}, + host::{ + Attempt, Blocked, BuildAsk, BuildPath, Built, Host, Output, Provided, RetryCause, + RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage, + }, + outcome::{OutcomeCallbacks, OutcomeWrite}, + prerouting::{PreRoute, PreRouted}, + registry::{ChangeSet, Deployment, DeploymentSpec, Registry, RegistryVersion}, + select::{Randomness, SeededRandomness, ThreadRandomness}, + settings::Settings, +}; +use crate::{ + health::{GroupShape, routing_group_has_alternatives, team_model_has_alternatives}, + registry::DeploymentFilter, + select::SelectInputs, +}; + +/// The registry and settings one call reads, swapped atomically by `apply`. +pub(crate) struct RouterStateSnapshot { + pub registry: Arc, + pub settings: Arc, +} + +impl RouterStateSnapshot { + /// `_get_routing_context` for a pick of `name`: unless the request overrides the strategy, + /// a stale member map naming a group no longer configured raises `KeyError(group)`. + pub(crate) fn missing_routing_group( + &self, + request: &RouterRequest, + name: &str, + ) -> Result<(), Error> { + if strategy::request_override(request.routing_strategy.as_deref()).is_some() { + return Ok(()); + } + match self.registry.missing_member_group(name) { + Some(group) => Err(Error::MissingRoutingGroup { + group: group.to_owned(), + }), + None => Ok(()), + } + } +} + +pub(crate) struct Core { + host: Arc, + rng: Arc, + health: Health, + config: RouterConfig, + snapshot: ArcSwap, + writer: Mutex<()>, + retired: AtomicBool, +} + +impl Core { + /// Reject what this host cannot serve, before any request. + fn validate(&self, registry: &Registry, settings: &Settings) -> Result<(), ConfigError> { + let can_select = self.host.can_select(); + let strategies = std::iter::once(&settings.routing_strategy).chain( + registry + .routing_groups() + .iter() + .map(|group| &group.routing_strategy), + ); + for strategy in strategies { + if !strategy.is_native() && !can_select { + return Err(Unsupported::Strategy(strategy.clone()).into()); + } + } + if let Some(plugin) = settings.plugins.first() + && !can_select + { + return Err(Unsupported::Plugin(plugin.name.clone()).into()); + } + for deployment in registry.deployments(&DeploymentFilter::default()) { + self.validate_deployment(&deployment)?; + } + Ok(()) + } + + /// Reject a deployment whose provider this host cannot run a served op on. + fn validate_deployment(&self, deployment: &Deployment) -> Result<(), ConfigError> { + for op in &self.config.served_ops { + if !self.host.supports(op, deployment.provider()) { + return Err(Unsupported::Op { + op: op.clone(), + provider: deployment.provider().to_owned(), + } + .into()); + } + } + Ok(()) + } +} + +/// How a reported failure is recorded. +#[derive(Clone, Debug, Default)] +pub struct FailureReport { + /// Cool down for the deployment's own `cooldown_time` (then the provider's retry-after), + /// as the failure callback does, rather than the router default. + pub deployment_cooldown: bool, + /// The `cooldown_time` the failed call's own params carried (`model_info`, then + /// `litellm_params`), which wins over the provider's retry-after for a callback report. + pub cooldown_time: Option, + /// Count the failure only: its status is one no cooldown rule reads (an empty status). + pub count_only: bool, + /// The model group the request was for, whose fallbacks keep a lone deployment cooling. + pub requested_group: Option, + /// Cool down without counting the failure (`set_cooldown_deployments` alone). + pub uncounted: bool, + /// Also count the failed request against the deployment's per-minute requests + /// (`async_deployment_callback_on_failure`). + pub counts_request: bool, + /// The model the request's usage is keyed by (metadata's `deployment`, which a wildcard route + /// fills), when it is not the deployment's own. + pub usage_model: Option, + /// The failure callbacks of a logged failure; a report no callback makes keeps the default. + pub callbacks: OutcomeCallbacks, +} + +/// How a reported success is recorded. +#[derive(Clone, Debug, Default)] +pub struct SuccessReport { + /// See [`host::Output::tracks_usage`]. + pub tracks_usage: Option, + /// See [`FailureReport::usage_model`]. + pub usage_model: Option, + /// See [`FailureReport::callbacks`]. + pub callbacks: OutcomeCallbacks, +} + +/// Builds a [`Router`], with test seams. +pub struct RouterBuilder { + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + rng: Arc, + clock: Arc, +} + +impl RouterBuilder { + #[must_use] + pub fn randomness(self, rng: Arc) -> Self { + Self { rng, ..self } + } + + #[must_use] + pub fn clock(self, clock: Arc) -> Self { + Self { clock, ..self } + } + + pub fn build(self) -> Result { + let registry = Registry::build( + self.config.model_list.clone(), + self.dynamic.model_alias_map.clone(), + self.config.routing_groups.clone(), + )? + .with_member_groups(self.config.routing_group_members.clone()); + let settings = Settings::new(&self.config, Arc::new(self.dynamic)); + let core = Core { + health: Health::new(self.host.cache(), self.clock), + host: self.host, + rng: self.rng, + config: self.config, + snapshot: ArcSwap::from_pointee(RouterStateSnapshot { + registry: Arc::new(Registry::default()), + settings: Arc::new(settings.clone()), + }), + writer: Mutex::new(()), + retired: AtomicBool::new(false), + }; + core.validate(®istry, &settings)?; + core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: Arc::new(settings), + })); + Ok(Router { + core: Arc::new(core), + }) + } +} + +/// The router: the entry points `litellm.Router` delegates to. +#[derive(Clone)] +pub struct Router { + core: Arc, +} + +impl Router { + pub fn new( + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + ) -> Result { + Self::builder(config, dynamic, host).build() + } + + pub fn builder( + config: RouterConfig, + dynamic: RouterDynamicConfig, + host: Arc, + ) -> RouterBuilder { + RouterBuilder { + config, + dynamic, + host, + rng: Arc::new(ThreadRandomness), + clock: Arc::new(SystemClock), + } + } + + fn live(&self) -> Result<(), Error> { + if self.core.retired.load(Ordering::Acquire) { + return Err(Error::Retired); + } + Ok(()) + } + + /// Select, run the attempt, and retry or fall back until it succeeds or the policy is + /// exhausted. + pub async fn call(&self, op: Op, request: RouterRequest) -> Result { + self.live()?; + self.core.call(op, Arc::new(request)).await + } + + /// Resolve, filter and pick a deployment without calling it. + pub async fn select(&self, op: &Op, request: &RouterRequest) -> Result, Error> { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + routing_plugins::refuse_sync_pick( + snapshot.settings.routing_plugins, + &snapshot.settings.routing_strategy, + request, + )?; + let routed = self + .core + .pre_routed(PreRoute { + op, + request, + name: &request.model, + fallback_depth: 0, + retry: 0, + }) + .await?; + let name = match routed { + PreRouted::Model(name) => name, + PreRouted::Deployment(deployment) => return Ok(deployment), + }; + snapshot.missing_routing_group(request, &name)?; + let candidates = self + .core + .candidates(&snapshot, op, request, &name, &SelectInputs::default()) + .await?; + self.core.pick(&snapshot, op, request, &candidates).await + } + + /// The deployments a pick for `request` would choose among: resolved, team- and + /// cooldown-filtered, through the plugins and the lowest `order`. + pub async fn healthy( + &self, + op: &Op, + request: &RouterRequest, + ) -> Result { + self.live()?; + let snapshot = self.core.snapshot.load_full(); + let candidates = self + .core + .candidates( + &snapshot, + op, + request, + &request.model, + &SelectInputs::default(), + ) + .await?; + // `_core_async_get_healthy_deployments` raises when nothing is left. + if candidates.healthy.is_empty() { + return Err(Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + }); + } + Ok(candidates) + } + + pub fn registry(&self) -> Arc { + self.core.snapshot.load().registry.clone() + } + + pub fn settings(&self) -> Arc { + self.core.snapshot.load().settings.clone() + } + + pub fn health(&self) -> &Health { + &self.core.health + } + + /// Record an outcome the router did not see itself, such as a health check. Returns whether + /// the deployment was cooled down. + pub async fn report(&self, id: &str, outcome: Outcome) -> Result { + match outcome { + Outcome::Success { total_tokens } => self + .report_success(id, total_tokens as f64, 1.0, SuccessReport::default()) + .await + .map(|_| false), + Outcome::Failure(failure) => self + .report_failure(id, &failure, FailureReport::default()) + .await + .map(|cooldown| cooldown.is_some()), + } + } + + /// A success a logging callback saw (`deployment_callback_on_success`): counted for a + /// configured deployment, with its tokens and requests when it tracks usage. Returns the + /// success counter's key, or `None` for an unknown deployment. + pub async fn report_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + report: SuccessReport, + ) -> Result, Error> { + report + .callbacks + .record( + OutcomeWrite::Success, + self.reported_success(id, total_tokens, requests, &report), + ) + .await + } + + async fn reported_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + report: &SuccessReport, + ) -> Result, Error> { + let snapshot = self.core.snapshot.load_full(); + let Some(deployment) = snapshot.registry.deployment(id) else { + return Ok(None); + }; + let key = self.core.health.count_success(id).await?; + if report + .tracks_usage + .unwrap_or_else(|| deployment.tracks_usage()) + { + self.core + .health + .add_usage( + &deployment, + report.usage_model.as_deref(), + total_tokens, + requests, + ) + .await?; + } + Ok(Some(key)) + } + + /// `sync_deployment_callback_on_success`: count a success against `id`, configured or not, + /// and return the counter's key. + pub async fn count_success( + &self, + id: &str, + callbacks: OutcomeCallbacks, + ) -> Result, Error> { + callbacks + .record(OutcomeWrite::Success, async { + self.core.health.count_success(id).await.map(Some) + }) + .await + } + + /// A failure the router did not see itself: counted against `id` (known or not), then + /// cooled down by the rules (`record_failure`, `deployment_callback_on_failure`). Returns the + /// cooldown applied, if any. + pub async fn report_failure( + &self, + id: &str, + failure: &AttemptError, + report: FailureReport, + ) -> Result, Error> { + report + .callbacks + .record( + OutcomeWrite::Failure, + self.reported_failure(id, failure, &report), + ) + .await + } + + async fn reported_failure( + &self, + id: &str, + failure: &AttemptError, + report: &FailureReport, + ) -> Result, Error> { + if !report.uncounted && !failure.outside_deployment { + self.core.health.count_failure(id).await?; + } + let snapshot = self.core.snapshot.load_full(); + let Some(deployment) = snapshot.registry.deployment(id) else { + return Ok(None); + }; + if report.counts_request { + self.core + .health + .count_failed_request(&deployment, report.usage_model.as_deref()) + .await?; + } + if failure.outside_deployment { + return Ok(None); + } + // An empty status is no cooldown (`_is_cooldown_required`), unless the deployment's + // own policy names the error. + if report.count_only + && crate::health::deployment_allowed_fails( + deployment.model_info.allowed_fails_policy.as_ref(), + failure, + ) + .is_none() + { + return Ok(None); + } + let cooldown = if report.deployment_cooldown { + report + .cooldown_time + .filter(|seconds| *seconds >= 0.0) + .and_then(|seconds| std::time::Duration::try_from_secs_f64(seconds).ok()) + .or(failure.retry_after) + .unwrap_or(snapshot.settings.cooldown_time) + } else { + snapshot.settings.cooldown_time + }; + let shape = GroupShape { + single_deployment: snapshot + .registry + .group(&deployment.model_name) + .is_some_and(|group| group.len() == 1) + && !report + .requested_group + .as_deref() + .is_some_and(|group| routing_group_has_alternatives(&snapshot.registry, group)) + && !team_model_has_alternatives(&snapshot.registry, &deployment), + has_fallback_group: snapshot + .settings + .dynamic + .fallback_targets(&[report + .requested_group + .as_deref() + .unwrap_or(&deployment.model_name)]) + .is_some(), + }; + self.core + .health + .cool_down_after(&snapshot.settings, &deployment, shape, failure, cooldown) + .await + } + + /// Apply one change set atomically and return the new registry version. + pub fn apply(&self, changes: ChangeSet) -> Result { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let aliases = changes + .settings + .as_ref() + .map(|dynamic| dynamic.model_alias_map.clone()); + let registry = current.registry.apply(&changes, aliases)?; + let settings = match changes.settings { + Some(dynamic) => Settings::new(&self.core.config, Arc::new(dynamic)), + None => Settings::clone(¤t.settings), + }; + self.core.validate(®istry, &settings)?; + let version = registry.version(); + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: Arc::new(settings), + })); + Ok(version) + } + + /// `get_router_model_info` for one deployment, its cost-map reads asked of the host now + /// ([`model_info::router_model_info`]). + pub async fn router_model_info( + &self, + query: &model_info::RouterModelInfoQuery, + ) -> Result, Error> { + model_info::router_model_info(self.core.host.as_ref(), query).await + } + + /// `_set_model_group_info` for the deployments `query` names ([`group_info::model_group_info`]). + pub async fn model_group_info( + &self, + query: &group_info::GroupInfoQuery, + ) -> Result, Error> { + group_info::model_group_info(self.core.host.as_ref(), query).await + } + + /// `get_configured_token_limits` for deployment `id` ([`group_info::configured_token_limits`]). + pub async fn configured_token_limits( + &self, + id: serde_json::Value, + configured: &serde_json::Map, + ) -> Result<(Option, Option), Error> { + group_info::configured_token_limits(self.core.host.as_ref(), id, configured).await + } + + /// `get_model_listing_info` over a listed name's deployments ([`group_info::model_listing_info`]). + pub async fn model_listing_info( + &self, + deployments: &[serde_json::Value], + ) -> Result { + group_info::model_listing_info(self.core.host.as_ref(), deployments).await + } + + /// `get_deployment_model_info` for deployment `model_id` serving `model_name` + /// ([`model_info::deployment_model_info`]). + pub async fn deployment_model_info( + &self, + model_id: &str, + model_name: &serde_json::Value, + ) -> Result>, Error> { + model_info::deployment_model_info(self.core.host.as_ref(), model_id, model_name).await + } + + /// Build `entries` through the host ([`build::build`]), installing each deployment it + /// accepts as soon as it does, as `install` says: a replace empties the registry first (and + /// takes `set_model_list`'s `model_names` once every entry is built), an append or upsert + /// adds at the end. When a step fails, what was installed before it stays, as the Python + /// router keeps the deployments it added before the one that raised. + /// + /// The build installs into a registry of its own, indexed in place, and serves it once it + /// ends (or fails), so n deployments build in O(n). A write that lands while it runs (a step + /// answer that applies a changeset) is kept: what the build installed is added over it. + pub async fn build( + &self, + path: BuildPath, + entries: Vec>, + resolve_secrets: bool, + install: build::Install, + ) -> Result, Error> { + let model_names: Vec = entries + .iter() + .map(|entry| match entry.get("model_name") { + Some(serde_json::Value::String(name)) => name.clone(), + None | Some(serde_json::Value::Null) => "None".to_owned(), + Some(other) => other.to_string(), + }) + .collect(); + let replace = install == build::Install::Replace; + let base = self.core.snapshot.load_full().registry.clone(); + let mut working = if replace { + base.emptied() + } else { + Registry::clone(&base) + }; + let mut installed: Vec = Vec::new(); + let add = path == BuildPath::Add; + let core = &self.core; + let built = build::build( + core.host.as_ref(), + path, + entries, + resolve_secrets, + |deployment| { + working.push_built(deployment.spec.clone(), add)?; + installed.push(deployment.spec.clone()); + core.validate_deployment(working.last().expect("just installed")) + .map_err(Error::from) + }, + ) + .await; + if built.is_ok() && replace && path == BuildPath::Create { + working.set_model_names(model_names); + } + self.serve_built(&base, working, installed, add)?; + built + } + + /// Serve the registry a build made from `base`. When another write replaced `base` while + /// the build ran, the build's deployments are added over that write instead. + fn serve_built( + &self, + base: &Arc, + built: Registry, + installed: Vec, + add: bool, + ) -> Result<(), ConfigError> { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let registry = if Arc::ptr_eq(¤t.registry, base) { + built + } else { + let mut next = Registry::clone(¤t.registry); + for spec in installed { + next.push_built(spec, add)?; + } + next + }; + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: current.settings.clone(), + })); + Ok(()) + } + + /// Keep the providers `from`'s host resolved for the deployments this router serves too (a + /// router rebuilt over the same deployments for new static settings). + pub fn adopt_providers(&self, from: &Self) { + let _writer = self.core.writer.lock(); + let current = self.core.snapshot.load_full(); + let registry = current + .registry + .with_providers_of(&from.core.snapshot.load().registry); + self.core.snapshot.store(Arc::new(RouterStateSnapshot { + registry: Arc::new(registry), + settings: current.settings.clone(), + })); + } + + /// Stop serving: every later call and selection fails with [`Error::Retired`]. + pub fn retire(&self) { + self.core.retired.store(true, Ordering::Release); + } +} diff --git a/litellm-rust/crates/router-core/src/lookup.rs b/litellm-rust/crates/router-core/src/lookup.rs new file mode 100644 index 00000000000..39c235d6f7b --- /dev/null +++ b/litellm-rust/crates/router-core/src/lookup.rs @@ -0,0 +1,883 @@ +//! The registry lookups the host serves as `registry()` queries, answered over the deployments and +//! the host indexes ([`crate::host_index::HostIndexes`]) the way the Python router answers them. +//! +//! A lookup answers with [`Row`]s: positions in the deployment list (each with how the row renames +//! the deployment), wildcard routes, or a position the Python list could not be read at. The host +//! turns a position into the entry object it stores, so identity and in-place reads stay its own; +//! a [`Row::Raw`] is handed back for the host to index its list by, which raises exactly where the +//! Python router's own read raised (an index a caller wrote junk into). + +use std::sync::Arc; + +use serde_json::{Map, Value, json}; + +use crate::{ + host_index::{HostIndexes, python_str, truthy}, + registry::{Deployment, Registry}, +}; + +/// One row of a lookup's answer. +#[derive(Clone, Debug, PartialEq)] +pub enum Row { + /// The deployment at `position`, under `name` when the row renames it (an alias or a routing + /// group), as a routing group's member `member` of `group` when `group` is set (its row drops + /// the member's `access_groups` and takes the group's priority as its `order`). + At { + position: usize, + name: Option, + group: Option<(String, String)>, + }, + /// A wildcard route: a copy of a pattern router's deployment, under `name`. + Pattern { entry: Value, name: String }, + /// A position the list cannot be read at: the host indexes its list by it, and raises. + Raw(Value), + /// A position container that is not a list: the host iterates it as its list's positions. + RawPositions(Value), +} + +impl Row { + /// The row as the host reads it. + pub fn wire(&self) -> Value { + match self { + Self::At { + position, + name, + group, + } => { + let mut row = Map::new(); + row.insert("at".to_owned(), json!(position)); + if let Some(name) = name { + row.insert("name".to_owned(), json!(name)); + } + if let Some((group, member)) = group { + row.insert("group".to_owned(), json!(group)); + row.insert("member".to_owned(), json!(member)); + } + Value::Object(row) + } + Self::Pattern { entry, name } => json!({"pattern": entry, "name": name}), + Self::Raw(value) => json!({"raw": value}), + Self::RawPositions(value) => json!({"raw_positions": value}), + } + } +} + +/// An alias `model_list` lists: its name and the group it targets (`hidden` aliases already left +/// out by the host when the whole list is asked for). +#[derive(Clone, Debug, PartialEq, serde::Deserialize)] +pub struct AliasItem { + pub alias: String, + pub target: String, +} + +/// A routing group callable by its name (the host leaves out one a deployment or an alias +/// shadows): its name and member model groups. +#[derive(Clone, Debug, PartialEq, serde::Deserialize)] +pub struct CallableGroup { + pub name: String, + pub members: Vec, +} + +/// What `get_model_list` composes beyond the deployments: the aliases and callable routing groups +/// it lists, and the wildcard requests a name no deployment serves is routed by. +#[derive(Clone, Debug, Default, PartialEq, serde::Deserialize)] +#[serde(default)] +pub struct ListScope { + pub aliases: Vec, + pub groups: Vec, + /// `get_deployments_by_pattern`'s requests: the name, then the name under its provider. + pub wildcard: Vec, + /// Only one part of the list: `"aliases"` (`_core_get_model_list_from_model_alias`) or + /// `"groups"` (`_core_get_model_list_from_routing_groups`). + pub only: Option, +} + +/// A row read for a lookup's filters: the deployment's fields as the row shows them. +struct View<'a> { + name: &'a str, + info: &'a Map, + params: &'a Map, + strips_access_groups: bool, +} + +/// The lookups over one registry version. +pub struct Lookup<'a> { + deployments: &'a [Arc], + host: &'a HostIndexes, +} + +impl<'a> Lookup<'a> { + pub fn new(registry: &'a Registry) -> Self { + Self { + deployments: registry.deployments_from(0), + host: registry.host_indexes(), + } + } + + /// `list[value]` as Python reads it: an int (or bool) in range, negative from the end. + fn position(&self, value: &Value) -> Option { + let len = i64::try_from(self.deployments.len()).ok()?; + let index = match value { + Value::Bool(flag) => i64::from(*flag), + Value::Number(number) => number.as_i64()?, + _ => return None, + }; + let index = if index < 0 { len + index } else { index }; + (0..len) + .contains(&index) + .then(|| usize::try_from(index).ok()) + .flatten() + } + + /// Every position `held` (what an index holds) names, read before any is filtered, as the + /// Python router reads them into a tuple first: `Err` with the row that raises. + fn positions(&self, held: &Value) -> Result, Row> { + let items = match held { + Value::Array(items) => items, + // Iterating an empty string or dict reads no position; anything else raises. + Value::String(text) if text.is_empty() => return Ok(Vec::new()), + Value::Object(map) if map.is_empty() => return Ok(Vec::new()), + _ => return Err(Row::RawPositions(held.clone())), + }; + items + .iter() + .map(|item| self.position(item).ok_or_else(|| Row::Raw(item.clone()))) + .collect() + } + + fn view(&self, position: usize) -> View<'a> { + let deployment = &self.deployments[position]; + View { + name: &deployment.model_name, + info: &deployment.model_info.raw, + params: &deployment.raw_params, + strips_access_groups: false, + } + } + + fn row_view<'r>(&self, row: &'r Row) -> Option> + where + 'a: 'r, + { + match row { + Row::At { + position, + name, + group, + } => { + let base = self.view(*position); + Some(View { + name: name.as_deref().unwrap_or(base.name), + strips_access_groups: group.is_some(), + ..base + }) + } + Row::Pattern { entry, name } => Some(View { + name, + info: entry + .get("model_info") + .and_then(Value::as_object) + .unwrap_or(empty()), + params: entry + .get("litellm_params") + .and_then(Value::as_object) + .unwrap_or(empty()), + strips_access_groups: false, + }), + Row::Raw(_) | Row::RawPositions(_) => None, + } + } + + // -- deployments by name ------------------------------------------------------------------- + + /// `_core_get_all_deployments`: the team's deployments under public name `model_name`, else + /// the group's, else (for a team) every deployment the name includes; each under `rename`. + pub fn all(&self, model_name: &str, team_id: Option<&str>, rename: Option<&str>) -> Vec { + let rows = |positions: Vec| -> Vec { + positions + .into_iter() + .filter(|position| should_include(model_name, &self.view(*position), team_id)) + .map(|position| Row::At { + position, + name: rename.map(str::to_owned), + group: None, + }) + .collect() + }; + if let Some(team) = team_id + && let Some(held) = self.host.team_named(team, model_name) + { + match self.positions(held) { + Ok(positions) => { + let found = rows(positions); + if !found.is_empty() { + return found; + } + } + Err(raw) => return vec![raw], + } + } + if let Some(held) = self.host.named(model_name) { + return match self.positions(held) { + Ok(positions) => rows(positions), + Err(raw) => vec![raw], + }; + } + if team_id.is_some() { + return rows((0..self.deployments.len()).collect()); + } + Vec::new() + } + + /// `_team_deployments_across_teams`: every team's deployments under public name `model`, and + /// how many distinct teams own them. + pub fn across_teams(&self, model: &str) -> (Vec, usize) { + let mut rows = Vec::new(); + for (_, public, held) in &self.host.team_model { + if public != model { + continue; + } + match self.positions(held) { + Ok(positions) => rows.extend(positions.into_iter().map(|position| Row::At { + position, + name: None, + group: None, + })), + Err(raw) => return (vec![raw], 0), + } + } + let mut teams: Vec<&Value> = Vec::new(); + for row in &rows { + if let Some(view) = self.row_view(row) + && let Some(team) = view.info.get("team_id").filter(|team| !team.is_null()) + && !teams.contains(&team) + { + teams.push(team); + } + } + let count = teams.len(); + (rows, count) + } + + /// `_core_config_deployments`: the deployments a config declared (no `db_model: true`). + pub fn config(&self) -> Vec { + (0..self.deployments.len()) + .filter(|position| { + self.view(*position).info.get("db_model") != Some(&Value::Bool(true)) + }) + .map(|position| Row::At { + position, + name: None, + group: None, + }) + .collect() + } + + /// `_core_get_model_ids`: the ids of the group's deployments, or (no group) of the id map's + /// entries, without team deployments when `exclude_team_models`. `Err` with the row that raises. + pub fn model_ids( + &self, + model_name: Option<&str>, + exclude_team_models: bool, + ) -> Result, Row> { + let kept = |view: &View<'_>| { + view.info.contains_key("id") + && !(exclude_team_models && view.info.get("team_id").is_some_and(truthy)) + }; + if let Some(name) = model_name { + let Some(held) = self.host.named(name) else { + return Ok(Vec::new()); + }; + return Ok(self + .positions(held)? + .into_iter() + .map(|position| self.view(position)) + .filter(kept) + .map(|view| python_str(&view.info["id"])) + .collect()); + } + let mut ids = Vec::new(); + for (id, held) in &self.host.id_map { + let position = self.position(held).ok_or_else(|| Row::Raw(held.clone()))?; + if kept(&self.view(position)) { + ids.push(id.clone()); + } + } + Ok(ids) + } + + // -- the model list -------------------------------------------------------------------------- + + /// `_core_get_model_list`: the name's deployments, its alias rows and routing group rows, its + /// wildcard routes when nothing else serves it, or (no name) every alias and group row and + /// then every deployment. + pub fn model_list( + &self, + model_name: Option<&str>, + team_id: Option<&str>, + scope: &ListScope, + ) -> Vec { + let only = scope.only.as_deref(); + let mut rows = Vec::new(); + if let Some(name) = model_name + && only.is_none() + { + rows.extend(self.all(name, team_id, None)); + } + let aliases: &[AliasItem] = if only == Some("groups") { + &[] + } else { + &scope.aliases + }; + for alias in aliases { + match scope.groups.iter().find(|group| group.name == alias.target) { + Some(group) => { + rows.extend(self.group_rows(group).into_iter().map(|row| match row { + Row::At { + position, group, .. + } => Row::At { + position, + name: Some(alias.alias.clone()), + group, + }, + other => other, + })) + } + None => rows.extend(self.all(&alias.target, None, Some(&alias.alias))), + } + } + match (model_name, only) { + (_, Some("aliases")) => {} + (Some(name), _) => { + if let Some(group) = scope.groups.iter().find(|group| group.name == name) { + rows.extend(self.group_rows(group)); + } + } + (None, _) => { + for group in &scope.groups { + rows.extend(self.group_rows(group)); + } + } + } + if only.is_some() { + return rows; + } + if let Some(name) = model_name + && rows.is_empty() + { + rows.extend(self.wildcard(name, team_id, &scope.wildcard)); + } + if model_name.is_none() { + rows.extend((0..self.deployments.len()).map(|position| Row::At { + position, + name: None, + group: None, + })); + } + rows + } + + /// The whole list's rows, or the first row the host's read of the list raises at. + fn readable(&self, rows: Vec) -> Result, Row> { + match rows + .iter() + .find(|row| matches!(row, Row::Raw(_) | Row::RawPositions(_))) + { + Some(raw) => Err(raw.clone()), + None => Ok(rows), + } + } + + /// A callable routing group's rows: each member's deployments under the group's name. + fn group_rows(&self, group: &CallableGroup) -> Vec { + let mut rows = Vec::new(); + for member in &group.members { + for row in self.all(member, None, Some(&group.name)) { + rows.push(match row { + Row::At { position, name, .. } => Row::At { + position, + name, + group: Some((group.name.clone(), member.clone())), + }, + other => other, + }); + } + } + rows + } + + /// `_get_wildcard_deployments`: the global router's routes for the requests, then the team's. + fn wildcard(&self, name: &str, team_id: Option<&str>, requests: &[String]) -> Vec { + let route = |router: &crate::host_index::PatternRouter| -> Vec { + requests + .iter() + .find_map(|request| { + router + .route(request, None) + .filter(|routed| !routed.is_empty()) + }) + .unwrap_or_default() + }; + let mut routed = route(&self.host.patterns); + if let Some(team) = team_id + && let Some((_, router)) = self.host.team_patterns.iter().find(|(t, _)| t == team) + { + routed.extend(route(router)); + } + routed + .into_iter() + .map(|entry| Row::Pattern { + entry, + name: name.to_owned(), + }) + .collect() + } + + /// The deployments `team_id`'s own wildcard pattern routes `name` to + /// (`_core_team_pattern_routers`, keyed by `team_public_model_name`). + pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec> { + let Some((_, router)) = self.host.team_patterns.iter().find(|(t, _)| t == team_id) else { + return Vec::new(); + }; + router + .route(name, None) + .unwrap_or_default() + .iter() + .filter_map(|entry| entry.get("model_info")?.get("id").map(python_str)) + .filter_map(|id| self.deployments.iter().find(|d| d.id == id).cloned()) + .collect() + } + + /// `_core_get_model_names`: per row of the whole list, a team deployment's public name for + /// its own team, else the row's name. + pub fn model_names( + &self, + team_id: Option<&str>, + scope: &ListScope, + ) -> Result, Row> { + let rows = self.readable(self.model_list(None, None, scope))?; + let mut names = Vec::new(); + for row in &rows { + let Some(view) = self.row_view(row) else { + continue; + }; + match view.info.get("team_id").filter(|team| truthy(team)) { + Some(team) => { + if team_id.is_some_and(|id| team == &Value::String(id.to_owned())) + && let Some(public) = view + .info + .get("team_public_model_name") + .filter(|public| truthy(public)) + { + names.push(python_str(public)); + } + } + None => names.push(view.name.to_owned()), + } + } + Ok(names) + } + + /// `_core_get_model_access_groups`: access group to the names of the rows that declare it, + /// in the order they appear. + pub fn access_groups( + &self, + model_name: Option<&str>, + access_group: Option<&str>, + team_id: Option<&str>, + scope: &ListScope, + ) -> Result)>, Row> { + let mut groups: Vec<(String, Vec)> = Vec::new(); + for row in self.readable(self.model_list(model_name, team_id, scope))? { + let Some(view) = self.row_view(&row) else { + continue; + }; + if view.info.is_empty() || view.strips_access_groups { + continue; + } + // `access_groups or []`, iterated: a list's items, a string's characters. + let declared: Vec = match view.info.get("access_groups") { + Some(Value::Array(items)) => items.iter().map(python_str).collect(), + Some(Value::String(text)) => text.chars().map(String::from).collect(), + _ => continue, + }; + for group in declared { + if access_group.is_some_and(|wanted| wanted != group) { + continue; + } + match groups.iter_mut().find(|(held, _)| *held == group) { + Some((_, names)) => names.push(view.name.to_owned()), + None => groups.push((group, vec![view.name.to_owned()])), + } + } + } + Ok(groups) + } + + /// `_core_resolve_model_name_from_model_id`'s scan of the whole list: whether it lists any + /// row, and the name of the first row whose `litellm_params.model` is `model_id`, or ends in + /// `/model_id` or `:model_id`. + pub fn name_serving( + &self, + model_id: &str, + scope: &ListScope, + ) -> Result<(bool, Option), Row> { + let rows = self.readable(self.model_list(None, None, scope))?; + let (slash, colon) = (format!("/{model_id}"), format!(":{model_id}")); + let name = rows.iter().find_map(|row| { + let view = self.row_view(row)?; + let model = view.params.get("model").and_then(Value::as_str)?; + (model == model_id || model.ends_with(&slash) || model.ends_with(&colon)) + .then_some(view.name) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + }); + Ok((!rows.is_empty(), name)) + } + + // -- teams and credentials ------------------------------------------------------------------- + + /// `_core_team_model_has_alternatives`: whether the team's public name of deployment `id` has + /// another unpaused deployment of that team. + pub fn team_model_has_alternatives(&self, id: &str) -> Result { + let Some(held) = self.host.id_entry(id) else { + return Ok(false); + }; + let position = self.position(held).ok_or_else(|| Row::Raw(held.clone()))?; + let info = self.view(position).info; + let (Some(Value::String(team)), Some(Value::String(public))) = + (info.get("team_id"), info.get("team_public_model_name")) + else { + return Ok(false); + }; + let Some(siblings) = self.host.team_named(team, public) else { + return Ok(false); + }; + let unpaused = self + .positions(siblings)? + .into_iter() + .filter(|position| self.view(*position).info.get("blocked") != Some(&Value::Bool(true))) + .count(); + Ok(unpaused > 1) + } + + /// `_get_model_group_deployment_usable_by_team`: the first deployment of the group the team + /// may use. + pub fn group_deployment_usable_by_team( + &self, + model_group: &str, + team_id: Option<&str>, + ) -> Result, Row> { + // `.get(name) or ()`: nothing held, or a falsy value, reads no position. + let Some(held) = self.host.named(model_group).filter(|held| truthy(held)) else { + return Ok(None); + }; + let Value::Array(items) = held else { + return Err(Row::RawPositions(held.clone())); + }; + for item in items { + let position = self.position(item).ok_or_else(|| Row::Raw(item.clone()))?; + if usable_by_team(&self.view(position), team_id) { + return Ok(Some(position)); + } + } + Ok(None) + } + + /// `_get_team_public_name_deployment`: the first deployment of the team's public name. + pub fn team_public_name_deployment( + &self, + model_id: &str, + team_id: &str, + ) -> Result, Row> { + match self.host.team_named(team_id, model_id) { + Some(Value::Array(items)) => match items.first() { + Some(item) => self + .position(item) + .map(Some) + .ok_or_else(|| Row::Raw(item.clone())), + None => Ok(None), + }, + Some(held) if truthy(held) => Err(Row::RawPositions(held.clone())), + Some(_) | None => Ok(None), + } + } + + // -- routable deployments -------------------------------------------------------------------- + + /// `_routable_deployments`: the routable group's deployments for the team (those it may use + /// when any), without paused ones. + pub fn routable(&self, model_group: &str, team_id: Option<&str>) -> Vec { + let named = self.all(model_group, team_id, None); + if named + .iter() + .any(|row| matches!(row, Row::Raw(_) | Row::RawPositions(_))) + { + return named; + } + let usable: Vec<&Row> = named + .iter() + .filter(|row| { + self.row_view(row) + .is_some_and(|view| usable_by_team(&view, team_id)) + }) + .collect(); + let pool: Vec<&Row> = if usable.is_empty() { + named.iter().collect() + } else { + usable + }; + pool.into_iter() + .filter(|row| { + self.row_view(row) + .is_some_and(|view| view.info.get("blocked") != Some(&Value::Bool(true))) + }) + .cloned() + .collect() + } + + /// `_core_get_candidate_model_ids_for_route`'s ids of `rows`, in first-seen order. + pub fn ids_of(&self, rows: &[Row]) -> Result, Row> { + let mut ids: Vec = Vec::new(); + for row in rows { + let view = match row { + Row::Raw(_) | Row::RawPositions(_) => return Err(row.clone()), + _ => self.row_view(row).expect("a readable row"), + }; + if let Some(id) = view.info.get("id").filter(|id| !id.is_null()) { + let id = python_str(id); + if !ids.contains(&id) { + ids.push(id); + } + } + } + Ok(ids) + } + + /// `_core_get_candidate_model_ids_for_route` once the alias resolved `resolved`: the + /// members' ids when it names a callable routing group (`members`), else, for a name that is + /// no model group, the team's deployments under it or its wildcard routes, else the group's. + pub fn candidate_ids( + &self, + resolved: &str, + team_id: Option<&str>, + members: Option<&[String]>, + requests: &[String], + ) -> Result, Row> { + if let Some(members) = members { + let mut ids: Vec = Vec::new(); + for member in members { + for id in self.ids_of(&self.all(member, team_id, None))? { + if !ids.contains(&id) { + ids.push(id); + } + } + } + return Ok(ids); + } + let named = Value::String(resolved.to_owned()); + if !self.host.model_names.contains(&named) { + if team_id.is_some() { + let team_rows = self.all(resolved, team_id, None); + if !team_rows.is_empty() { + return self.ids_of(&team_rows); + } + } + let wildcard = self.wildcard_route(resolved, team_id, requests); + if !wildcard.is_empty() { + return self.ids_of(&wildcard); + } + } + self.ids_of(&self.all(resolved, team_id, None)) + } + + /// `_core_wildcard_route_deployments`: the team router's routes for `model`, else the global + /// router's. + pub fn wildcard_route( + &self, + model: &str, + team_id: Option<&str>, + requests: &[String], + ) -> Vec { + let route = |router: &crate::host_index::PatternRouter| -> Vec { + requests + .iter() + .find_map(|request| { + router + .route(request, None) + .filter(|routed| !routed.is_empty()) + }) + .unwrap_or_default() + }; + let team_owned = team_id + .and_then(|team| self.host.team_patterns.iter().find(|(t, _)| t == team)) + .map(|(_, router)| route(router)) + .unwrap_or_default(); + let routed = if team_owned.is_empty() { + route(&self.host.patterns) + } else { + team_owned + }; + routed + .into_iter() + .map(|entry| Row::Pattern { + entry, + name: model.to_owned(), + }) + .collect() + } +} + +/// `_core_should_include_deployment`: the team's deployment under its public name, or a +/// deployment of the group the team may use. +fn should_include(model_name: &str, view: &View<'_>, team_id: Option<&str>) -> bool { + let team_value = team_id.map(|team| Value::String(team.to_owned())); + let model_team = view.info.get("team_id"); + if let Some(team) = &team_value + && model_team == Some(team) + && view.info.get("team_public_model_name") == Some(&Value::String(model_name.to_owned())) + { + return true; + } + if view.name == model_name { + return team_value.is_none() + || model_team.is_none_or(Value::is_null) + || model_team == team_value.as_ref(); + } + false +} + +/// `deployment_usable_by_team`: a deployment no team owns, or one the team owns. +fn usable_by_team(view: &View<'_>, team_id: Option<&str>) -> bool { + match view.info.get("team_id").filter(|team| !team.is_null()) { + None => true, + Some(owner) => team_id.is_some_and(|team| owner == &Value::String(team.to_owned())), + } +} + +fn empty() -> &'static Map { + static EMPTY: std::sync::OnceLock> = std::sync::OnceLock::new(); + EMPTY.get_or_init(Map::new) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn registry(entries: Value) -> Registry { + let mut registry = Registry::default(); + for entry in entries.as_array().unwrap() { + let spec: DeploymentSpec = serde_json::from_value(entry.clone()).unwrap(); + registry.push_built(spec, false).unwrap(); + } + registry + } + + #[test] + fn a_teams_wildcard_routes_only_the_names_its_public_pattern_matches() { + let registry = registry(json!([ + {"model_name": "t1-w", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw1", "team_id": "T1", "team_public_model_name": "tw-*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "gp"}}, + ])); + let lookup = Lookup::new(®istry); + let ids = |name: &str, team: &str| -> Vec { + let routed = lookup.team_wildcard(name, team); + routed.iter().map(|d| d.id.clone()).collect() + }; + assert_eq!(ids("tw-gpt-4o", "T1"), ["tw1"]); + assert!(ids("gpt-4o", "T1").is_empty()); + assert!(ids("tw-gpt-4o", "T2").is_empty()); + } + + fn entry(name: &str, id: &str, info: Value) -> Value { + let mut info = info.as_object().cloned().unwrap_or_default(); + info.insert("id".to_owned(), json!(id)); + json!({"model_name": name, "litellm_params": {"model": format!("openai/{id}")}, "model_info": info}) + } + + fn at(position: usize) -> Row { + Row::At { + position, + name: None, + group: None, + } + } + + #[test] + fn all_prefers_the_teams_public_name_then_the_group() { + let registry = registry(json!([ + entry("gpt", "a", json!({})), + entry( + "team-gpt", + "t", + json!({"team_id": "T", "team_public_model_name": "gpt"}) + ), + entry("gpt", "b", json!({"team_id": "U"})), + ])); + let lookup = Lookup::new(®istry); + assert_eq!(lookup.all("gpt", Some("T"), None), vec![at(1)]); + assert_eq!(lookup.all("gpt", Some("V"), None), vec![at(0)]); + assert_eq!(lookup.all("gpt", None, None), vec![at(0), at(2)]); + assert_eq!(lookup.all("missing", None, None), vec![]); + } + + #[test] + fn a_junk_position_is_handed_back_to_raise() { + let mut registry = registry(json!([entry("gpt", "a", json!({}))])); + let mut model_name = registry.host_indexes().model_name.clone(); + model_name.insert("gpt".to_owned(), json!([0, "x"])); + let changes = crate::registry::ChangeSet { + host_index: Some(crate::host_index::IndexWrite { + model_name: Some(model_name), + ..Default::default() + }), + ..Default::default() + }; + registry = registry.apply(&changes, None).unwrap(); + let lookup = Lookup::new(®istry); + assert_eq!(lookup.all("gpt", None, None), vec![Row::Raw(json!("x"))]); + } + + #[test] + fn the_model_list_lists_aliases_and_groups_then_every_deployment() { + let registry = registry(json!([ + entry("gpt", "a", json!({"access_groups": ["g1"]})), + entry("claude", "b", json!({})), + ])); + let lookup = Lookup::new(®istry); + let scope = ListScope { + aliases: vec![AliasItem { + alias: "fast".to_owned(), + target: "gpt".to_owned(), + }], + groups: vec![CallableGroup { + name: "pool".to_owned(), + members: vec!["gpt".to_owned(), "claude".to_owned()], + }], + wildcard: Vec::new(), + only: None, + }; + let rows = lookup.model_list(None, None, &scope); + assert_eq!(rows.len(), 5); + assert_eq!( + rows[0], + Row::At { + position: 0, + name: Some("fast".to_owned()), + group: None + } + ); + assert_eq!( + lookup.model_names(None, &scope).unwrap(), + ["fast", "pool", "pool", "gpt", "claude"] + ); + assert_eq!( + lookup.access_groups(None, None, None, &scope).unwrap(), + vec![("g1".to_owned(), vec!["fast".to_owned(), "gpt".to_owned()])] + ); + } +} diff --git a/litellm-rust/crates/router-core/src/mock.rs b/litellm-rust/crates/router-core/src/mock.rs new file mode 100644 index 00000000000..898782e6203 --- /dev/null +++ b/litellm-rust/crates/router-core/src/mock.rs @@ -0,0 +1,44 @@ +//! The request's `mock_testing_*` flags: a failure the router raises itself, without calling a +//! deployment (`_handle_mock_testing_fallbacks` and `_handle_mock_testing_rate_limit_error`). + +use serde::Deserialize; + +use crate::error::{AttemptError, FailureKind}; + +/// Which mock failure a request asks for. +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::IntoStaticStr, strum::EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum MockFailure { + /// `mock_testing_fallbacks`: an `InternalServerError` before the first level's retries. + Fallbacks, + /// `mock_testing_context_fallbacks`: a `ContextWindowExceededError`, likewise. + ContextWindow, + /// `mock_testing_content_policy_fallbacks`: a `ContentPolicyViolationError`, likewise. + ContentPolicy, + /// `mock_testing_rate_limit_error`: a `RateLimitError` in place of every level's first + /// attempt, which the retry loop handles like a failed attempt. + RateLimit, +} + +impl MockFailure { + /// Whether the failure stands in for the first level's retry loop (raised before its retries). + pub fn before_retries(self) -> bool { + !matches!(self, Self::RateLimit) + } + + /// The failure as the rules see it. + pub fn failure(self, num_retries: Option) -> AttemptError { + let (kind, status) = match self { + Self::Fallbacks => (FailureKind::InternalServer, 500), + Self::ContextWindow => (FailureKind::ContextWindowExceeded, 400), + Self::ContentPolicy => (FailureKind::ContentPolicyViolation, 400), + Self::RateLimit => (FailureKind::RateLimit, 429), + }; + let mut failure = AttemptError::new(kind, Some(status)); + failure.num_retries = num_retries; + failure + } +} diff --git a/litellm-rust/crates/router-core/src/model_info.rs b/litellm-rust/crates/router-core/src/model_info.rs new file mode 100644 index 00000000000..861388f36ba --- /dev/null +++ b/litellm-rust/crates/router-core/src/model_info.rs @@ -0,0 +1,716 @@ +//! Model info: what the router reports a deployment can do and costs, merged from the process +//! cost map, what the background refresh discovered, and the deployment's own `model_info`. +//! +//! The merge rules are the Python router core's (`get_router_model_info`, +//! `get_deployment_model_info`); every cost-map read is a [`Host::model_info`] lookup asked while +//! the query runs, so a price reload or a registration the host made since the router was built is +//! what the next query reads. The host keeps choosing the keys a lookup tries (`get_model_info`); +//! the core chooses which lookups a query makes. +//! +//! Values the host cannot spell as JSON travel as opaque tokens ([`is_opaque`]): the merge moves +//! them between layers without reading them, and the host restores them. + +use serde_json::{Map, Number, Value}; + +use crate::{ + error::Error, + host::{Host, LogLevel, ModelInfoLookup}, +}; + +/// The key the host wraps a value it cannot spell as JSON in. +pub const OPAQUE_KEY: &str = "__litellm_python_value__"; + +/// A value the host passed as an opaque token: the merge moves it, never reads it. +pub fn is_opaque(value: &Value) -> bool { + matches!(value, Value::Object(map) if map.len() == 1 && map.contains_key(OPAQUE_KEY)) +} + +/// One `get_router_model_info` query, as the host reads it off the deployment. +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(default)] +pub struct RouterModelInfoQuery { + /// `model_info.id`. + pub deployment_id: Value, + /// `model_info.base_model`, then `litellm_params.base_model`: the first that is not `None` + /// names the model whose limits and costs the deployment reports. + pub model_info_base_model: Value, + pub params_base_model: Value, + /// The validated `litellm_params.model`. + pub params_model: String, + /// `base_model` among the validated params' extra fields. + pub extra_base_model: Value, + /// The name the caller asked for, which a wildcard deployment matches. + pub received_model_name: String, +} + +/// A merged model info, and the keys whose values are the deployment's own (the last layer). +#[derive(Clone, Debug, PartialEq)] +pub struct MergedModelInfo { + pub info: Map, + pub user_keys: Vec, +} + +pub(crate) fn raised(exception: &'static str, detail: impl Into) -> Error { + Error::ModelInfo { + exception, + detail: detail.into(), + } +} + +pub(crate) fn malformed(lookup: &str) -> Error { + raised( + "ValueError", + format!("the host answered a `{lookup}` lookup malformed"), + ) +} + +/// Python truthiness of a JSON value. +pub(crate) fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_some_and(|n| n != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(map) => is_opaque(value) || !map.is_empty(), + } +} + +/// `(value or 0) > 0`, for a value that is a number (anything else reads as not positive). +fn positive_or_zero(value: Option<&Value>) -> bool { + value + .filter(|value| truthy(value)) + .and_then(Value::as_f64) + .is_some_and(|n| n > 0.0) +} + +pub(crate) async fn lookup(host: &dyn Host, lookup: ModelInfoLookup) -> Result { + host.model_info(&lookup).await +} + +/// `get_model_info(model=...)`: the row, or `None` when the lookup tolerates a failure and failed. +async fn catalog_row( + host: &dyn Host, + model: Value, + tolerate: bool, +) -> Result>, Error> { + match lookup( + host, + ModelInfoLookup::ModelInfo { + model, + tolerate, + exact: false, + }, + ) + .await? + { + Value::Null => Ok(None), + Value::Object(row) if tolerate && row.get("failed") == Some(&Value::Bool(true)) => Ok(None), + Value::Object(row) => Ok(Some(row)), + _ => Err(malformed("model_info")), + } +} + +/// `model_cost.get(key)`: `None` when the key is absent, else the entry as stored. +async fn cost_entry(host: &dyn Host, key: &str, process: bool) -> Result, Error> { + match lookup( + host, + ModelInfoLookup::CostEntry { + key: key.to_owned(), + process, + }, + ) + .await? + { + Value::Null => Ok(None), + Value::Object(mut answer) => Ok(Some(answer.remove("entry").unwrap_or(Value::Null))), + _ => Err(malformed("cost_entry")), + } +} + +pub(crate) async fn discovered(host: &dyn Host, id: Value) -> Result, Error> { + match lookup(host, ModelInfoLookup::Discovered { id }).await? { + Value::Null => Ok(Map::new()), + Value::Object(limits) => Ok(limits), + _ => Err(malformed("discovered")), + } +} + +async fn log(host: &dyn Host, level: LogLevel, message: String) -> Result<(), Error> { + lookup(host, ModelInfoLookup::Log { level, message }) + .await + .map(|_| ()) +} + +/// `_provider_model_id`: the model id a cost-map key or base model names, bedrock base model ids +/// normalized. +async fn provider_model_id(host: &dyn Host, name: &str, provider: &str) -> Result { + let model_id = name.rsplit('/').next().unwrap_or(name).to_owned(); + if !provider.starts_with("bedrock") { + return Ok(model_id); + } + match lookup(host, ModelInfoLookup::BedrockBase { model_id }).await? { + Value::String(base) => Ok(base), + _ => Err(malformed("bedrock_base")), + } +} + +/// `_exact_base_model_max_input_tokens`: the base model's own window, unless its row came from a +/// fallback-generalization rule or it is unmapped. +async fn exact_base_model_max_input_tokens( + host: &dyn Host, + base_model: &str, +) -> Result, Error> { + let answer = lookup( + host, + ModelInfoLookup::ModelInfo { + model: Value::String(base_model.to_owned()), + tolerate: true, + exact: true, + }, + ) + .await?; + let Value::Object(mut answer) = answer else { + return Ok(None); + }; + if answer.get("failed") == Some(&Value::Bool(true)) + || answer.get("generalized") == Some(&Value::Bool(true)) + { + return Ok(None); + } + let Some(Value::Object(row)) = answer.remove("row") else { + return Ok(None); + }; + Ok(row + .get("max_input_tokens") + .filter(|tokens| !tokens.is_null()) + .cloned()) +} + +/// `_base_model_widened_window`: a base model naming the same model as the route can only widen +/// its context window. +async fn base_model_widened_window( + host: &dyn Host, + route: &Map, + base_model: Option<&str>, + provider: &str, +) -> Result, Error> { + let Some(route_max) = route + .get("max_input_tokens") + .filter(|tokens| !tokens.is_null()) + else { + return Ok(None); + }; + let Some(base_model) = base_model else { + return Ok(None); + }; + let route_key = match route.get("key") { + Some(Value::String(key)) => key.clone(), + Some(_) => return Err(raised("AttributeError", "key")), + None => return Err(raised("KeyError", "key")), + }; + let route_id = provider_model_id(host, &route_key, provider).await?; + let base_id = provider_model_id(host, base_model, provider).await?; + let unversioned = base_id + .split_once('@') + .map_or(base_id.as_str(), |(id, _)| id); + if route_id != base_id && route_id != unversioned { + return Ok(None); + } + let Some(base_max) = exact_base_model_max_input_tokens(host, base_model).await? else { + return Ok(None); + }; + match (base_max.as_f64(), route_max.as_f64()) { + (Some(base), Some(route)) if base > route => Ok(Some(base_max)), + (Some(_), Some(_)) => Ok(None), + _ => Err(raised( + "TypeError", + "'<=' not supported between the base model's and the route's max_input_tokens", + )), + } +} + +/// `_declared_base_model`: a `base_model` that is a string. +fn declared_base_model(value: &Value) -> Option<&str> { + value.as_str() +} + +/// `get_router_model_info` after the host validated the deployment's params: the cost-map row of +/// the model the deployment serves, widened by its base model, overlaid with what the background +/// refresh discovered and then with every value the deployment sets itself. `None` when the +/// lookup answers `None`; an unmapped model raises as the lookup raises. +pub async fn router_model_info( + host: &dyn Host, + query: &RouterModelInfoQuery, +) -> Result, Error> { + let base_model = if query.model_info_base_model.is_null() { + query.params_base_model.clone() + } else { + query.model_info_base_model.clone() + }; + let mut model = base_model.clone(); + + let (resolved_model, provider) = match lookup( + host, + ModelInfoLookup::Provider { + model: query.params_model.clone(), + }, + ) + .await? + { + Value::Array(pair) => match pair.as_slice() { + [Value::String(model), Value::String(provider), ..] => { + (model.clone(), provider.clone()) + } + _ => return Err(malformed("provider")), + }, + _ => return Err(malformed("provider")), + }; + + if provider == "azure" && base_model.is_null() { + // Router init registers every deployment name into the cost map as a zeroed stub, so + // membership alone cannot tell a resolvable name apart; it needs usable limits or costs. + let fallback_key = if resolved_model.starts_with("azure/") { + resolved_model.clone() + } else { + format!("azure/{resolved_model}") + }; + let resolves = match cost_entry(host, &fallback_key, false).await? { + Some(Value::Object(entry)) if !is_opaque(&Value::Object(entry.clone())) => { + positive_or_zero(entry.get("max_input_tokens")) + || positive_or_zero(entry.get("input_cost_per_token")) + } + _ => false, + }; + if resolves { + log( + host, + LogLevel::Debug, + format!( + "Azure deployment '{resolved_model}' has no base_model set; using '{fallback_key}' from the model cost map for max tokens, cost tracking, etc." + ), + ) + .await?; + } else { + log( + host, + LogLevel::Error, + format!( + "Could not identify azure model '{resolved_model}'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models" + ), + ) + .await?; + } + } else if provider != "azure" { + model = Value::String(resolved_model.clone()); + if resolved_model.contains('*') { + // Only a wildcard deployment asks which model its pattern routed the name to. + if let Value::Array(routed) = lookup( + host, + ModelInfoLookup::PatternModels { + name: query.received_model_name.clone(), + }, + ) + .await? + && let Some(routed) = routed + .iter() + .find(|routed| routed.get("id").unwrap_or(&Value::Null) == &query.deployment_id) + { + model = routed.get("model").cloned().unwrap_or(Value::Null); + } + } + } + + if model.is_null() { + // An azure deployment without a base_model is looked up by its own model. + model = Value::String(resolved_model); + } + let Value::String(model) = model else { + return Err(raised( + "AttributeError", + format!( + "'{}' object has no attribute 'startswith'", + crate::group_info::type_name(&model) + ), + )); + }; + let model_info_name = if model.starts_with(&format!("{provider}/")) { + model + } else { + format!("{provider}/{model}") + }; + + let Some(row) = catalog_row(host, Value::String(model_info_name), false).await? else { + return Ok(None); + }; + + let user_model_info = match lookup(host, ModelInfoLookup::UserModelInfo).await? { + Value::Object(info) => info, + Value::Null => Map::new(), + _ => return Err(malformed("user_model_info")), + }; + let declared = user_model_info + .get("base_model") + .and_then(declared_base_model) + .or_else(|| declared_base_model(&query.extra_base_model)); + let widened = base_model_widened_window(host, &row, declared, &provider).await?; + let discovered = discovered(host, query.deployment_id.clone()).await?; + + let mut info = row; + if let Some(max_input_tokens) = widened { + info.insert("max_input_tokens".to_owned(), max_input_tokens); + } + info.extend(discovered); + let mut user_keys = Vec::new(); + for (key, value) in &user_model_info { + if !value.is_null() { + info.insert(key.clone(), value.clone()); + user_keys.push(key.clone()); + } + } + Ok(Some(MergedModelInfo { info, user_keys })) +} + +/// `_convert_stringified_numbers`: a string that reads as a number becomes one (a float when it +/// has an exponent or a point, else an int); any other string stays as written. +fn convert_stringified_number(text: &str) -> Value { + let trimmed = text.trim(); + let digits = without_digit_underscores(trimmed); + let lower = text.to_lowercase(); + if lower.contains('e') || text.contains('.') { + return digits + .and_then(|digits| python_float(&digits)) + .and_then(Number::from_f64) + .map_or_else(|| Value::String(text.to_owned()), Value::Number); + } + digits + .and_then(|digits| python_int(&digits)) + .map_or_else(|| Value::String(text.to_owned()), Value::Number) +} + +/// The text with the underscores Python's `int()`/`float()` accept between digits removed, or +/// `None` when an underscore sits anywhere else. +pub(crate) fn without_digit_underscores(text: &str) -> Option { + let chars: Vec = text.chars().collect(); + let mut out = String::with_capacity(text.len()); + for (index, ch) in chars.iter().enumerate() { + if *ch == '_' { + let before = index.checked_sub(1).and_then(|i| chars.get(i)); + let after = chars.get(index + 1); + if !(before.is_some_and(char::is_ascii_digit) + && after.is_some_and(char::is_ascii_digit)) + { + return None; + } + continue; + } + out.push(*ch); + } + Some(out) +} + +pub(crate) fn python_float(text: &str) -> Option { + let lower = text.to_ascii_lowercase(); + let unsigned = lower.trim_start_matches(['+', '-']); + if matches!(unsigned, "inf" | "infinity" | "nan") { + // Not finite: JSON cannot carry it, so the value stays as written. + return None; + } + let well_formed = text + .chars() + .all(|ch| ch.is_ascii_digit() || matches!(ch, '+' | '-' | '.' | 'e' | 'E')); + if !well_formed { + return None; + } + text.parse::().ok().filter(|n| n.is_finite()) +} + +pub(crate) fn python_int(text: &str) -> Option { + let unsigned = text.strip_prefix(['+', '-']).unwrap_or(text); + if unsigned.is_empty() || !unsigned.chars().all(|ch| ch.is_ascii_digit()) { + return None; + } + let text = text.strip_prefix('+').unwrap_or(text); + text.parse::() + .map(Number::from) + .ok() + .or_else(|| text.parse::().map(Number::from).ok()) +} + +/// `update_dictionary`: `new`'s values over `existing`, skipping `None`s; a string that reads as a +/// number becomes one, and a dict merges one level into the dict it replaces. +pub fn update_dictionary(existing: &mut Map, new: &Map) { + for (key, value) in new { + match value { + Value::Null => {} + Value::String(text) => { + existing.insert(key.clone(), convert_stringified_number(text)); + } + Value::Object(nested) if !is_opaque(value) => { + let merged = match existing.get(key) { + Some(current @ Value::Object(current_map)) if !is_opaque(current) => { + let mut merged = current_map.clone(); + merged.extend(nested.clone()); + merged + } + _ => nested.clone(), + }; + existing.insert(key.clone(), Value::Object(merged)); + } + _ => { + existing.insert(key.clone(), value.clone()); + } + } + } +} + +/// `cost_map_omits_token_price`: the cost-map entries under `keys` exist, and none carries a +/// per-token price. +async fn cost_map_omits_token_price(host: &dyn Host, keys: &[Option<&str>]) -> Result { + let mut entries = 0; + for key in keys.iter().flatten() { + if let Some(entry) = cost_entry(host, key, true).await? + && !is_opaque(&entry) + && let Value::Object(map) = entry + { + entries += 1; + if map.contains_key("input_cost_per_token") || map.contains_key("output_cost_per_token") + { + return Ok(false); + } + } + } + Ok(entries > 0) +} + +/// `get_deployment_model_info`: deployment `model_id`'s own cost-map entry (with what the refresh +/// discovered, over its base model's row), merged over the row of its litellm model `model_name`. +/// A lookup that fails leaves its layer out, as the Python router ignores it. +pub async fn deployment_model_info( + host: &dyn Host, + model_id: &str, + model_name: &Value, +) -> Result>, Error> { + let mut custom: Option> = None; + if let Some(entry) = cost_entry(host, model_id, false).await? { + let entry = if truthy(&entry) { + match entry { + Value::Object(map) if !is_opaque(&Value::Object(map.clone())) => Some(map), + // Not a mapping: the Python router's TypeError, which it ignores. + _ => None, + } + } else { + Some(Map::new()) + }; + if let Some(mut entry) = entry { + entry.extend(discovered(host, Value::String(model_id.to_owned())).await?); + custom = Some(entry); + } + } + + let builtin = catalog_row(host, model_name.clone(), true).await?; + + let mut base_model_key: Option = None; + if let Some(current) = &custom + && let Some(base_model) = current.get("base_model").filter(|base| !base.is_null()) + && let Some(mut base_row) = catalog_row(host, base_model.clone(), true).await? + { + base_model_key = base_row + .get("key") + .and_then(Value::as_str) + .map(str::to_owned); + // The base model provides defaults; the deployment's own entry overrides them. + update_dictionary(&mut base_row, current); + custom = Some(base_row); + } + + let builtin_key = builtin + .as_ref() + .and_then(|row| row.get("key")) + .and_then(Value::as_str) + .map(str::to_owned); + let info = match (custom, builtin) { + (Some(custom), Some(mut builtin)) => { + update_dictionary(&mut builtin, &custom); + builtin + } + (None, Some(builtin)) => builtin, + (Some(custom), None) => custom, + (None, None) => return Ok(None), + }; + if cost_map_omits_token_price( + host, + &[ + Some(model_id), + builtin_key.as_deref(), + base_model_key.as_deref(), + ], + ) + .await? + { + let mut info = info; + info.insert("input_cost_per_token".to_owned(), Value::Null); + info.insert("output_cost_per_token".to_owned(), Value::Null); + return Ok(Some(info)); + } + Ok(Some(info)) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + }; + + use async_trait::async_trait; + use serde_json::json; + + use super::*; + use crate::{ + cache::{InMemoryRouterCache, RouterCache}, + error::AttemptError, + host::{Attempt, Output, RouterEvent, RouterRequest, RoutingContext, SelectStage}, + registry::Deployment, + }; + + /// A cost map and a provider, answering lookups the way `PythonHost` does. + #[derive(Default)] + struct Catalog { + rows: BTreeMap, + provider: (String, String), + user_model_info: Value, + lookups: Mutex>, + } + + #[async_trait] + impl Host for Catalog { + async fn invoke(&self, _attempt: &Attempt) -> Result { + unreachable!("a model-info query runs no attempt") + } + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + Ok(candidates) + } + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + Ok(true) + } + async fn sleep(&self, _duration: std::time::Duration) {} + fn observe(&self, _event: &RouterEvent) {} + fn cache(&self) -> Arc { + Arc::new(InMemoryRouterCache::default()) + } + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + self.lookups + .lock() + .unwrap() + .push(serde_json::to_value(lookup).unwrap()); + Ok(match lookup { + ModelInfoLookup::ModelInfo { + model, tolerate, .. + } => match model.as_str().and_then(|model| self.rows.get(model)) { + Some(row) => row.clone(), + None if *tolerate => json!({"failed": true}), + None => return Err(raised("ValueError", "unmapped")), + }, + ModelInfoLookup::CostEntry { key, .. } => self + .rows + .get(key) + .map_or(Value::Null, |entry| json!({"entry": entry})), + ModelInfoLookup::Provider { .. } => { + json!([self.provider.0, self.provider.1]) + } + ModelInfoLookup::UserModelInfo => self.user_model_info.clone(), + _ => Value::Null, + }) + } + } + + #[test] + fn update_dictionary_converts_numbers_and_merges_nested_dicts_one_level() { + let mut existing = json!({"a": 1, "n": {"x": 1, "y": 2}, "k": "keep"}) + .as_object() + .unwrap() + .clone(); + let new = json!({"a": None::, "n": {"y": 3}, "i": "8_192", "f": "3e-07", "s": "abc"}) + .as_object() + .unwrap() + .clone(); + update_dictionary(&mut existing, &new); + assert_eq!( + Value::Object(existing), + json!({"a": 1, "n": {"x": 1, "y": 3}, "k": "keep", "i": 8192, "f": 3e-07, "s": "abc"}) + ); + } + + #[tokio::test] + async fn router_model_info_overlays_the_deployments_own_values_on_the_row() { + let host = Catalog { + rows: BTreeMap::from([( + "openai/gpt-4o".to_owned(), + json!({"key": "gpt-4o", "max_input_tokens": 100, "input_cost_per_token": 1.0}), + )]), + provider: ("gpt-4o".to_owned(), "openai".to_owned()), + user_model_info: json!({"id": "d", "input_cost_per_token": 2.0, "mode": None::}), + ..Catalog::default() + }; + let query = RouterModelInfoQuery { + deployment_id: json!("d"), + params_model: "openai/gpt-4o".to_owned(), + received_model_name: "g".to_owned(), + ..RouterModelInfoQuery::default() + }; + let merged = router_model_info(&host, &query).await.unwrap().unwrap(); + assert_eq!( + Value::Object(merged.info), + json!({"key": "gpt-4o", "max_input_tokens": 100, "input_cost_per_token": 2.0, "id": "d"}) + ); + assert_eq!(merged.user_keys, vec!["id", "input_cost_per_token"]); + } + + #[tokio::test] + async fn deployment_model_info_merges_the_entry_over_the_builtin_row() { + let host = Catalog { + rows: BTreeMap::from([ + ( + "gpt-4o".to_owned(), + json!({"key": "gpt-4o", "max_tokens": 10, "input_cost_per_token": 1.0}), + ), + ("d".to_owned(), json!({"max_tokens": "20"})), + ]), + ..Catalog::default() + }; + let info = deployment_model_info(&host, "d", &json!("gpt-4o")) + .await + .unwrap() + .unwrap(); + assert_eq!( + Value::Object(info), + json!({"key": "gpt-4o", "max_tokens": 20, "input_cost_per_token": 1.0}) + ); + } + + #[tokio::test] + async fn an_unpriced_entry_reports_no_token_prices() { + let host = Catalog { + rows: BTreeMap::from([("d".to_owned(), json!({"max_tokens": 5}))]), + ..Catalog::default() + }; + let info = deployment_model_info(&host, "d", &json!("unmapped")) + .await + .unwrap() + .unwrap(); + assert_eq!( + Value::Object(info), + json!({"max_tokens": 5, "input_cost_per_token": null, "output_cost_per_token": null}) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/outcome.rs b/litellm-rust/crates/router-core/src/outcome.rs new file mode 100644 index 00000000000..59e0d14b2c2 --- /dev/null +++ b/litellm-rust/crates/router-core/src/outcome.rs @@ -0,0 +1,125 @@ +//! Which of a request's outcomes reach the deployment's health. The Python router learns them from +//! logging callbacks it registers on the process lists; `_core_serve_through_request_logging` and +//! `_core_discard` take those off, and `_core_bind_to_request_logging` puts them back on one +//! request's logging object. This core registers none, so the host answers, for each request, what +//! would have run ([`OutcomeCallbacks`]), and every write a callback owns passes through +//! [`OutcomeCallbacks::record`]. +//! +//! What the Python router writes outside its callbacks is not gated, and reaches [`crate::Health`] +//! with the default value: `make_call`'s usage, `async_routing_strategy_pre_call_checks`' +//! cooldown, `_trigger_cooldown_for_failed_deployment`, `RouterHealth.record_failure` and a +//! realtime session's success. + +use std::future::Future; + +use serde::Deserialize; + +use crate::error::Error; + +/// Which of the router's outcome callbacks litellm's logging runs for a request. The default is a +/// router whose callbacks are on the process lists. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct OutcomeCallbacks { + /// `deployment_callback_on_success`, `sync_deployment_callback_on_success` for a sync call. + /// `_core_bind_to_request_logging` binds only the former. + pub success: bool, + /// `deployment_callback_on_failure` and `async_deployment_callback_on_failure`. + pub failure: bool, + /// The attempt runs through `make_call`, which counts its usage whatever the callbacks do. + pub make_call: bool, +} + +impl Default for OutcomeCallbacks { + fn default() -> Self { + Self { + success: true, + failure: true, + make_call: true, + } + } +} + +/// What an outcome writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutcomeWrite { + /// The success counter, or the usage the success callback adds at the end of a stream. + Success, + /// The tokens and request of an attempt that returned: `make_call`'s write, or the success + /// callback's for an endpoint that skips `make_call`. + AttemptUsage, + /// The failure counter, the allowed-fails counter, the cooldown and the failed request. + Failure, +} + +impl OutcomeCallbacks { + /// Run `record` when `write` happens for this request, else record nothing. + pub(crate) async fn record( + self, + write: OutcomeWrite, + record: impl Future>, + ) -> Result { + let runs = match write { + OutcomeWrite::Success => self.success, + OutcomeWrite::AttemptUsage => self.success || self.make_call, + OutcomeWrite::Failure => self.failure, + }; + if runs { record.await } else { Ok(T::default()) } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const REGISTERED: OutcomeCallbacks = OutcomeCallbacks { + success: true, + failure: true, + make_call: true, + }; + const UNBOUND: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: false, + make_call: true, + }; + const UNBOUND_WITHOUT_MAKE_CALL: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: false, + make_call: false, + }; + const BOUND_SYNC: OutcomeCallbacks = OutcomeCallbacks { + success: false, + failure: true, + make_call: true, + }; + + #[rstest] + #[case::registered_success(REGISTERED, OutcomeWrite::Success, true)] + #[case::registered_usage(REGISTERED, OutcomeWrite::AttemptUsage, true)] + #[case::registered_failure(REGISTERED, OutcomeWrite::Failure, true)] + #[case::unbound_success(UNBOUND, OutcomeWrite::Success, false)] + #[case::unbound_usage(UNBOUND, OutcomeWrite::AttemptUsage, true)] + #[case::unbound_failure(UNBOUND, OutcomeWrite::Failure, false)] + #[case::no_make_call_usage(UNBOUND_WITHOUT_MAKE_CALL, OutcomeWrite::AttemptUsage, false)] + #[case::bound_sync_success(BOUND_SYNC, OutcomeWrite::Success, false)] + #[case::bound_sync_failure(BOUND_SYNC, OutcomeWrite::Failure, true)] + #[tokio::test] + async fn a_write_runs_when_its_callback_does( + #[case] callbacks: OutcomeCallbacks, + #[case] write: OutcomeWrite, + #[case] runs: bool, + ) { + let recorded = callbacks.record(write, async { Ok(true) }).await; + + assert_eq!(recorded.ok(), Some(runs)); + } + + #[test] + fn a_router_on_the_process_lists_is_the_default() { + let parsed: OutcomeCallbacks = serde_json::from_str("{}").expect("an empty document"); + + assert_eq!(parsed, REGISTERED); + } +} diff --git a/litellm-rust/crates/router-core/src/precall.rs b/litellm-rust/crates/router-core/src/precall.rs new file mode 100644 index 00000000000..04dd22f1c90 --- /dev/null +++ b/litellm-rust/crates/router-core/src/precall.rs @@ -0,0 +1,300 @@ +//! `enable_pre_call_checks` (`_pre_call_checks`): before a pick, drop the deployments whose +//! context window the prompt exceeds, those outside the request's `allowed_model_region`, and +//! those that cannot take a requested param, then decide which error a group with nothing left +//! raises. The facts that need live model data (the prompt's token count, each deployment's +//! context window, which params it takes) come from [`Host::pre_call_facts`]; every decision is +//! made here. +//! +//! [`Host::pre_call_facts`]: crate::host::Host::pre_call_facts + +use std::{collections::BTreeMap, sync::Arc}; + +use serde::Deserialize; +use serde_json::Value; + +use crate::{error::Error, registry::Deployment}; + +/// What the host knows about one deployment for the pre-call checks. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +#[serde(default)] +pub struct DeploymentFacts { + /// The deployment's context window (`get_router_model_info`'s `max_input_tokens`), when it + /// declares one and resolves. + pub max_input_tokens: Option, + /// `_deployment_lacks_params`: it does not support one of the requested params. + pub lacks_params: bool, + /// `_deployment_rejects_params`: it lacks one the request and deployment neither drop nor allow. + pub rejects_params: bool, +} + +/// What the host knows about a request for the pre-call checks. +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +#[serde(default)] +pub struct PreCallFacts { + /// The request carries `messages` or `input`: without either no check runs. + pub has_input: bool, + /// The prompt may be counted against a context window (no compaction is pending). + pub countable: bool, + /// The prompt's token count, when it was counted. + pub input_tokens: Option, + /// The request's `allowed_model_region`. + pub allowed_model_region: Option, + /// The requested params the check filters on, sorted (`_pre_call_check_requested_chat_params`). + pub requested_params: Vec, + /// Facts by deployment id, for the candidates and the group's deployments. + pub deployments: BTreeMap, +} + +impl PreCallFacts { + fn of(&self, deployment: &Deployment) -> DeploymentFacts { + self.deployments + .get(&deployment.id) + .cloned() + .unwrap_or_default() + } +} + +/// The name a context-window error gives a deployment: its base model, else its model. +fn deployment_model(deployment: &Deployment) -> String { + let truthy = |value: Option<&Value>| match value { + Some(Value::String(text)) if !text.is_empty() => Some(text.clone()), + Some(Value::Null | Value::Bool(false)) | None => None, + Some(Value::String(_)) => None, + Some(other) => Some(python_str(other)), + }; + truthy(deployment.model_info.raw.get("base_model")) + .or_else(|| truthy(deployment.raw_params.get("base_model"))) + .unwrap_or_else(|| deployment.params.model.clone()) +} + +/// `str(value)` for the scalar values a base model may hold. +fn python_str(value: &Value) -> String { + match value { + Value::Bool(true) => "True".to_owned(), + Value::Number(number) => number.to_string(), + other => other.to_string(), + } +} + +/// `is_region_allowed`: the deployment's `region_name` equals the allowed region. +fn in_region(deployment: &Deployment, allowed: &Value) -> bool { + deployment.raw_params.get("region_name") == Some(allowed) +} + +/// `_pre_call_checks` over `candidates`; `pool` is the group's deployments before health filters +/// (`request_deployments`), which decide whether nothing left is an unsupported-params error. +pub fn filter( + model: &str, + candidates: Vec>, + pool: &[Arc], + facts: &PreCallFacts, +) -> Result>, Error> { + if !facts.has_input { + return Ok(candidates); + } + let mut invalid = vec![false; candidates.len()]; + let mut context_window_error = false; + let mut detail = String::new(); + for (index, deployment) in candidates.iter().enumerate() { + let found = facts.of(deployment); + if let (Some(limit), true, Some(tokens)) = + (found.max_input_tokens, facts.countable, facts.input_tokens) + && tokens > limit + { + invalid[index] = true; + context_window_error = true; + detail.push_str(&format!( + "Model={}, Max Input Tokens={limit}, Got={tokens}", + deployment_model(deployment) + )); + continue; + } + if let Some(allowed) = &facts.allowed_model_region + && !allowed.is_null() + && !in_region(deployment, allowed) + { + invalid[index] = true; + } + } + let params_checked = !facts.requested_params.is_empty(); + let param_candidates: Vec = (0..candidates.len()).filter(|i| !invalid[*i]).collect(); + let lacking: Vec = param_candidates + .iter() + .map(|i| params_checked && facts.of(&candidates[*i]).lacks_params) + .collect(); + let none_takes = !lacking.is_empty() && lacking.iter().all(|lacks| *lacks); + let failing: Vec = if none_takes { + param_candidates + .iter() + .map(|i| params_checked && facts.of(&candidates[*i]).rejects_params) + .collect() + } else { + lacking + }; + for (index, fails) in param_candidates.iter().zip(failing) { + if fails { + invalid[*index] = true; + } + } + let excluded = invalid.iter().filter(|x| **x).count(); + if excluded == candidates.len() { + if !context_window_error + && !pool.is_empty() + && pool + .iter() + .all(|d| params_checked && facts.of(d).rejects_params) + { + return Err(Error::UnsupportedParams { + model: model.to_owned(), + params: facts.requested_params.clone(), + }); + } + if context_window_error { + return Err(Error::ContextWindowExceeded { + model: model.to_owned(), + detail, + }); + } + } + Ok(candidates + .into_iter() + .zip(invalid) + .filter_map(|(deployment, invalid)| (!invalid).then_some(deployment)) + .collect()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, params: Value, model_info: Value) -> Arc { + let mut info = model_info; + info["id"] = json!(id); + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": params, "model_info": info, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + fn facts(entries: &[(&str, DeploymentFacts)]) -> PreCallFacts { + PreCallFacts { + has_input: true, + countable: true, + deployments: entries + .iter() + .map(|(id, f)| ((*id).to_owned(), f.clone())) + .collect(), + ..PreCallFacts::default() + } + } + + fn window(limit: i64) -> DeploymentFacts { + DeploymentFacts { + max_input_tokens: Some(limit), + ..DeploymentFacts::default() + } + } + + #[rstest] + fn a_prompt_over_a_context_window_drops_that_deployment() { + let small = deployment("small", json!({"model": "openai/a"}), json!({})); + let big = deployment("big", json!({"model": "openai/b"}), json!({})); + let mut found = facts(&[("small", window(5)), ("big", window(10_000))]); + found.input_tokens = Some(100); + let kept = filter("g", vec![small, big], &[], &found).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["big"] + ); + } + + #[rstest] + fn no_window_left_names_each_deployment_back_to_back() { + let one = deployment("one", json!({"model": "openai/one"}), json!({})); + let two = deployment( + "two", + json!({"model": "openai/two"}), + json!({"base_model": "azure/base"}), + ); + let mut found = facts(&[("one", window(5)), ("two", window(6))]); + found.input_tokens = Some(9); + let error = filter("g", vec![one, two], &[], &found).unwrap_err(); + let Error::ContextWindowExceeded { model, detail } = error else { + panic!("expected a context window error, got {error:?}"); + }; + assert_eq!(model, "g"); + assert_eq!( + detail, + "Model=openai/one, Max Input Tokens=5, Got=9Model=azure/base, Max Input Tokens=6, Got=9" + ); + } + + #[rstest] + fn without_input_nothing_is_checked() { + let small = deployment("small", json!({"model": "openai/a"}), json!({})); + let mut found = facts(&[("small", window(5))]); + found.input_tokens = Some(100); + found.has_input = false; + assert_eq!(filter("g", vec![small], &[], &found).unwrap().len(), 1); + } + + #[rstest] + #[case::in_region("eu", 1)] + #[case::out_of_region("ap", 0)] + fn the_allowed_region_keeps_only_its_deployments(#[case] region: &str, #[case] kept: usize) { + let eu = deployment( + "eu", + json!({"model": "openai/a", "region_name": "eu"}), + json!({}), + ); + let none = deployment("none", json!({"model": "openai/a"}), json!({})); + let mut found = facts(&[]); + found.allowed_model_region = Some(json!(region)); + assert_eq!( + filter("g", vec![eu, none], &[], &found).unwrap().len(), + kept + ); + } + + #[rstest] + fn a_group_where_every_deployment_rejects_the_param_is_unsupported() { + let old = deployment("old", json!({"model": "openai/old"}), json!({})); + let mut found = facts(&[( + "old", + DeploymentFacts { + lacks_params: true, + rejects_params: true, + ..DeploymentFacts::default() + }, + )]); + found.requested_params = vec!["response_format".to_owned()]; + let error = filter("g", vec![old.clone()], &[old], &found).unwrap_err(); + assert!( + matches!(error, Error::UnsupportedParams { params, .. } if params == ["response_format"]) + ); + } + + #[rstest] + fn a_deployment_that_takes_the_param_wins_over_one_that_would_drop_it() { + let old = deployment("old", json!({"model": "openai/old"}), json!({})); + let new = deployment("new", json!({"model": "openai/new"}), json!({})); + let mut found = facts(&[( + "old", + DeploymentFacts { + lacks_params: true, + ..DeploymentFacts::default() + }, + )]); + found.requested_params = vec!["response_format".to_owned()]; + let kept = filter("g", vec![old, new], &[], &found).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["new"] + ); + } +} diff --git a/litellm-rust/crates/router-core/src/prerouting.rs b/litellm-rust/crates/router-core/src/prerouting.rs new file mode 100644 index 00000000000..d6e3df05e7a --- /dev/null +++ b/litellm-rust/crates/router-core/src/prerouting.rs @@ -0,0 +1,53 @@ +//! Pre-routing: before an async pick resolves its name, the host may route the request to another +//! model group (Python's `_core_async_pre_routing_hook`). The complexity, auto, adaptive and +//! quality routers registered from `auto_router/...` marker deployments, and the Claude Code +//! session binding, live host-side; the core asks once per pick, before resolution, and resolves +//! the name the host answers. The attempt still serves the requested name (its `model_group`): +//! only the deployments come from the routed group. +//! +//! A custom routing strategy (`register_plugin`, `set_custom_routing_strategy`, lar1) replaces the +//! whole pick (`_core_async_get_available_deployment` and `get_available_deployment`): the host +//! answers the same step with the deployment document the strategy returned, which may name a +//! deployment the registry does not hold, and the core serves that deployment as is. + +use std::sync::Arc; + +use crate::{Core, config::Op, error::Error, host::RouterRequest, registry::Deployment}; + +/// What the host's pre-routing step sees. +#[derive(Clone, Copy, Debug)] +pub struct PreRoute<'a> { + pub op: &'a Op, + pub request: &'a RouterRequest, + /// The name this pick resolves: the requested name or the fallback target. + pub name: &'a str, + /// How many fallbacks deep the chain is (Python's `fallback_depth`): only a first attempt + /// binds a Claude Code session. + pub fallback_depth: u32, + /// 0 for the first attempt in this group, then the retry number. + pub retry: u32, +} + +/// The host's answer to a pre-routing step. +#[derive(Clone, Debug)] +pub enum PreRouted { + /// Resolve this model group instead of the requested name. + Model(String), + /// A custom strategy's pick: serve this deployment, without resolving or filtering. + Deployment(Arc), +} + +impl Core { + /// What a pick of `ask.name` serves: the host's pre-routed group or custom pick, else the + /// name itself. Only picks that ask for it pre-route (`request.pre_routes`). + pub(crate) async fn pre_routed(&self, ask: PreRoute<'_>) -> Result { + if !ask.request.pre_routes { + return Ok(PreRouted::Model(ask.name.to_owned())); + } + Ok(self + .host + .pre_route(&ask) + .await? + .unwrap_or_else(|| PreRouted::Model(ask.name.to_owned()))) + } +} diff --git a/litellm-rust/crates/router-core/src/ptu.rs b/litellm-rust/crates/router-core/src/ptu.rs new file mode 100644 index 00000000000..9110ac6a67f --- /dev/null +++ b/litellm-rust/crates/router-core/src/ptu.rs @@ -0,0 +1,96 @@ +//! PTU shares (`litellm_core_utils/ptu_shares.py`): a deployment whose `model_info.ptu_shares` +//! splits it across teams serves only those teams, and the teams holding a share are kept to +//! their share. + +use std::{collections::BTreeSet, sync::Arc}; + +use serde_json::{Map, Value}; + +use crate::registry::Deployment; + +/// `parsed_ptu_shares`: the teams of a non-empty `ptu_shares` map whose every entry is a team id +/// and a positive whole number of PTUs (`2.0` reads as 2, a bool is no number), else None. +pub fn share_teams(model_info: &Map) -> Option> { + let shares = model_info.get("ptu_shares")?.as_object()?; + if shares.is_empty() { + return None; + } + shares + .iter() + .map(|(team, share)| (!team.is_empty() && whole_ptus(share)).then(|| team.clone())) + .collect() +} + +/// `_whole_ptus`: a positive whole number. +fn whole_ptus(share: &Value) -> bool { + match share { + Value::Number(number) => number.as_i64().map_or_else( + || { + number + .as_u64() + .map(|whole| whole > 0) + .or_else(|| number.as_f64().map(|f| f.fract() == 0.0 && f > 0.0)) + .unwrap_or(false) + }, + |whole| whole > 0, + ), + _ => false, + } +} + +/// `filter_ptu_shared_deployments`: the deployments `team_id` holds a share on when it holds one, +/// else the ones split into no shares; a caller without a team holds none. Also whether any +/// deployment was withheld. +pub fn filter_shared( + deployments: Vec>, + team_id: Option<&str>, +) -> (Vec>, bool) { + let team = team_id.unwrap_or(""); + let checks: Vec<(Arc, Option>)> = deployments + .into_iter() + .map(|deployment| { + let shares = share_teams(&deployment.model_info.raw); + (deployment, shares) + }) + .collect(); + let holds_share = checks + .iter() + .any(|(_, shares)| shares.as_ref().is_some_and(|teams| teams.contains(team))); + let total = checks.len(); + let kept: Vec> = checks + .into_iter() + .filter(|(_, shares)| match shares { + Some(teams) => teams.contains(team), + None => !holds_share, + }) + .map(|(deployment, _)| deployment) + .collect(); + let withheld = kept.len() < total; + (kept, withheld) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case(json!({"TA": 50}), Some(vec!["TA"]))] + #[case(json!({"TA": 2.0, "TB": 1}), Some(vec!["TA", "TB"]))] + #[case(json!({"TA": 2.5}), None)] + #[case(json!({"TA": true}), None)] + #[case(json!({"TA": 0}), None)] + #[case(json!({}), None)] + #[case(json!({"": 1}), None)] + #[case(json!("TA"), None)] + fn reads_the_share_teams(#[case] shares: Value, #[case] expected: Option>) { + let mut model_info = Map::new(); + model_info.insert("ptu_shares".to_owned(), shares); + assert_eq!( + share_teams(&model_info), + expected.map(|teams| teams.into_iter().map(str::to_owned).collect()) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/registry.rs b/litellm-rust/crates/router-core/src/registry.rs new file mode 100644 index 00000000000..78c7d3b67e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/registry.rs @@ -0,0 +1,1548 @@ +//! The configured deployments, indexed by id, model group, alias, wildcard pattern, routing +//! group and team. A [`Registry`] is immutable; [`Registry::apply`] builds the next version. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + fmt::Write as _, + sync::Arc, +}; + +use litellm_router_types::{LitellmParams, Spelled}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::{ + error::ConfigError, + host_index::{HostIndexes, IndexWrite, cache_flags_left_to_the_catalog, deployment_model_name}, +}; + +/// A registry's version; every change produces a higher one. +pub type RegistryVersion = u64; + +/// A deployment as a config or an `apply` spells it. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +pub struct DeploymentSpec { + pub model_name: String, + pub litellm_params: Map, + #[serde(default)] + pub model_info: Map, + /// The entry's other top-level keys (a config's `tpm`, ...), kept so the host's view of the + /// deployment list reads back the entries it stored. + #[serde(flatten)] + pub extra: Map, + /// The provider the host resolved when it built the deployment ([`crate::build`]); never + /// read from a config or an `apply`. + #[serde(skip)] + pub provider: Option, +} + +/// `routing_groups`: a named subset of model groups sharing a routing strategy. +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct RoutingGroupSpec { + pub group_name: String, + pub models: Vec, + pub routing_strategy: crate::config::RoutingStrategy, + /// A priority group's member priorities, served as each member's `order`. + #[serde(default)] + pub model_priorities: Option>, +} + +/// The `model_info` fields the router reads; every other key stays in [`Self::raw`]. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelInfo { + pub team_id: Option, + pub team_public_model_name: Option, + pub access_groups: Vec, + pub blocked: bool, + pub cooldown_time: Option, + pub allowed_fails: Option, + /// `model_info.allowed_fails_policy` when it is a dict, read as Python reads it: field name + /// to allowed fails, values as given. + pub allowed_fails_policy: Option>, + pub raw: Map, +} + +impl ModelInfo { + fn from_raw(raw: Map) -> Self { + let text = |key: &str| raw.get(key).and_then(Value::as_str).map(str::to_owned); + Self { + team_id: text("team_id"), + team_public_model_name: text("team_public_model_name"), + access_groups: raw + .get("access_groups") + .and_then(Value::as_array) + .map(|groups| { + groups + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(), + blocked: raw.get("blocked").and_then(Value::as_bool) == Some(true), + cooldown_time: raw.get("cooldown_time").and_then(Value::as_f64), + allowed_fails: raw + .get("allowed_fails") + .and_then(Value::as_u64) + .and_then(|fails| u32::try_from(fails).ok()), + allowed_fails_policy: raw + .get("allowed_fails_policy") + .and_then(Value::as_object) + .cloned(), + raw, + } + } +} + +/// One configured deployment. +#[derive(Clone, Debug, PartialEq)] +pub struct Deployment { + pub id: String, + pub model_name: String, + pub params: LitellmParams, + /// `litellm_params` as written, in order: what the id hashes and the host receives. + pub raw_params: Map, + pub model_info: ModelInfo, + /// The entry's other top-level keys, as written. + pub extra: Map, + /// The provider the host resolved for it, when it was built through the host. + pub resolved_provider: Option, +} + +impl Deployment { + pub fn from_spec(spec: DeploymentSpec) -> Result { + let params: LitellmParams = serde_json::from_value(Value::Object( + spec.litellm_params.clone(), + )) + .map_err(|source| ConfigError::InvalidDeployment { + model_name: spec.model_name.clone(), + source, + })?; + let id = match spec.model_info.get("id") { + Some(Value::String(id)) => id.clone(), + Some(Value::Number(id)) => id.to_string(), + _ => deployment_id(&spec.model_name, &spec.litellm_params), + }; + Ok(Self { + id, + model_name: spec.model_name, + params, + raw_params: spec.litellm_params, + model_info: ModelInfo::from_raw(spec.model_info), + extra: spec.extra, + resolved_provider: spec.provider, + }) + } + + /// The deployment as a `model_list` entry: its other top-level keys, then `model_name`, + /// `litellm_params` and `model_info` (with its id). + pub fn record(&self) -> Value { + let mut model_info = self.model_info.raw.clone(); + model_info.insert("id".to_owned(), Value::String(self.id.clone())); + let mut record = self.extra.clone(); + record.insert( + "model_name".to_owned(), + Value::String(self.model_name.clone()), + ); + record.insert( + "litellm_params".to_owned(), + Value::Object(self.raw_params.clone()), + ); + record.insert("model_info".to_owned(), Value::Object(model_info)); + Value::Object(record) + } + + /// The stand-in an unrouted attempt carries: the requested name as its group and model, no id. + pub fn unrouted(name: &str) -> Result { + let mut params = Map::new(); + params.insert("model".to_owned(), Value::String(name.to_owned())); + let mut info = Map::new(); + info.insert("id".to_owned(), Value::String(String::new())); + Self::from_spec(DeploymentSpec { + model_name: name.to_owned(), + litellm_params: params, + model_info: info, + extra: Map::new(), + provider: None, + }) + } + + /// The provider: the one the host resolved when it built the deployment, else + /// `custom_llm_provider`, else the `model` prefix before `/`. + pub fn provider(&self) -> &str { + self.resolved_provider + .as_deref() + .or(self.params.custom_llm_provider.as_deref()) + .or_else(|| { + self.params + .model + .split_once('/') + .map(|(provider, _)| provider) + }) + .unwrap_or("openai") + } + + /// A numeric `litellm_params` value, as `simple_shuffle._metric_weight` reads it. + pub fn metric(&self, key: &str) -> Option { + let spelled = |value: &Option>| match value { + Some(Spelled::Value(value)) => Some(*value), + Some(Spelled::Text(_)) | None => None, + }; + match key { + "rpm" => spelled(&self.params.rpm), + "tpm" => spelled(&self.params.tpm), + _ => self.raw_params.get(key).and_then(Value::as_f64), + } + } + + /// `simple_shuffle._metric_weight`: 0 when unset, a number (a bool counts as 0 or 1), or + /// `None` when the value is not numeric. A `rpm`/`tpm` spelled as text reads as unset. + pub fn metric_weight(&self, key: &str) -> Option { + if key != "weight" { + return Some(self.metric(key).unwrap_or(0.0)); + } + match self.raw_params.get(key) { + None | Some(Value::Null) => Some(0.0), + Some(Value::Bool(flag)) => Some(f64::from(u8::from(*flag))), + Some(Value::Number(number)) => number.as_f64(), + Some(_) => None, + } + } + + /// `get_deployment_order`: `litellm_params.order`, else `model_info.order`, which is where a + /// team model added through the API keeps it. Neither is validated as an int, so a bool, a + /// whole float and a numeric string are read as the level they compare as. + pub fn order(&self) -> Option { + let order = self + .raw_params + .get("order") + .filter(|order| !order.is_null()) + .or_else(|| self.model_info.raw.get("order"))?; + match order { + Value::Bool(flag) => Some(i64::from(*flag)), + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .filter(|float| float.fract() == 0.0) + .map(|float| float as i64) + }), + Value::String(text) => text.parse().ok(), + Value::Null | Value::Array(_) | Value::Object(_) => None, + } + } + + /// `_deployment_supports_web_search`: `model_info.supports_web_search` when it is set, read + /// for truthiness, else true. + pub(crate) fn supports_web_search(&self) -> bool { + self.model_info + .raw + .get("supports_web_search") + .is_none_or(|flag| flag.is_null() || crate::host_index::truthy(flag)) + } + + /// Whether `team_id` may use this deployment (`deployment_usable_by_team`). + pub fn usable_by_team(&self, team_id: Option<&str>) -> bool { + self.model_info + .team_id + .as_deref() + .is_none_or(|owner| Some(owner) == team_id) + } + + /// Whether `_core_get_model_list(team_id=team_id)` lists this deployment: no team lists + /// every team's (`_core_should_include_deployment`). + pub fn listed_for_team(&self, team_id: Option<&str>) -> bool { + team_id.is_none() || self.usable_by_team(team_id) + } + + /// Whether the deployment declares a limit the router counts per-minute usage for + /// (`_deployment_tracks_usage` and `deployment_has_io_token_limits`, without the limits only + /// the model catalog knows, which the host decides by): `tpm` or `rpm`, `itpm` or `otpm`, on + /// the entry itself, in `litellm_params` or in `model_info`. + pub fn tracks_usage(&self) -> bool { + let declared = |section: &Map| { + ["tpm", "rpm", "itpm", "otpm"] + .iter() + .any(|key| section.get(*key).is_some_and(|limit| !limit.is_null())) + }; + self.params.tpm.is_some() + || self.params.rpm.is_some() + || self.params.itpm.is_some() + || self.params.otpm.is_some() + || declared(&self.extra) + || declared(&self.model_info.raw) + } +} + +/// `_return_pattern_matched_deployments` for one deployment: a copy serving `request`, its +/// `litellm_params.model` filled with the `groups` the pattern matched and the prompt-cache flags +/// left to the catalog. A deployment whose model holds no wildcard is served as it is. +pub(crate) fn routed_deployment( + deployment: &Arc, + groups: &[String], + request: &str, +) -> Arc { + let model = &deployment.params.model; + if !model.contains('*') { + return deployment.clone(); + } + let named = deployment_model_name(groups, request, model); + let mut routed = Deployment::clone(deployment); + for flag in cache_flags_left_to_the_catalog(&deployment.raw_params) { + routed.model_info.raw.remove(*flag); + } + routed + .raw_params + .insert("model".to_owned(), Value::String(named.clone())); + routed.params.model = named; + Arc::new(routed) +} + +/// The connection params of one deployment (`credentials(id)`). +#[derive(Clone, Debug, PartialEq)] +pub struct Credentials { + pub provider: String, + pub api_base: Option, + pub api_version: Option, + pub credential_name: Option, + pub params: LitellmParams, +} + +/// What a name resolved through. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResolveKind { + SpecificDeployment, + Id, + /// A name nothing else serves, routed to the router's `default_deployment` (a copy whose + /// `litellm_params.model` is the name). Served like a deployment named by its id. + Default, + Alias, + RoutingGroup, + Group, + Wildcard, +} + +impl ResolveKind { + /// The name resolved to one deployment that is served as is: a deployment id, or the + /// default deployment (`_common_checks_available_deployment` returns a dict for both). + pub fn is_single(&self) -> bool { + matches!(self, Self::Id | Self::Default) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Resolution { + pub kind: ResolveKind, + /// The model group the deployments are routed as. + pub group: String, + pub deployments: Vec>, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct ResolveScope { + /// The caller named a deployment's `litellm_params.model` (`specific_deployment=True`). + pub specific_deployment: bool, +} + +#[derive(Clone, Debug, Default)] +pub struct DeploymentFilter<'a> { + pub model_name: Option<&'a str>, + pub team_id: Option<&'a str>, + pub provider: Option<&'a str>, +} + +#[derive(Clone, Debug, Default)] +pub struct NameFilter { + pub include_aliases: bool, +} + +/// The registry indexes as the host wrote them over the ones the deployments build (the +/// deprecated `model_names`, `model_name_to_deployment_indices` and +/// `team_model_to_deployment_indices` writes), which resolution reads instead from then on. +/// Indices point into the deployment list; one past its end is skipped. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct IndexOverrides { + /// `_core_model_names`: the names `name in model_names` finds. + pub model_names: Option>, + /// `_core_model_name_to_deployment_indices`: a model name's deployments. + pub model_name: Option>>, + /// `_core_team_model_to_deployment_indices`, as `(team_id, public name, indices)` entries. + pub team_model: Option)>>, +} + +impl IndexOverrides { + fn is_empty(&self) -> bool { + self.model_names.is_none() && self.model_name.is_none() && self.team_model.is_none() + } +} + +/// One atomic change: optionally a whole new deployment list, then upserts, deletes and, +/// optionally, a new dynamic config. +#[derive(Clone, Debug, Default)] +pub struct ChangeSet { + /// Every deployment, in order, replacing the current list (the host registry's `model_list` + /// after a write; it may hold one id more than once, as the Python router's list can). + pub replace: Option>, + /// Deployments added at the end, beside any that share their id (a deployment the host + /// retries after its provider resolved). + pub appends: Vec, + pub upserts: Vec, + pub deletes: Vec, + pub settings: Option, + /// The indexes the host wrote ([`IndexOverrides`]), replacing the ones held; an empty one + /// goes back to the indexes the deployments build. + pub index: Option, + /// The deployment at the position the id map holds for this id, removed with every index + /// shifted (`_delete_deployment`, and an upsert's removal of the deployment it edits). + pub remove: Option, + /// A host write to the indexes it hands out ([`HostIndexes`]). + pub host_index: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct Registry { + version: RegistryVersion, + /// The version the deployments before the newest appends last changed at: a host that read + /// the list at or after it reads only what was appended since ([`Self::deployments_from`]). + rewritten: RegistryVersion, + deployments: Vec>, + by_id: HashMap, + by_group: BTreeMap>, + by_team: HashMap>, + wildcards: Vec<(String, Vec)>, + aliases: BTreeMap, + routing_groups: Vec, + overrides: Option>, + /// The indexes the host hands out, kept by the Python router's history rules. + host: Arc, + /// `_model_to_group` as Python last built it, when a raw `routing_groups` write replaced the + /// groups since: member calls keep routing by it (see [`Self::member_routing_group`]). + member_groups: Option>>, +} + +impl Registry { + pub fn build( + specs: Vec, + aliases: BTreeMap, + routing_groups: Vec, + ) -> Result { + // One id may name several deployments (a per-organization fan-out, or two entries + // declaring one id): each is kept, and the id finds the last, as the Python router keeps them. + let deployments = specs + .into_iter() + .map(|spec| Deployment::from_spec(spec).map(Arc::new)) + .collect::, _>>()?; + // The indexes `set_model_list` leaves, as a build installing each deployment in turn does. + let mut host = HostIndexes::default(); + for (position, deployment) in deployments.iter().enumerate() { + let record = deployment.record(); + host.added( + record.as_object().expect("a record is an object"), + position, + false, + ); + } + host.set_model_names(deployments.iter().map(|d| d.model_name.clone()).collect()); + Ok(Self { + host: Arc::new(host), + ..Self::index(0, deployments, aliases, routing_groups) + }) + } + + fn index( + version: RegistryVersion, + deployments: Vec>, + aliases: BTreeMap, + routing_groups: Vec, + ) -> Self { + let mut by_id = HashMap::new(); + let mut by_group: BTreeMap> = BTreeMap::new(); + let mut by_team: HashMap> = HashMap::new(); + for (index, deployment) in deployments.iter().enumerate() { + by_id.insert(deployment.id.clone(), index); + by_group + .entry(deployment.model_name.clone()) + .or_default() + .push(index); + if let Some(team) = &deployment.model_info.team_id { + by_team.entry(team.clone()).or_default().push(index); + } + } + // In the order each pattern first appears, then most specific first, as + // `PatternUtils.sorted_patterns` keeps them: a stable sort, longest regex first, + // then the most regex syntax. + let mut wildcards: Vec<(String, Vec)> = Vec::new(); + for deployment in &deployments { + let name = &deployment.model_name; + if name.contains('*') && !wildcards.iter().any(|(pattern, _)| pattern == name) { + wildcards.push(( + name.clone(), + by_group.get(name).cloned().unwrap_or_default(), + )); + } + } + wildcards.sort_by_key(|(pattern, _)| std::cmp::Reverse(pattern_specificity(pattern))); + Self { + version, + rewritten: version, + deployments, + by_id, + by_group, + by_team, + wildcards, + aliases, + routing_groups, + overrides: None, + host: Arc::default(), + member_groups: None, + } + } + + /// This registry with the stale routing-group member map (`None` derives it from the groups). + #[must_use] + pub fn with_member_groups(self, member_groups: Option>) -> Self { + Self { + member_groups: member_groups.map(Arc::new), + ..self + } + } + + pub fn version(&self) -> RegistryVersion { + self.version + } + + /// The version the deployments before the newest appends last changed at (see `rewritten`). + pub fn rewritten(&self) -> RegistryVersion { + self.rewritten + } + + /// Every deployment id, in list order (`names(kind="ids")`). + pub fn ids(&self) -> Vec { + self.deployments.iter().map(|d| d.id.clone()).collect() + } + + /// The deployments from position `start` on, in list order. + pub fn deployments_from(&self, start: usize) -> &[Arc] { + self.deployments.get(start..).unwrap_or_default() + } + + /// The next registry: upserts, then deletes (`apply_changes`). An upserted deployment whose + /// id exists is removed and appended at the end, as the Python router does on an edit. + pub fn apply( + &self, + changes: &ChangeSet, + aliases: Option>, + ) -> Result { + let mut deployments = match &changes.replace { + Some(specs) => specs + .iter() + .map(|spec| self.carried(spec.clone()).map(Arc::new)) + .collect::, _>>()?, + None => self.deployments.clone(), + }; + for spec in &changes.appends { + deployments.push(Arc::new(self.carried(spec.clone())?)); + } + for spec in &changes.upserts { + let deployment = self.carried(spec.clone())?; + deployments.retain(|existing| existing.id != deployment.id); + deployments.push(Arc::new(deployment)); + } + deployments.retain(|deployment| !changes.deletes.contains(&deployment.id)); + let mut host = HostIndexes::clone(&self.host); + if let Some(id) = &changes.remove + && let Some(position) = host.position_of(id) + && position < deployments.len() + { + deployments.remove(position); + host.removed(id, position) + .map_err(|error| ConfigError::Removal(error.to_string()))?; + } + if let Some(write) = &changes.host_index { + host.write(write.clone()); + } + let overrides = match &changes.index { + Some(index) => (!index.is_empty()).then(|| Arc::new(index.clone())), + None => self.overrides.clone(), + }; + // Appends alone leave every deployment before them as it was. + let appended_only = changes.replace.is_none() + && changes.upserts.is_empty() + && changes.deletes.is_empty() + && changes.remove.is_none(); + let next = Self::index( + self.version + 1, + deployments, + aliases.unwrap_or_else(|| self.aliases.clone()), + self.routing_groups.clone(), + ); + Ok(Self { + overrides, + host: Arc::new(host), + member_groups: self.member_groups.clone(), + rewritten: if appended_only { + self.rewritten + } else { + next.rewritten + }, + ..next + }) + } + + /// The next registry for a build starting over (`set_model_list`): no deployments, and the + /// indexes [`HostIndexes::reset`] clears cleared. + #[must_use] + pub fn emptied(&self) -> Self { + let mut host = HostIndexes::clone(&self.host); + host.reset(); + Self { + overrides: self.overrides.clone(), + host: Arc::new(host), + member_groups: self.member_groups.clone(), + ..Self::index( + self.version + 1, + Vec::new(), + self.aliases.clone(), + self.routing_groups.clone(), + ) + } + } + + /// Add one deployment a build accepted at the end and index it in place (`add` is + /// `add_deployment`, which also names its group). A build installs into a registry of its + /// own this way, so a list of n deployments builds in O(n) rather than reindexing per + /// deployment. + pub fn push_built(&mut self, spec: DeploymentSpec, add: bool) -> Result<(), ConfigError> { + let deployment = Arc::new(self.carried(spec)?); + let index = self.deployments.len(); + let record = deployment.record(); + Arc::make_mut(&mut self.host).added( + record.as_object().expect("a record is an object"), + index, + add, + ); + self.by_id.insert(deployment.id.clone(), index); + let name = &deployment.model_name; + self.by_group.entry(name.clone()).or_default().push(index); + if let Some(team) = &deployment.model_info.team_id { + self.by_team.entry(team.clone()).or_default().push(index); + } + if name.contains('*') { + match self + .wildcards + .iter_mut() + .find(|(pattern, _)| pattern == name) + { + Some((_, indices)) => indices.push(index), + None => { + // A new pattern joins after every one it ties with, as `index` orders them: + // first appearance, then a stable sort by specificity. + self.wildcards.push((name.clone(), vec![index])); + self.wildcards.sort_by_key(|(pattern, _)| { + std::cmp::Reverse(pattern_specificity(pattern)) + }); + } + } + } + self.deployments.push(deployment); + self.version += 1; + Ok(()) + } + + /// The last deployment added. + pub fn last(&self) -> Option<&Arc> { + self.deployments.last() + } + + /// Take `set_model_list`'s `model_names`: the names it was given. + pub fn set_model_names(&mut self, names: Vec) { + Arc::make_mut(&mut self.host).set_model_names(names); + self.version += 1; + } + + /// The indexes the host hands out. + pub fn host_indexes(&self) -> &HostIndexes { + &self.host + } + + /// The deployment `spec` describes, keeping the provider the host resolved for the + /// deployment it replaces (same id, same params) when the spec carries none: a host that + /// hands back its whole list after a write does not resolve every provider again. + fn carried(&self, mut spec: DeploymentSpec) -> Result { + let deployment = Deployment::from_spec(spec.clone())?; + if spec.provider.is_none() + && let Some(previous) = self + .by_id + .get(&deployment.id) + .and_then(|index| self.deployments.get(*index)) + && previous.raw_params == deployment.raw_params + && previous.model_name == deployment.model_name + { + spec.provider.clone_from(&previous.resolved_provider); + return Deployment::from_spec(spec); + } + Ok(deployment) + } + + /// This registry with the providers `from` resolved for the same deployments (same id, + /// same params) and `from`'s host indexes: a router rebuilt for new static settings over the + /// same list keeps what its host resolved and the indexes their history built. + #[must_use] + pub fn with_providers_of(&self, from: &Self) -> Self { + let deployments = self + .deployments + .iter() + .map(|deployment| { + let previous = from + .by_id + .get(&deployment.id) + .and_then(|index| from.deployments.get(*index)) + .filter(|previous| { + previous.raw_params == deployment.raw_params + && previous.model_name == deployment.model_name + }); + match previous { + Some(previous) if deployment.resolved_provider.is_none() => { + Arc::new(Deployment { + resolved_provider: previous.resolved_provider.clone(), + ..Deployment::clone(deployment) + }) + } + _ => deployment.clone(), + } + }) + .collect(); + Self { + deployments, + host: from.host.clone(), + ..self.clone() + } + } + + fn at(&self, indices: &[usize]) -> Vec> { + indices + .iter() + .filter_map(|index| self.deployments.get(*index).cloned()) + .collect() + } + + /// `_core_deployments_named(name)`: the deployments the model-name index holds for `name` + /// (whatever their own name), or `None` when it does not index the name. + pub(crate) fn named(&self, name: &str) -> Option>> { + match self.overrides.as_ref().and_then(|o| o.model_name.as_ref()) { + Some(index) => index.get(name).map(|indices| self.at(indices)), + None => self.by_group.get(name).map(|indices| self.at(indices)), + } + } + + /// Whether the model-name index holds `name` (`name in model_name_to_deployment_indices`). + fn indexes_name(&self, name: &str) -> bool { + match self.overrides.as_ref().and_then(|o| o.model_name.as_ref()) { + Some(index) => index.contains_key(name), + None => self.by_group.contains_key(name), + } + } + + /// `_team_deployments_named(team_id, name)`: the deployments the team index holds for the + /// pair, or `None` when it does not index it. + pub(crate) fn team_named(&self, team_id: &str, name: &str) -> Option>> { + match self.overrides.as_ref().and_then(|o| o.team_model.as_ref()) { + Some(entries) => entries + .iter() + .find(|(team, public, _)| team == team_id && public == name) + .map(|(_, _, indices)| self.at(indices)), + None => { + let own: Vec> = self + .deployments + .iter() + .filter(|d| { + d.model_info.team_id.as_deref() == Some(team_id) + && d.model_info.team_public_model_name.as_deref() == Some(name) + }) + .cloned() + .collect(); + (!own.is_empty()).then_some(own) + } + } + } + + /// Every deployment the team index holds under public name `name`, any team's + /// (`_team_deployments_across_teams`). + pub(crate) fn team_named_across_teams(&self, name: &str) -> Vec> { + match self.overrides.as_ref().and_then(|o| o.team_model.as_ref()) { + Some(entries) => entries + .iter() + .filter(|(_, public, _)| public == name) + .flat_map(|(_, _, indices)| self.at(indices)) + .collect(), + None => self + .deployments + .iter() + .filter(|d| { + d.model_info.team_id.is_some() + && d.model_info.team_public_model_name.as_deref() == Some(name) + }) + .cloned() + .collect(), + } + } + + /// Whether a written index entry names a position past the model list. + fn overflows(&self, indices: &[usize]) -> bool { + indices.iter().any(|index| *index >= self.deployments.len()) + } + + /// The written model-name index entry for `name` names a position past the model list. + pub(crate) fn name_index_overflows(&self, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.model_name.as_ref()) + .and_then(|index| index.get(name)) + .is_some_and(|indices| self.overflows(indices)) + } + + /// The written team index entry for `(team_id, name)` names a position past the model list. + pub(crate) fn team_index_overflows(&self, team_id: &str, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.team_model.as_ref()) + .and_then(|entries| { + entries + .iter() + .find(|(team, public, _)| team == team_id && public == name) + }) + .is_some_and(|(_, _, indices)| self.overflows(indices)) + } + + /// A written team index entry under public name `name` names a position past the list. + pub(crate) fn across_teams_index_overflows(&self, name: &str) -> bool { + self.overrides + .as_ref() + .and_then(|o| o.team_model.as_ref()) + .is_some_and(|entries| { + entries + .iter() + .any(|(_, public, indices)| public == name && self.overflows(indices)) + }) + } + + /// The group a pick of `name` reads its strategy from, when the stale member map names a + /// group that is no longer configured (`_get_routing_context` raises `KeyError`): `name` + /// after its alias, unless it is a callable routing group. + pub fn missing_member_group(&self, name: &str) -> Option<&str> { + let members = self.member_groups.as_ref()?; + let resolved = self.alias_of(name).unwrap_or(name); + if self.routing_group_named(resolved).is_some() { + return None; + } + let group = members.get(resolved)?; + (!self.routing_groups.iter().any(|g| g.group_name == *group)).then_some(group.as_str()) + } + + /// The routing group a member call of `model_group` routes by: the stale member map's + /// group when there is one (`_model_to_group`), else the non-priority group listing it. + pub(crate) fn member_routing_group(&self, model_group: &str) -> Option<&RoutingGroupSpec> { + match &self.member_groups { + Some(members) => members + .get(model_group) + .and_then(|name| self.routing_groups.iter().find(|g| g.group_name == *name)), + None => self + .routing_groups + .iter() + .filter(|group| !crate::routing_groups::is_priority(group)) + .find(|group| group.models.iter().any(|model| model == model_group)), + } + } + + pub fn deployment(&self, id: &str) -> Option> { + self.by_id + .get(id) + .map(|index| self.deployments[*index].clone()) + } + + pub fn deployments(&self, filter: &DeploymentFilter<'_>) -> Vec> { + let candidates = match (filter.model_name, filter.team_id) { + (Some(name), _) => self + .by_group + .get(name) + .map(|i| self.at(i)) + .unwrap_or_default(), + (None, Some(team)) => self + .by_team + .get(team) + .map(|i| self.at(i)) + .unwrap_or_default(), + (None, None) => self.deployments.clone(), + }; + candidates + .into_iter() + .filter(|d| { + filter + .team_id + .is_none_or(|team| d.model_info.team_id.as_deref() == Some(team)) + }) + .filter(|d| { + filter + .provider + .is_none_or(|provider| d.provider() == provider) + }) + .collect() + } + + /// The model group names, in name order, with aliases when asked. + pub fn names(&self, filter: &NameFilter) -> Vec { + let mut names: Vec = self.by_group.keys().cloned().collect(); + if filter.include_aliases { + names.extend(self.aliases.keys().cloned()); + } + names + } + + pub fn group(&self, name: &str) -> Option>> { + if self + .overrides + .as_ref() + .is_some_and(|o| o.model_name.is_some()) + { + return self.named(name).map(|deployments| { + deployments + .into_iter() + .filter(|d| d.model_name == name) + .collect() + }); + } + self.by_group.get(name).map(|indices| self.at(indices)) + } + + /// How many rows `_core_get_model_list(model_name=name)` lists: the group's deployments, + /// an alias's target group, a routing group's members, and only when none of those, the + /// deployments of the wildcard the name matches. + pub fn model_list_len(&self, name: &str) -> usize { + let count = |group: &str| self.group(group).map_or(0, |deployments| deployments.len()); + let listed = count(name) + + self.aliases.get(name).map_or(0, |target| count(target)) + + self + .routing_groups + .iter() + .find(|group| group.group_name == name) + .map_or(0, |group| { + group.models.iter().map(|model| count(model)).sum() + }); + if listed > 0 { + return listed; + } + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .map_or(0, |(_, indices)| indices.len()) + } + + /// The routing group a model group belongs to, if any. + pub fn routing_group_of(&self, model_name: &str) -> Option<&RoutingGroupSpec> { + self.routing_groups + .iter() + .find(|group| group.models.iter().any(|model| model == model_name)) + } + + /// The strategy a routing group named `group_name` declares. + pub fn resolve_routing_group_strategy( + &self, + group_name: &str, + ) -> Option { + self.routing_groups + .iter() + .find(|group| group.group_name == group_name) + .map(|group| group.routing_strategy.clone()) + } + + pub(crate) fn routing_groups(&self) -> &[RoutingGroupSpec] { + &self.routing_groups + } + + /// Every deployment, in configured order. + pub(crate) fn all(&self) -> &[Arc] { + &self.deployments + } + + /// Whether `name` is a deployment `model_name` (`name in _core_model_names`). + pub(crate) fn has_name(&self, name: &str) -> bool { + match self.overrides.as_ref().and_then(|o| o.model_names.as_ref()) { + Some(names) => names.contains(name), + None => self.by_group.contains_key(name), + } + } + + /// The group `model_group_alias` rewrites `name` to. + pub(crate) fn alias_of(&self, name: &str) -> Option<&str> { + self.aliases.get(name).map(String::as_str) + } + + /// The callable routing group named `name`: a model group of that name shadows it. + pub(crate) fn routing_group_named(&self, name: &str) -> Option<&RoutingGroupSpec> { + (!self.indexes_name(name)) + .then(|| self.routing_groups.iter().find(|g| g.group_name == name)) + .flatten() + } + + /// Whether any deployment is listed under a wildcard pattern. + pub(crate) fn has_wildcards(&self) -> bool { + !self.wildcards.is_empty() + } + + /// The most specific wildcard pattern matching `name`, with its deployments. + pub(crate) fn wildcard_for(&self, name: &str) -> Option<(String, Vec>)> { + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .map(|(pattern, indices)| (pattern.clone(), self.at(indices))) + } + + /// Resolve a requested name, in the order of `_common_checks_available_deployment`: + /// a specific deployment, an id, an alias, a routing group, a model group, a wildcard. + pub fn resolve(&self, name: &str, scope: ResolveScope) -> Option { + let resolution = |kind, group: &str, deployments: Vec>| { + (!deployments.is_empty()).then(|| Resolution { + kind, + group: group.to_owned(), + deployments, + }) + }; + if scope.specific_deployment { + let matching = self + .deployments + .iter() + .filter(|d| d.params.model == name) + .cloned() + .collect(); + // An empty match still resolves: the pick then finds no deployment rather than + // falling back to a default (`_common_checks_available_deployment` returns `[]`). + return Some(Resolution { + kind: ResolveKind::SpecificDeployment, + group: name.to_owned(), + deployments: matching, + }); + } + if !self.by_group.contains_key(name) + && let Some(deployment) = self.deployment(name) + { + let group = deployment.model_name.clone(); + return resolution(ResolveKind::Id, &group, vec![deployment]); + } + let routing_group_named = |name: &str| { + // A model group of the same name shadows a routing group (`get_routing_group`). + (!self.by_group.contains_key(name)) + .then(|| self.routing_groups.iter().find(|g| g.group_name == name)) + .flatten() + }; + if let Some(target) = self.aliases.get(name) { + if let Some(routing_group) = routing_group_named(target) { + let deployments = crate::routing_groups::members(routing_group, |model| { + self.by_group + .get(model) + .map(|indices| self.at(indices)) + .unwrap_or_default() + }); + return resolution(ResolveKind::RoutingGroup, target, deployments); + } + return self + .group(target) + .and_then(|deployments| resolution(ResolveKind::Alias, target, deployments)); + } + if let Some(routing_group) = routing_group_named(name) { + let deployments = crate::routing_groups::members(routing_group, |model| { + self.by_group + .get(model) + .map(|indices| self.at(indices)) + .unwrap_or_default() + }); + return resolution(ResolveKind::RoutingGroup, name, deployments); + } + if let Some(indices) = self.by_group.get(name) { + return resolution(ResolveKind::Group, name, self.at(indices)); + } + self.wildcards + .iter() + .find(|(pattern, _)| wildcard_matches(pattern, name)) + .and_then(|(pattern, indices)| { + resolution(ResolveKind::Wildcard, pattern, self.at(indices)) + }) + } + + /// Access group name to the model groups in it. + pub fn access_groups(&self) -> BTreeMap> { + let mut groups: BTreeMap> = BTreeMap::new(); + for deployment in &self.deployments { + for group in &deployment.model_info.access_groups { + groups + .entry(group.clone()) + .or_default() + .insert(deployment.model_name.clone()); + } + } + groups + } + + pub fn credentials(&self, id: &str) -> Option { + self.deployment(id).map(|deployment| Credentials { + provider: deployment.provider().to_owned(), + api_base: deployment.params.api_base.clone(), + api_version: deployment.params.api_version.clone(), + credential_name: deployment.params.litellm_credential_name.clone(), + params: deployment.params.clone(), + }) + } +} + +/// `PatternUtils.calculate_pattern_specificity` over the pattern's regex +/// (`re.escape(pattern).replace(r"\*", "(.*)")`): its length, then how much regex syntax it has. +fn pattern_specificity(pattern: &str) -> (usize, usize) { + let mut regex = String::new(); + for c in pattern.chars() { + match c { + '*' => regex.push_str("(.*)"), + '(' | ')' | '[' | ']' | '{' | '}' | '?' | '+' | '-' | '|' | '^' | '$' | '\\' | '.' + | '&' | '~' | '#' | ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c' => { + regex.push('\\'); + regex.push(c); + } + _ => regex.push(c), + } + } + let complexity = regex + .chars() + .filter(|c| matches!(c, '*' | '+' | '?' | '\\' | '^' | '$' | '|' | '(' | ')')) + .count(); + (regex.chars().count(), complexity) +} + +/// `*` matches any run of characters, as the pattern router's `(.*)` does under `re.match`: +/// anchored at the start only, so the name may run on past the pattern's last literal. +fn wildcard_matches(pattern: &str, name: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or_default(); + let Some(mut rest) = name.strip_prefix(first) else { + return false; + }; + let parts: Vec<&str> = parts.collect(); + for part in parts { + match rest.find(part) { + Some(at) => rest = &rest[at + part.len()..], + None => return false, + } + } + true +} + +/// `deployment_id` in `litellm/router_api.py`: sha256 over the model group and every +/// `litellm_params` key and value in order. A string is taken as is, a dict as `json.dumps`, and +/// anything else as Python's `str()`. +pub fn deployment_id(model_group: &str, litellm_params: &Map) -> String { + let mut text = String::from(model_group); + for (key, value) in litellm_params { + text.push_str(key); + match value { + Value::String(value) => text.push_str(value), + Value::Object(_) => json_dumps(value, &mut text), + Value::Null | Value::Bool(_) | Value::Number(_) | Value::Array(_) => { + python_str(value, &mut text); + } + } + } + let digest = Sha256::digest(text.as_bytes()); + let mut hex = String::with_capacity(64); + for byte in digest { + let _ = write!(hex, "{byte:02x}"); + } + hex +} + +/// Python's `repr(float)`: the shortest round-trip digits, positional for exponents in +/// `-4..16`, scientific otherwise. +fn python_float(value: f64, out: &mut String) { + if value.is_nan() { + out.push_str("nan"); + return; + } + if value.is_infinite() { + out.push_str(if value > 0.0 { "inf" } else { "-inf" }); + return; + } + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific.split_once('e').unwrap_or((&scientific, "0")); + let exponent: i32 = exponent.parse().unwrap_or(0); + let (sign, mantissa) = mantissa + .strip_prefix('-') + .map_or(("", mantissa), |m| ("-", m)); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + out.push_str(sign); + if (-4..16).contains(&exponent) { + if exponent < 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-exponent - 1) as usize)); + out.push_str(&digits); + } else { + let point = exponent as usize + 1; + if digits.len() <= point { + out.push_str(&digits); + out.extend(std::iter::repeat_n('0', point - digits.len())); + out.push_str(".0"); + } else { + out.push_str(&digits[..point]); + out.push('.'); + out.push_str(&digits[point..]); + } + } + } else { + out.push_str(&digits[..1]); + if digits.len() > 1 { + out.push('.'); + out.push_str(&digits[1..]); + } + let _ = write!( + out, + "e{}{:02}", + if exponent < 0 { '-' } else { '+' }, + exponent.abs() + ); + } +} + +fn number(value: &serde_json::Number, out: &mut String) { + match value.as_f64() { + Some(float) if !value.is_i64() && !value.is_u64() => python_float(float, out), + Some(_) | None => out.push_str(&value.to_string()), + } +} + +/// `json.dumps` with its defaults: `", "` and `": "` separators and `ensure_ascii`. +fn json_dumps(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }), + Value::Number(value) => number(value, out), + Value::String(text) => json_string(text, out), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + json_dumps(item, out); + } + out.push(']'); + } + Value::Object(entries) => { + out.push('{'); + for (index, (key, item)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + json_string(key, out); + out.push_str(": "); + json_dumps(item, out); + } + out.push('}'); + } + } +} + +fn json_string(text: &str, out: &mut String) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{08}' => out.push_str("\\b"), + '\u{0c}' => out.push_str("\\f"), + ch if ch.is_ascii() && !ch.is_ascii_control() => out.push(ch), + ch => { + let mut units = [0u16; 2]; + for unit in ch.encode_utf16(&mut units) { + let _ = write!(out, "\\u{unit:04x}"); + } + } + } + } + out.push('"'); +} + +/// Python's `str()` of a JSON value: `repr` for the items of a list or dict. +fn python_str(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("None"), + Value::Bool(flag) => out.push_str(if *flag { "True" } else { "False" }), + Value::Number(value) => number(value, out), + Value::String(text) => python_repr_str(text, out), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + python_str(item, out); + } + out.push(']'); + } + Value::Object(entries) => { + out.push('{'); + for (index, (key, item)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + python_repr_str(key, out); + out.push_str(": "); + python_str(item, out); + } + out.push('}'); + } + } +} + +fn python_repr_str(text: &str, out: &mut String) { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + out.push(quote); + for ch in text.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + ch if ch == quote => { + out.push('\\'); + out.push(ch); + } + ch if (ch as u32) < 0x20 || ch as u32 == 0x7f => { + let _ = write!(out, "\\x{:02x}", ch as u32); + } + ch => out.push(ch), + } + } + out.push(quote); +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn spec(model_name: &str, params: Value, info: Value) -> DeploymentSpec { + serde_json::from_value(json!({ + "model_name": model_name, "litellm_params": params, "model_info": info, + })) + .unwrap() + } + + #[rstest] + #[case::none(json!({}), json!({}), json!({}), false)] + #[case::params_rpm(json!({"rpm": 1}), json!({}), json!({}), true)] + #[case::params_itpm(json!({"itpm": 1}), json!({}), json!({}), true)] + #[case::model_info_rpm(json!({}), json!({"rpm": 1}), json!({}), true)] + #[case::model_info_tpm(json!({}), json!({"tpm": 1}), json!({}), true)] + #[case::model_info_otpm(json!({}), json!({"otpm": 1}), json!({}), true)] + #[case::model_info_null(json!({}), json!({"rpm": null}), json!({}), false)] + #[case::top_level_tpm(json!({}), json!({}), json!({"tpm": 1}), true)] + fn a_limit_anywhere_on_the_deployment_tracks_usage( + #[case] params: Value, + #[case] info: Value, + #[case] entry: Value, + #[case] tracks: bool, + ) { + let mut spec = json!({ + "model_name": "g", + "litellm_params": {"model": "openai/a"}, + "model_info": {"id": "a"}, + }); + let extend = |section: &mut Value, more: Value| { + section + .as_object_mut() + .unwrap() + .extend(more.as_object().unwrap().clone()); + }; + extend(&mut spec["litellm_params"], params); + extend(&mut spec["model_info"], info); + extend(&mut spec, entry); + let spec: DeploymentSpec = serde_json::from_value(spec).unwrap(); + assert_eq!(Deployment::from_spec(spec).unwrap().tracks_usage(), tracks); + } + + #[rstest] + #[case::params_int(json!({"model": "m", "order": 2}), json!({}), Some(2))] + #[case::params_win(json!({"model": "m", "order": 2}), json!({"order": 1}), Some(2))] + #[case::model_info(json!({"model": "m"}), json!({"order": 3}), Some(3))] + #[case::params_null(json!({"model": "m", "order": null}), json!({"order": 3}), Some(3))] + #[case::numeric_string(json!({"model": "m", "order": "4"}), json!({}), Some(4))] + #[case::model_info_string(json!({"model": "m"}), json!({"order": "5"}), Some(5))] + #[case::whole_float(json!({"model": "m", "order": 6.0}), json!({}), Some(6))] + #[case::none(json!({"model": "m"}), json!({}), None)] + fn order_reads_as_get_deployment_order( + #[case] params: Value, + #[case] info: Value, + #[case] expected: Option, + ) { + let deployment = Deployment::from_spec(spec("g", params, info)).unwrap(); + assert_eq!(deployment.order(), expected); + } + + #[rstest] + #[case::unset(json!({}), true)] + #[case::declared(json!({"supports_web_search": true}), true)] + #[case::refused(json!({"supports_web_search": false}), false)] + #[case::null_is_unset(json!({"supports_web_search": null}), true)] + fn web_search_support_is_assumed_unless_declared(#[case] info: Value, #[case] expected: bool) { + let deployment = Deployment::from_spec(spec("g", json!({"model": "m"}), info)).unwrap(); + assert_eq!(deployment.supports_web_search(), expected); + } + + #[test] + fn a_routed_wildcard_fills_its_model_and_drops_the_cache_flags() { + let deployment = Arc::new( + Deployment::from_spec(spec( + "anthropic/*", + json!({"model": "anthropic/*"}), + json!({"id": "w", "supports_prompt_caching": true, "keep": 1}), + )) + .unwrap(), + ); + let routed = routed_deployment( + &deployment, + &["claude-3-haiku".to_owned()], + "anthropic/claude-3-haiku", + ); + assert_eq!(routed.params.model, "anthropic/claude-3-haiku"); + assert_eq!(routed.raw_params["model"], "anthropic/claude-3-haiku"); + assert_eq!(routed.record()["model_info"], json!({"id": "w", "keep": 1})); + } + + #[rstest] + #[case::no_team_lists_a_team_owned_one(Some("t1"), None, true)] + #[case::no_team_lists_an_unowned_one(None, None, true)] + #[case::the_owner_lists_it(Some("t1"), Some("t1"), true)] + #[case::another_team_does_not(Some("t1"), Some("t2"), false)] + #[case::a_team_lists_an_unowned_one(None, Some("t2"), true)] + fn listed_for_team_lets_no_team_see_every_deployment( + #[case] owner: Option<&str>, + #[case] team: Option<&str>, + #[case] listed: bool, + ) { + let deployment = Deployment::from_spec(spec( + "g", + json!({"model": "openai/a"}), + json!({"id": "a", "team_id": owner}), + )) + .unwrap(); + assert_eq!(deployment.listed_for_team(team), listed); + } + + /// Expected ids computed with `litellm.router_api.deployment_id`. + #[rstest] + #[case::strings( + "g", + json!({"model": "openai/gpt-4o", "api_key": "sk-1"}), + "30256ba697cc98ce2aff32c3b73c4b90514cb199876aef429ffcd91ae826d025" + )] + #[case::scalars( + "claude", + json!({"model": "anthropic/claude-3", "rpm": 10, "timeout": 1.5, "stream": true, "x": null}), + "645fd1504089c1084316573ccfc5dd1dd9622623b169026657484fb64a2c89a2" + )] + #[case::list_and_dict( + "m", + json!({"model": "azure/x", "tags": ["a", "b'c"], "extra_body": {"k": [1, 2.0, "é"], "z": {"q": false}}}), + "64f11e16dabed321268bd92132de2020290ef193c0cf554b2c74faee8badbb96" + )] + fn deployment_id_matches_python( + #[case] group: &str, + #[case] params: Value, + #[case] expected: &str, + ) { + let Value::Object(params) = params else { + unreachable!() + }; + assert_eq!(deployment_id(group, ¶ms), expected); + } + + #[rstest] + #[case::integral(2.0, "2.0")] + #[case::small(0.00001, "1e-05")] + #[case::large(1e16, "1e+16")] + #[case::fraction(0.1, "0.1")] + fn floats_print_as_python_repr(#[case] value: f64, #[case] expected: &str) { + let mut out = String::new(); + python_float(value, &mut out); + assert_eq!(out, expected); + } + + #[rstest] + #[case::group("gpt-4", ResolveKind::Group, "gpt-4")] + #[case::alias("smart", ResolveKind::Alias, "gpt-4")] + #[case::id("dep-b", ResolveKind::Id, "gpt-4")] + #[case::wildcard("anthropic/claude-x", ResolveKind::Wildcard, "anthropic/*")] + fn resolve_follows_python_order( + #[case] name: &str, + #[case] kind: ResolveKind, + #[case] group: &str, + ) { + let registry = Registry::build( + vec![ + spec( + "gpt-4", + json!({"model": "openai/gpt-4"}), + json!({"id": "dep-a"}), + ), + spec( + "gpt-4", + json!({"model": "azure/gpt-4"}), + json!({"id": "dep-b"}), + ), + spec("anthropic/*", json!({"model": "anthropic/*"}), json!({})), + ], + BTreeMap::from([("smart".to_owned(), "gpt-4".to_owned())]), + Vec::new(), + ) + .unwrap(); + let resolution = registry.resolve(name, ResolveScope::default()).unwrap(); + assert_eq!((resolution.kind, resolution.group.as_str()), (kind, group)); + } + + #[rstest] + fn apply_moves_an_edited_deployment_to_the_end() { + let registry = Registry::build( + vec![ + spec("a", json!({"model": "openai/a"}), json!({"id": "1"})), + spec("a", json!({"model": "openai/b"}), json!({"id": "2"})), + ], + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let next = registry + .apply( + &ChangeSet { + upserts: vec![spec("a", json!({"model": "openai/c"}), json!({"id": "1"}))], + ..ChangeSet::default() + }, + None, + ) + .unwrap(); + let ids: Vec = next + .group("a") + .unwrap() + .iter() + .map(|d| d.id.clone()) + .collect(); + assert_eq!( + (next.version(), ids), + (1, vec!["2".to_owned(), "1".to_owned()]) + ); + } +} diff --git a/litellm-rust/crates/router-core/src/resolve.rs b/litellm-rust/crates/router-core/src/resolve.rs new file mode 100644 index 00000000000..9075fc36c53 --- /dev/null +++ b/litellm-rust/crates/router-core/src/resolve.rs @@ -0,0 +1,585 @@ +//! A requested name to the deployments that may serve this caller: a port of +//! `_common_checks_available_deployment`. In order: a specific deployment, a deployment id, the +//! alias, a routing group, then for a name no deployment has, a team's public name, every team's +//! public name for a proxy admin, a wildcard; else the group (team-first), kept to the caller's +//! access groups, then deployments whose litellm model is the name, then the first default +//! fallback. Every arm drops strategy markers and deployments that refuse the request's endpoint, +//! and withholds deployments reserved for other teams. + +use std::{collections::BTreeSet, sync::Arc}; + +use jiff::Timestamp; + +use serde_json::Value; + +use crate::{ + access::{filter_by_access_groups, filter_reserved}, + eligibility::eligible_deployments, + error::Error, + host_index::{pattern_to_regex, regex_match}, + registry::{Deployment, DeploymentSpec, Registry, Resolution, ResolveKind, routed_deployment}, + select::is_strategy_marker, +}; + +/// Who is asking, as the request's auth metadata says. +#[derive(Clone, Copy, Debug, Default)] +pub struct Caller<'a> { + /// `user_api_key_team_id`. + pub team_id: Option<&'a str>, + /// The caller's `user_role` is `proxy_admin`. + pub proxy_admin: bool, + /// The caller's `models` plus `team_models`. + pub allowed_models: &'a [String], + /// The endpoint the pick runs under (`endpoint_selection`), which deployments may refuse. + pub endpoint: Option<&'a str>, + /// The model `endpoint_selection` was given: the name an endpoint refusal reads. + pub requested: &'a str, + /// `f"{provider}/{model}"`, the spelling `PatternMatchRouter.get_pattern` tries when no + /// pattern matches the name itself. + pub wildcard_request: Option<&'a str>, +} + +/// `_core_should_include_deployment`: the team's own deployment under its public name, or a +/// deployment named `model_name` the team may use. +fn included(deployment: &Deployment, model_name: &str, team_id: Option<&str>) -> bool { + let info = &deployment.model_info; + if team_id.is_some() + && info.team_id.as_deref() == team_id + && info.team_public_model_name.as_deref() == Some(model_name) + { + return true; + } + deployment.model_name == model_name + && (team_id.is_none() || info.team_id.is_none() || info.team_id.as_deref() == team_id) +} + +/// `_core_get_all_deployments(model_name, team_id)`: the team's deployments under that public +/// name when it has any, else the deployments named `model_name` the team may use; a name the +/// model-name index does not hold is looked for in every deployment, for a team. +pub fn all_deployments( + registry: &Registry, + model_name: &str, + team_id: Option<&str>, +) -> Result>, Error> { + let keep = |deployments: Vec>| -> Vec> { + deployments + .into_iter() + .filter(|d| included(d, model_name, team_id)) + .collect() + }; + // A written index position past the model list raises `IndexError` where it is read. + let out_of_range = || Error::IndexOutOfRange { + model: model_name.to_owned(), + }; + if let Some(team) = team_id { + if registry.team_index_overflows(team, model_name) { + return Err(out_of_range()); + } + if let Some(own) = registry.team_named(team, model_name) { + let own = keep(own); + if !own.is_empty() { + return Ok(own); + } + } + } + if registry.name_index_overflows(model_name) { + return Err(out_of_range()); + } + Ok(match registry.named(model_name) { + Some(named) => keep(named), + None if team_id.is_some() => keep(registry.all().to_vec()), + None => Vec::new(), + }) +} + +/// `_team_deployments_across_teams`: every team's deployments under public name `model`, for a +/// proxy admin calling without a team; a name several teams share is refused. +fn across_teams(registry: &Registry, model: &str) -> Result>, Error> { + if registry.across_teams_index_overflows(model) { + return Err(Error::IndexOutOfRange { + model: model.to_owned(), + }); + } + let deployments = registry.team_named_across_teams(model); + let teams: BTreeSet<&str> = deployments + .iter() + .filter_map(|d| d.model_info.team_id.as_deref()) + .collect(); + if teams.len() > 1 { + return Err(Error::MultipleTeams { + model: model.to_owned(), + }); + } + Ok(deployments) +} + +/// The access groups of the rows `_core_get_model_list(model_name, team_id)` lists: the +/// group's deployments, an alias's target deployments, a routing group's rows (which carry no +/// access groups), and only when there are none of those, the wildcard's deployments. +fn model_access_groups( + registry: &Registry, + model: &str, + team_id: Option<&str>, +) -> BTreeSet { + let mut rows = all_deployments(registry, model, team_id).unwrap_or_default(); + let mut stripped_rows = 0; + if let Some(target) = registry.alias_of(model) { + match registry.routing_group_named(target) { + Some(group) => stripped_rows += routing_group_rows(registry, group, None).len(), + None => rows.extend(all_deployments(registry, target, None).unwrap_or_default()), + } + } + if let Some(group) = registry.routing_group_named(model) { + stripped_rows += routing_group_rows(registry, group, None).len(); + } + if rows.is_empty() && stripped_rows == 0 { + rows = registry + .wildcard_for(model) + .map(|(_, deployments)| deployments) + .unwrap_or_default(); + } + rows.iter() + .flat_map(|d| d.model_info.access_groups.iter().cloned()) + .collect() +} + +fn routing_group_rows( + registry: &Registry, + group: &crate::registry::RoutingGroupSpec, + team_id: Option<&str>, +) -> Vec> { + crate::routing_groups::members(group, |member| { + all_deployments(registry, member, team_id).unwrap_or_default() + }) +} + +/// `_filter_reserved_deployments` for deployments resolved as `model`: markers dropped (only +/// markers is refused), those that refuse the caller's endpoint dropped (`_drop_strategy_markers`), +/// then the deployments an active access window withholds; when that leaves nothing, the window +/// is named in the refusal. +fn reserved( + model: &str, + deployments: Vec>, + caller: Caller<'_>, + now: Timestamp, +) -> Result>, Error> { + let any = !deployments.is_empty(); + let selectable: Vec> = deployments + .into_iter() + .filter(|d| !is_strategy_marker(d)) + .collect(); + if any && selectable.is_empty() { + return Err(Error::OnlyMarkers { + model: model.to_owned(), + }); + } + let selectable = eligible_deployments(model, caller, selectable)?; + let (kept, blocking) = filter_reserved(selectable, caller.team_id, now); + match blocking { + Some(window) if kept.is_empty() => Err(Error::Reserved { + model: model.to_owned(), + until: window.end_hhmm(), + timezone: window.timezone, + }), + _ => Ok(kept), + } +} + +/// `PatternMatchRouter.route`: copies of a pattern's deployments serving `request`. +fn routed_wildcard( + pattern: &str, + request: &str, + deployments: &[Arc], +) -> Vec> { + let groups = regex_match(&pattern_to_regex(pattern), request).unwrap_or_default(); + deployments + .iter() + .map(|deployment| routed_deployment(deployment, &groups, request)) + .collect() +} + +/// `get_pattern` for `model`: the most specific pattern matching it as spelled, else its +/// provider-prefixed spelling, with the pattern's deployments serving that request. +fn wildcard_route( + registry: &Registry, + model: &str, + wildcard_request: Option<&str>, +) -> Option<(String, Vec>)> { + let route = |request: &str| { + registry + .wildcard_for(request) + .map(|(pattern, deployments)| { + let routed = routed_wildcard(&pattern, request, &deployments); + (pattern, routed) + }) + }; + route(model).or_else(|| route(wildcard_request?)) +} + +/// The name resolution would look for under its provider (`get_pattern`'s second spelling), which +/// only the host can infer: one that reaches the pattern routers and that no pattern matches as +/// spelled. +pub(crate) fn wildcard_name<'a>( + registry: &'a Registry, + name: &'a str, + specific_deployment: bool, +) -> Option<&'a str> { + let model = registry.alias_of(name).unwrap_or(name); + (registry.has_wildcards() + && !specific_deployment + && !registry.has_name(model) + && (registry.has_name(name) || registry.deployment(name).is_none()) + && registry.routing_group_named(model).is_none() + && registry.wildcard_for(model).is_none()) + .then_some(model) +} + +/// The first default fallback a name no deployment serves is routed to. +pub fn default_target(dynamic: &crate::config::RouterDynamicConfig) -> Option<&str> { + dynamic + .fallbacks + .default_target() + .or_else(|| dynamic.default_fallbacks.first().map(String::as_str)) +} + +/// The router's `default_deployment`, as resolution serves it. +#[derive(Clone, Copy, Debug)] +pub enum DefaultDeployment<'a> { + Spec(&'a DeploymentSpec), + /// A value Python cannot copy for a name: resolving to it raises. + Raises, +} + +/// The default deployment as served for `model`: a copy whose `litellm_params.model` is the name. +fn default_for(spec: &DeploymentSpec, model: &str) -> Option> { + let mut spec = spec.clone(); + spec.litellm_params + .insert("model".to_owned(), Value::String(model.to_owned())); + Deployment::from_spec(spec).ok().map(Arc::new) +} + +/// `_core_get_routable_deployments`: the deployments resolution would consider for `request`, +/// before any team, cooldown or strategy filter; nothing when it resolves to nothing or refuses. +pub fn routable( + registry: &Registry, + dynamic: &crate::config::RouterDynamicConfig, + request: &crate::host::RouterRequest, +) -> Vec> { + let caller = Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + endpoint: request.endpoint.as_deref(), + requested: &request.model, + wildcard_request: request.wildcard_request.as_deref(), + }; + match resolve_request( + registry, + &request.model, + false, + caller, + default_target(dynamic), + dynamic.default_ref(), + Timestamp::now(), + ) { + Ok(Some(resolution)) => resolution.deployments, + Ok(None) | Err(_) => Vec::new(), + } +} + +/// Resolve `name` for `caller`. `None` when nothing serves it (`no_healthy_deployments`). +pub fn resolve_request( + registry: &Registry, + name: &str, + specific_deployment: bool, + caller: Caller<'_>, + default_target: Option<&str>, + default_deployment: Option>, + now: Timestamp, +) -> Result, Error> { + let team_id = caller.team_id; + let resolved = |kind, group: &str, deployments| { + Ok(Some(Resolution { + kind, + group: group.to_owned(), + deployments, + })) + }; + if specific_deployment { + let matching = registry + .all() + .iter() + .filter(|d| d.params.model == name) + .cloned() + .collect(); + let kept = reserved(name, matching, caller, now)?; + return resolved(ResolveKind::SpecificDeployment, name, kept); + } + if !registry.has_name(name) + && let Some(deployment) = registry.deployment(name) + { + let group = deployment.model_name.clone(); + let litellm_model = deployment.params.model.clone(); + let kept = reserved(&litellm_model, vec![deployment], caller, now)?; + return resolved(ResolveKind::Id, &group, kept); + } + let aliased = registry.alias_of(name); + let model = aliased.unwrap_or(name); + let routing_group = registry + .routing_group_named(model) + .map(|group| routing_group_rows(registry, group, team_id)); + if routing_group.is_none() && !registry.has_name(model) { + let early = match team_id { + Some(team) => Some(all_deployments(registry, model, Some(team))?), + None if caller.proxy_admin => Some(across_teams(registry, model)?), + None => None, + } + .filter(|deployments| !deployments.is_empty()); + if let Some(deployments) = early { + let kept = reserved(model, deployments, caller, now)?; + return resolved(ResolveKind::Group, model, kept); + } + // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one. + let team_owned = team_id + .map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team)) + .filter(|deployments| !deployments.is_empty()); + if let Some(deployments) = team_owned { + let routed = deployments.iter().flat_map(|deployment| { + let public = deployment.model_info.team_public_model_name.as_deref(); + let pattern = public.unwrap_or(model); + routed_wildcard(pattern, model, std::slice::from_ref(deployment)) + }); + let kept = reserved(model, routed.collect(), caller, now)?; + return resolved(ResolveKind::Wildcard, model, kept); + } + if let Some((pattern, deployments)) = + wildcard_route(registry, model, caller.wildcard_request) + { + let kept = reserved(model, deployments, caller, now)?; + return resolved(ResolveKind::Wildcard, &pattern, kept); + } + match default_deployment { + Some(DefaultDeployment::Raises) => { + return Err(Error::DefaultDeploymentRaises { + model: model.to_owned(), + }); + } + Some(DefaultDeployment::Spec(spec)) => { + if let Some(deployment) = default_for(spec, model) { + let kept = reserved(model, vec![deployment], caller, now)?; + return resolved(ResolveKind::Default, model, kept); + } + } + None => {} + } + } + let (kind, listed) = match routing_group { + Some(deployments) => (ResolveKind::RoutingGroup, deployments), + None if aliased.is_some() => ( + ResolveKind::Alias, + all_deployments(registry, model, team_id)?, + ), + None => ( + ResolveKind::Group, + all_deployments(registry, model, team_id)?, + ), + }; + let narrow = |model: &str, deployments: Vec>| { + let allowed = filter_by_access_groups(model, deployments, caller.allowed_models, || { + model_access_groups(registry, model, team_id) + }); + reserved(model, allowed, caller, now) + }; + let listed_any = !listed.is_empty(); + let deployments = narrow(model, listed)?; + if !deployments.is_empty() { + return resolved(kind, model, deployments); + } + let mut access_emptied = listed_any; + if !listed_any { + // The caller may have named a deployment's litellm model instead. + let by_model: Vec> = registry + .all() + .iter() + .filter(|d| d.params.model == model) + .cloned() + .collect(); + let found = !by_model.is_empty(); + let deployments = narrow(model, by_model)?; + if !deployments.is_empty() { + return resolved(ResolveKind::Group, model, deployments); + } + access_emptied = found; + } + if let Some(target) = default_target.filter(|target| !access_emptied && !target.is_empty()) { + let deployments = narrow(target, all_deployments(registry, target, team_id)?)?; + if !deployments.is_empty() { + return resolved(ResolveKind::Group, target, deployments); + } + // `_common_checks_available_deployment` re-assigned the name: its error names the target. + return Err(Error::UnknownModel { + model: target.to_owned(), + }); + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde_json::{Value, json}; + + use super::*; + use crate::registry::DeploymentSpec; + + fn registry(entries: Value) -> Registry { + let specs: Vec = serde_json::from_value(entries).unwrap(); + Registry::build(specs, BTreeMap::new(), Vec::new()).unwrap() + } + + fn ids(resolution: Option) -> Vec { + resolution + .map(|r| r.deployments.iter().map(|d| d.id.clone()).collect()) + .unwrap_or_default() + } + + #[test] + fn an_access_group_only_caller_is_kept_to_its_groups() { + let registry = registry(json!([ + {"model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a", "access_groups": ["A"]}}, + {"model_name": "g", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "b", "access_groups": ["B"]}}, + ])); + let allowed = vec!["A".to_owned()]; + let caller = Caller { + allowed_models: &allowed, + ..Caller::default() + }; + let resolved = resolve_request(®istry, "g", false, caller, None, None, Timestamp::now()); + assert_eq!(ids(resolved.unwrap()), ["a"]); + // The litellm model string is not narrowed (python-fixes-to-make.md). + let by_model = resolve_request( + ®istry, + "openai/b", + false, + caller, + None, + None, + Timestamp::now(), + ); + assert_eq!(ids(by_model.unwrap()), ["b"]); + } + + #[test] + fn a_proxy_admin_is_refused_a_public_name_several_teams_share() { + let registry = registry(json!([ + {"model_name": "x", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "x", "team_id": "tx", "team_public_model_name": "pub"}}, + {"model_name": "y", "litellm_params": {"model": "openai/y"}, "model_info": {"id": "y", "team_id": "ty", "team_public_model_name": "pub"}}, + ])); + let caller = Caller { + proxy_admin: true, + ..Caller::default() + }; + let refused = resolve_request( + ®istry, + "pub", + false, + caller, + None, + None, + Timestamp::now(), + ); + assert!(matches!(refused, Err(Error::MultipleTeams { model }) if model == "pub")); + } + + fn wildcard_group() -> Value { + json!([{ + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "w", "supports_prompt_cache_breakpoint": true}, + }]) + } + + #[test] + fn a_wildcard_serves_the_matched_model_without_its_cache_flags() { + let registry = registry(wildcard_group()); + let resolved = resolve_request( + ®istry, + "openai/gpt-4o", + false, + Caller::default(), + None, + None, + Timestamp::now(), + ); + let resolution = resolved.unwrap().unwrap(); + assert_eq!(resolution.kind, ResolveKind::Wildcard); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); + assert_eq!( + resolution.deployments[0].record()["model_info"], + json!({"id": "w"}) + ); + } + + #[test] + fn an_alias_to_a_wildcard_fills_the_model_from_its_target() { + let specs: Vec = serde_json::from_value(wildcard_group()).unwrap(); + let aliases = BTreeMap::from([("al".to_owned(), "openai/gpt-4o".to_owned())]); + let registry = Registry::build(specs, aliases, Vec::new()).unwrap(); + let resolved = resolve_request( + ®istry, + "al", + false, + Caller::default(), + None, + None, + Timestamp::now(), + ); + assert_eq!( + resolved.unwrap().unwrap().deployments[0].params.model, + "openai/gpt-4o" + ); + } + + #[test] + fn a_name_no_pattern_matches_is_tried_under_its_provider() { + let registry = registry(wildcard_group()); + assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o")); + assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None); + let caller = Caller { + wildcard_request: Some("openai/gpt-4o"), + ..Caller::default() + }; + let resolved = resolve_request( + ®istry, + "gpt-4o", + false, + caller, + None, + None, + Timestamp::now(), + ); + let resolution = resolved.unwrap().unwrap(); + assert_eq!(resolution.group, "openai/*"); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); + } + + #[test] + fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() { + let registry = registry(json!([ + {"model_name": "g", "litellm_params": {"model": "openai/a"}, "model_info": { + "id": "a", "enforce_supported_endpoints": true, "supported_endpoints": ["/v1/embeddings"], + }}, + ])); + let caller = Caller { + endpoint: Some("/v1/chat/completions"), + requested: "a", + ..Caller::default() + }; + let refused = resolve_request(®istry, "a", false, caller, None, None, Timestamp::now()); + assert!(matches!( + refused, + Err(Error::EndpointNotSupported { model, .. }) if model == "openai/a" + )); + } +} diff --git a/litellm-rust/crates/router-core/src/retry.rs b/litellm-rust/crates/router-core/src/retry.rs new file mode 100644 index 00000000000..3de0ad601e3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/retry.rs @@ -0,0 +1,219 @@ +//! Retry decisions: how many retries a `RetryPolicy` grants for a failure, and whether a failure +//! is retried in the same group at all. Mirrors `get_num_retries_from_retry_policy` and +//! `resolve_retry_policy` in `litellm_core_utils/get_retry_from_policy.py`, and +//! `Router.should_retry_this_error`. + +use std::collections::BTreeMap; + +use crate::{ + config::RetryPolicy, + error::{AttemptError, FailureKind}, + health::should_retry_status, +}; + +/// `resolve_retry_policy`: the group's own policy when `model_group_retry_policy` has one, else +/// the router's `retry_policy`. +pub fn resolve_retry_policy<'a>( + retry_policy: Option<&'a RetryPolicy>, + model_group: &str, + model_group_retry_policy: Option<&'a BTreeMap>, +) -> Option<&'a RetryPolicy> { + model_group_retry_policy + .and_then(|policies| policies.get(model_group)) + .or(retry_policy) +} + +impl RetryPolicy { + /// `get_num_retries_from_retry_policy`: `DefaultRetries` for an `UnsupportedEndpointError`; + /// else `NotFoundErrorRetries` for any 404 answer, then the most specific exception class + /// the policy names, then `DefaultRetries`. + pub fn retries_for(&self, failure: &AttemptError) -> Option { + if failure.is_a("UnsupportedEndpointError") { + return self.DefaultRetries; + } + let not_found = (failure.status == Some(404)) + .then_some(self.NotFoundErrorRetries) + .flatten(); + not_found + .or_else(|| { + failure + .class_names() + .iter() + .find_map(|class| self.field_for(class)) + }) + .or(self.DefaultRetries) + } + + /// `_RETRIES_BY_EXCEPTION_TYPE`: the policy field an exception class reads. + fn field_for(&self, class: &str) -> Option { + match class { + "AuthenticationError" => self.AuthenticationErrorRetries, + "Timeout" => self.TimeoutErrorRetries, + "RateLimitError" => self.RateLimitErrorRetries, + "ContentPolicyViolationError" => self.ContentPolicyViolationErrorRetries, + "BadRequestError" => self.BadRequestErrorRetries, + "ServiceUnavailableError" => self.ServiceUnavailableErrorRetries, + "InternalServerError" => self.InternalServerErrorRetries, + _ => None, + } + } +} + +/// What `should_retry_this_error` knows besides the failure. +#[derive(Clone, Copy, Debug, Default)] +pub struct RetryContext { + pub healthy: usize, + pub all: usize, + /// `context_window_fallbacks is not None`. + pub has_context_window_fallbacks: bool, + /// `content_policy_fallbacks is not None`. + pub has_content_policy_fallbacks: bool, + /// `regular_fallbacks is not None and len(regular_fallbacks) > 0`. + pub has_regular_fallbacks: bool, +} + +/// `should_retry_this_error`, as a predicate: `false` where Python raises the error. +pub fn should_retry(failure: &AttemptError, ctx: RetryContext) -> bool { + if failure.is_a("ContextWindowExceededError") && ctx.has_context_window_fallbacks { + return false; + } + if failure.is_a("ContentPolicyViolationError") && ctx.has_content_policy_fallbacks { + return false; + } + if let Some(status) = failure.status + && !should_retry_status(status) + && !matches!(status, 401 | 403) + && !failure.is_a("UnsupportedEndpointError") + { + return false; + } + if failure.is_a("NotFoundError") { + return false; + } + if failure.is_a("RateLimitError") && ctx.healthy == 0 && ctx.has_regular_fallbacks { + return false; + } + if (failure.is_a("AuthenticationError") || failure.is_a("PermissionDeniedError")) + && ctx.all <= 1 + { + return false; + } + ctx.healthy > 0 +} + +/// `ContentPolicyViolationError`'s classes, most specific first. +pub(crate) fn content_policy_classes() -> Vec> { + [ + "ContentPolicyViolationError", + "BadRequestError", + "APIStatusError", + "APIError", + "OpenAIError", + "Exception", + "BaseException", + "object", + ] + .into_iter() + .map(Into::into) + .collect() +} + +/// The exception classes a failure kind stands for, most specific first, for a host that +/// reports no class names (`litellm/exceptions.py`). +pub(crate) fn kind_classes(kind: FailureKind) -> &'static [&'static str] { + match kind { + FailureKind::RateLimit => &["RateLimitError"], + FailureKind::ContextWindowExceeded => &["ContextWindowExceededError", "BadRequestError"], + FailureKind::ContentPolicyViolation => &["ContentPolicyViolationError", "BadRequestError"], + FailureKind::Authentication => &["AuthenticationError"], + FailureKind::PermissionDenied => &["PermissionDeniedError"], + FailureKind::NotFound => &["NotFoundError"], + FailureKind::BadRequest => &["BadRequestError"], + FailureKind::Timeout => &["Timeout", "APITimeoutError", "APIConnectionError"], + FailureKind::ApiConnection => &["APIConnectionError"], + FailureKind::InternalServer => &["InternalServerError"], + FailureKind::ServiceUnavailable => &["ServiceUnavailableError"], + FailureKind::BadGateway => &["BadGatewayError"], + FailureKind::CallerCredentialAuthentication => { + &["CallerCredentialAuthenticationError", "AuthenticationError"] + } + FailureKind::CallerCredentialRateLimit => { + &["CallerCredentialRateLimitError", "RateLimitError"] + } + FailureKind::GuardrailIntervention | FailureKind::Declined | FailureKind::Other => &[], + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn failure(kind: FailureKind, status: Option, classes: &[&str]) -> AttemptError { + let mut failure = AttemptError::new(kind, status); + failure.classes = classes.iter().map(|class| (*class).into()).collect(); + failure + } + + #[rstest] + #[case::own_field(failure(FailureKind::InternalServer, Some(500), &[]), Some(4))] + #[case::default(failure(FailureKind::BadGateway, Some(502), &[]), Some(7))] + #[case::not_found_status(failure(FailureKind::Other, Some(404), &["APIError"]), Some(1))] + #[case::unsupported_endpoint( + failure(FailureKind::Declined, Some(400), &["UnsupportedEndpointError", "BadRequestError"]), + Some(7) + )] + #[case::malformed_reads_bad_request( + failure(FailureKind::Declined, Some(400), &["MalformedRequestFieldError", "BadRequestError"]), + Some(3) + )] + fn retries_follow_the_most_specific_class( + #[case] failure: AttemptError, + #[case] expected: Option, + ) { + let policy = RetryPolicy { + InternalServerErrorRetries: Some(4), + BadRequestErrorRetries: Some(3), + NotFoundErrorRetries: Some(1), + DefaultRetries: Some(7), + ..RetryPolicy::default() + }; + assert_eq!(policy.retries_for(&failure), expected); + } + + #[rstest] + #[case::rate_limit_with_healthy(FailureKind::RateLimit, Some(429), 1, 2, false, true)] + #[case::rate_limit_none_healthy_with_fallbacks( + FailureKind::RateLimit, + Some(429), + 0, + 2, + true, + false + )] + #[case::bad_request(FailureKind::BadRequest, Some(400), 2, 2, false, false)] + #[case::auth_lone_deployment(FailureKind::Authentication, Some(401), 1, 1, false, false)] + #[case::auth_with_siblings(FailureKind::Authentication, Some(401), 1, 2, false, true)] + #[case::not_found(FailureKind::NotFound, Some(404), 2, 2, false, false)] + #[case::server_error_no_healthy(FailureKind::InternalServer, Some(500), 0, 2, false, false)] + fn should_retry_follows_python( + #[case] kind: FailureKind, + #[case] status: Option, + #[case] healthy: usize, + #[case] all: usize, + #[case] has_regular_fallbacks: bool, + #[case] expected: bool, + ) { + let ctx = RetryContext { + healthy, + all, + has_regular_fallbacks, + ..RetryContext::default() + }; + assert_eq!( + should_retry(&AttemptError::new(kind, status), ctx), + expected + ); + } +} diff --git a/litellm-rust/crates/router-core/src/routing_groups.rs b/litellm-rust/crates/router-core/src/routing_groups.rs new file mode 100644 index 00000000000..fcf6c2febf3 --- /dev/null +++ b/litellm-rust/crates/router-core/src/routing_groups.rs @@ -0,0 +1,48 @@ +//! Routing groups: a callable name over the deployments of its member model groups +//! (`_get_routing_group_deployments`). A priority group serves each member's priority as the +//! member deployments' `order` (`apply_routing_group_priority`), so the order filter and order +//! fallbacks walk the priorities. + +use std::sync::Arc; + +use serde_json::Value; + +use crate::registry::{Deployment, RoutingGroupSpec}; + +/// The member deployments of `group`, in member order, with a priority group's priorities as +/// their `order`. +pub fn members( + group: &RoutingGroupSpec, + deployments_of: impl Fn(&str) -> Vec>, +) -> Vec> { + let priorities = group + .model_priorities + .as_ref() + .filter(|_| is_priority(group)); + group + .models + .iter() + .flat_map(|member| { + let priority = priorities.and_then(|p| p.get(member)).copied(); + deployments_of(member) + .into_iter() + .map(move |deployment| match priority { + Some(order) => with_order(&deployment, order), + None => deployment, + }) + }) + .collect() +} + +/// Whether `group` is a priority group. +pub fn is_priority(group: &RoutingGroupSpec) -> bool { + matches!(&group.routing_strategy, crate::config::RoutingStrategy::Other(name) if name == "priority") +} + +fn with_order(deployment: &Arc, order: i64) -> Arc { + let mut prioritized = Deployment::clone(deployment); + prioritized + .raw_params + .insert("order".to_owned(), Value::from(order)); + Arc::new(prioritized) +} diff --git a/litellm-rust/crates/router-core/src/routing_plugins.rs b/litellm-rust/crates/router-core/src/routing_plugins.rs new file mode 100644 index 00000000000..236de6fc521 --- /dev/null +++ b/litellm-rust/crates/router-core/src/routing_plugins.rs @@ -0,0 +1,114 @@ +//! `Router(plugins=[...])`: the routing-plugin pipeline. The host runs the plugins (live Python +//! objects) in the async pick's pre-routing hook and reports the candidate models they left on +//! the request (`_routing_plugin_candidate_models`); router-core decides which deployments that +//! keeps (`_filter_by_routing_plugin_candidates`) and refuses the paths that never run the +//! pipeline (`_get_available_deployment_unresolved`). + +use std::sync::Arc; + +use crate::{config::RoutingStrategy, error::Error, host::RouterRequest, registry::Deployment}; + +/// Whether the router's own strategy has an async selector: the others pick on the sync path +/// even for async callers (`_async_get_available_deployment`). +pub(crate) fn picks_async(strategy: &RoutingStrategy) -> bool { + matches!( + strategy, + RoutingStrategy::UsageBasedV2 + | RoutingStrategy::SimpleShuffle + | RoutingStrategy::CostBased + | RoutingStrategy::LatencyBased + | RoutingStrategy::LeastBusy + ) +} + +/// The sync pick never runs the pipeline, so a router with plugins refuses it rather than +/// letting a policy plugin be bypassed. `request` is a pick or a call, never a listing. +pub(crate) fn refuse_sync_pick( + has_plugins: bool, + strategy: &RoutingStrategy, + request: &RouterRequest, +) -> Result<(), Error> { + if has_plugins + && request.pass_through.is_none() + && (!request.pre_routes || !picks_async(strategy)) + { + return Err(Error::RoutingPluginsNeedAsync); + } + Ok(()) +} + +/// `_filter_by_routing_plugin_candidates`: keep the deployments whose litellm model the plugins +/// left; nothing left is an error, never a fallback to the unfiltered pool. +pub(crate) fn narrow( + healthy: Vec>, + models: Option<&[String]>, +) -> Result>, Error> { + let Some(models) = models else { + return Ok(healthy); + }; + let kept: Vec> = healthy + .into_iter() + .filter(|d| models.contains(&d.params.model)) + .collect(); + if kept.is_empty() { + return Err(Error::RoutingPluginsLeftNothing { + models: models.to_vec(), + }); + } + Ok(kept) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + fn deployment(id: &str, model: &str) -> Arc { + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": {"model": model}, "model_info": {"id": id}, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + #[rstest] + fn the_plugins_models_narrow_the_pool() { + let pool = vec![deployment("a", "openai/a"), deployment("b", "openai/b")]; + let kept = narrow(pool, Some(&["openai/b".to_owned()])).unwrap(); + assert_eq!( + kept.iter().map(|d| d.id.as_str()).collect::>(), + ["b"] + ); + } + + #[rstest] + fn narrowing_to_nothing_is_an_error() { + let pool = vec![deployment("a", "openai/a")]; + assert!(matches!( + narrow(pool, Some(&[])), + Err(Error::RoutingPluginsLeftNothing { models }) if models.is_empty() + )); + } + + #[rstest] + #[case::async_pick(true, RoutingStrategy::SimpleShuffle, false)] + #[case::sync_pick(false, RoutingStrategy::SimpleShuffle, true)] + #[case::v1_strategy(true, RoutingStrategy::UsageBased, true)] + fn only_the_async_pick_runs_plugins( + #[case] pre_routes: bool, + #[case] strategy: RoutingStrategy, + #[case] refused: bool, + ) { + let request = RouterRequest { + pre_routes, + ..RouterRequest::default() + }; + assert_eq!( + refuse_sync_pick(true, &strategy, &request).is_err(), + refused + ); + } +} diff --git a/litellm-rust/crates/router-core/src/select.rs b/litellm-rust/crates/router-core/src/select.rs new file mode 100644 index 00000000000..fefa7f25ad1 --- /dev/null +++ b/litellm-rust/crates/router-core/src/select.rs @@ -0,0 +1,795 @@ +//! Selection: resolve a name, filter its deployments in the order of +//! `Router._core_async_get_healthy_deployments` (team, web search, cooldown, blocked, plugins, +//! pre-call checks, tags, order, retry skips), then pick one with the strategy. Simple-shuffle is +//! native (`router_strategy/simple_shuffle.py`); other strategies go through `Host::select`. + +use std::{collections::BTreeMap, sync::Arc}; + +use parking_lot::Mutex; +use rand::{Rng, SeedableRng, rngs::StdRng}; + +use crate::{ + Core, RouterStateSnapshot, + config::{Op, RoutingStrategy}, + error::Error, + health_routing::WhenAllUnhealthy, + host::{RouterRequest, RoutingContext, SelectStage}, + registry::{Deployment, Registry, ResolveKind}, + resolve::{Caller, resolve_request}, + strategy::{StrategyScope, routing_context}, +}; + +/// Randomness for the shuffle and retry jitter, injectable so tests are deterministic. +pub trait Randomness: Send + Sync { + /// A float in `[0, 1)`, as Python's `random.random()`. + fn next_f64(&self) -> f64; + + /// An index below `len`, as Python's `random.choice` draws one. + fn index_below(&self, len: usize) -> usize { + ((self.next_f64() * len as f64) as usize).min(len - 1) + } +} + +pub struct ThreadRandomness; + +impl Randomness for ThreadRandomness { + fn next_f64(&self) -> f64 { + rand::thread_rng().r#gen() + } +} + +pub struct SeededRandomness(Mutex); + +impl SeededRandomness { + pub fn new(seed: u64) -> Self { + Self(Mutex::new(StdRng::seed_from_u64(seed))) + } +} + +impl Randomness for SeededRandomness { + fn next_f64(&self) -> f64 { + self.0.lock().r#gen() + } +} + +/// Which pass-through pick a request makes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PassThrough { + /// `get_available_deployment_for_pass_through` (and `select(kind="pass_through")`): the + /// pass-through deployments are chosen first, then cooldown and blocked filters, and only a + /// priority group filters by order. + Listed, + /// `async_get_available_deployment_for_pass_through`: the full healthy chain, then the + /// pass-through filter. + Healthy, +} + +/// A group's deployments before and after filtering. +#[derive(Clone, Debug)] +pub struct Candidates { + /// The name that was resolved (Python's `model`). + pub name: String, + pub model_group: String, + pub kind: ResolveKind, + /// Every deployment the caller may use (after the team filter). + pub all: Vec>, + pub healthy: Vec>, + /// The provider-prefixed spelling the name resolved under (`get_pattern`), when the host + /// was asked for one. + pub wildcard_request: Option, +} + +impl Candidates { + /// The `model` a no-deployments error names: what `_common_checks_available_deployment` + /// returned, so the alias-resolved name a wildcard serves and not its pattern. + pub(crate) fn failed_model(&self, registry: &Registry) -> String { + let model = match self.kind { + ResolveKind::Wildcard => registry.alias_of(&self.name).unwrap_or(&self.name), + _ => self.model_group.as_str(), + }; + model.to_owned() + } +} + +/// Per-attempt selection inputs the call loop carries. +#[derive(Clone, Debug, Default)] +pub(crate) struct SelectInputs { + pub target_order: Option, + pub skipped_ids: Vec, + /// A weighted failover hop's `_excluded_deployment_ids`: never picked. + pub excluded_ids: Arc<[String]>, + /// Stop before the plugins, order and skip filters: the group less cooldowns and blocked + /// deployments, which `_async_get_healthy_deployments` counts when deciding whether to retry. + pub count_only: bool, + /// The level's name, which an endpoint refusal reads (`endpoint_selection(model=...)`): the + /// name before pre-routing replaced it. + pub requested: Option, + /// A fallback hop: `run_async_fallback` pops the request's `_target_order`, so only the + /// hop's own order applies. + pub fallback_hop: bool, +} + +/// `simple_shuffle`: weighted by `_router_weights[group]`, else `weight`, else `rpm`, else +/// `tpm`, each normalized by its maximum, else uniform. A metric is read only when every set +/// before it weighed nothing, and a non-numeric one fails the pick then (`_metric_weight`). +pub fn simple_shuffle( + deployments: &[Arc], + model_group: &str, + router_weights: Option<&BTreeMap>>, + rng: &dyn Randomness, +) -> Result>, Error> { + if deployments.is_empty() { + return Ok(None); + } + let scoped = router_weights + .and_then(|weights| weights.get(model_group)) + .filter(|weights| !weights.is_empty()) + .map(|weights| { + deployments + .iter() + .map(|d| weights.get(&d.id).copied().unwrap_or(0.0)) + .collect::>() + }); + let metric_sets = ["weight", "rpm", "tpm"].into_iter().map(|metric| { + deployments + .iter() + .map(|d| d.metric_weight(metric)) + .collect::>>() + .ok_or(Error::NonNumericWeight { metric }) + }); + for weights in scoped.into_iter().map(Ok).chain(metric_sets) { + let weights = weights?; + let largest = weights.iter().copied().fold(0.0, f64::max); + if largest <= 0.0 { + continue; + } + let normalized: Vec = weights.iter().map(|weight| weight / largest).collect(); + let total: f64 = normalized.iter().sum(); + if total <= 0.0 { + continue; + } + // `random.choices`: bisect_right over the cumulative weights. + let target = rng.next_f64() * total; + let mut cumulative = 0.0; + let index = normalized + .iter() + .position(|weight| { + cumulative += weight; + target < cumulative + }) + .unwrap_or(deployments.len() - 1); + return Ok(Some(deployments[index].clone())); + } + Ok(Some( + deployments[rng.index_below(deployments.len())].clone(), + )) +} + +/// A pre-routing strategy marker (`classify_strategy_router_model`): its model is `auto_router/...`. +pub fn is_strategy_marker(deployment: &Deployment) -> bool { + deployment.params.model.starts_with("auto_router/") +} + +/// `litellm_params.use_in_pass_through`, read for truthiness as Python does. +fn uses_pass_through(deployment: &Deployment) -> bool { + match deployment.raw_params.get("use_in_pass_through") { + None | Some(serde_json::Value::Null) => false, + Some(serde_json::Value::Bool(value)) => *value, + Some(serde_json::Value::Number(value)) => value.as_f64().is_some_and(|n| n != 0.0), + Some(serde_json::Value::String(value)) => !value.is_empty(), + Some(serde_json::Value::Array(value)) => !value.is_empty(), + Some(serde_json::Value::Object(value)) => !value.is_empty(), + } +} + +/// `filter_team_based_models`: a team-owned deployment serves only its team, except that a proxy +/// admin without a team keeps deployments that are all team-owned and listed under the model the +/// request names, unless they belong to several teams. +fn filter_team_based( + deployments: Vec>, + request: &RouterRequest, +) -> Result>, Error> { + let team_id = request.team_id.as_deref(); + if team_id.is_none() + && request.proxy_admin + && let Some(requested) = request.admin_model.as_deref() + && !deployments.is_empty() + && deployments.iter().all(|d| { + d.model_info.team_id.is_some() + && (d.model_name == requested + || d.model_info.team_public_model_name.as_deref() == Some(requested)) + }) + { + let teams: std::collections::BTreeSet<&str> = deployments + .iter() + .filter_map(|d| d.model_info.team_id.as_deref()) + .collect(); + if teams.len() > 1 { + return Err(Error::MultipleTeams { + model: requested.to_owned(), + }); + } + return Ok(deployments); + } + Ok(deployments + .into_iter() + .filter(|d| d.usable_by_team(team_id)) + .collect()) +} + +impl Core { + fn context<'a>( + &'a self, + snapshot: &'a RouterStateSnapshot, + op: &'a Op, + request: &'a RouterRequest, + model_group: &'a str, + name: &'a str, + scope: StrategyScope, + ) -> RoutingContext<'a> { + RoutingContext { + registry: &snapshot.registry, + settings: &snapshot.settings, + health: &self.health, + request, + op, + model_group, + name, + scope, + pool: &[], + excluded_ids: &[], + skipped_ids: &[], + target_order: None, + } + } + + /// The eligible deployments for `name`, in filter-chain order. + pub(crate) async fn candidates( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + name: &str, + inputs: &SelectInputs, + ) -> Result { + let registry = &snapshot.registry; + // The async pick's pre-routing hook runs the routing plugins before anything resolves; + // every pick but a listed pass-through one reads what they left on the request. + let plugin_models = if snapshot.settings.routing_plugins + && !inputs.count_only + && request.pass_through != Some(PassThrough::Listed) + { + let ctx = self.context(snapshot, op, request, name, name, StrategyScope::Default); + self.host + .routing_plugin_models(&ctx, request.pre_routes) + .await? + } else { + None + }; + // No deployment for the name: route to the first default fallback. + let default_target = crate::resolve::default_target(&snapshot.settings.dynamic); + // `get_pattern`: a name no pattern matches is tried again under its inferred provider. + let wildcard_name = + crate::resolve::wildcard_name(registry, name, request.specific_deployment); + let wildcard_request = match wildcard_name { + Some(model) => { + let scope = StrategyScope::Default; + let ctx = self.context(snapshot, op, request, model, model, scope); + self.host.provider_request(&ctx).await? + } + None => None, + }; + // `endpoint_selection` wraps the attempt, not the count a retry decision reads. + let endpoint = request.endpoint.as_deref().filter(|_| !inputs.count_only); + let caller = Caller { + team_id: request.team_id.as_deref(), + proxy_admin: request.proxy_admin, + allowed_models: &request.allowed_models, + endpoint, + requested: inputs.requested.as_deref().unwrap_or(name), + wildcard_request: wildcard_request.as_deref(), + }; + let resolution = resolve_request( + registry, + name, + request.specific_deployment, + caller, + default_target, + snapshot.settings.dynamic.default_ref(), + jiff::Timestamp::now(), + )? + .ok_or_else(|| Error::UnknownModel { + // The name `_common_checks_available_deployment` looked for, past the alias. + model: registry.alias_of(name).unwrap_or(name).to_owned(), + })?; + let first_model = resolution + .deployments + .first() + .map(|d| d.params.model.clone()); + let team_id = request.team_id.as_deref(); + let usable = filter_team_based(resolution.deployments.clone(), request)?; + // `_filter_reserved_deployments`: PTU shares keep holders to their share and everyone + // else to the deployments split into none. + let (mut all, withheld) = crate::ptu::filter_shared(usable, team_id); + if withheld && all.is_empty() { + return Err(Error::PtuReserved { + model: match resolution.kind { + ResolveKind::Id => first_model.unwrap_or_else(|| name.to_owned()), + ResolveKind::Wildcard | ResolveKind::SpecificDeployment => name.to_owned(), + _ => resolution.group.clone(), + }, + }); + } + // `filter_web_search_deployments` runs before `request_deployments` is read, so every later + // stage sees the group without the deployments it dropped. + if request.web_search && !inputs.count_only && !resolution.kind.is_single() { + all.retain(|d| d.supports_web_search()); + } + // `_without_deployments_that_cannot_serve_route`, right after the web search filter: the + // deployments whose provider cannot serve the call, unless that leaves nothing. Only a pick + // runs it (`_route_capability_pin`): the retry count reads the whole group. + if let Some(handler) = request.route_handler.as_deref() + && !inputs.count_only + && !resolution.kind.is_single() + && !all.is_empty() + { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let stage = SelectStage::RouteIncapable { + handler: handler.to_owned(), + }; + let incapable = self.host.select(&stage, &ctx, all.clone()).await?; + let capable: Vec> = all + .iter() + .filter(|d| !incapable.iter().any(|i| i.id == d.id)) + .cloned() + .collect(); + if !capable.is_empty() { + all = capable; + } + } + let listed = + request.pass_through == Some(PassThrough::Listed) && !resolution.kind.is_single(); + if listed { + all.retain(|d| uses_pass_through(d)); + if all.is_empty() { + // The name `_common_checks_available_deployment` resolved: an alias's target, a + // routing group's name, the alias-resolved name a wildcard serves. + let model = match resolution.kind { + ResolveKind::Wildcard => registry.alias_of(name).unwrap_or(name), + ResolveKind::SpecificDeployment => name, + _ => resolution.group.as_str(), + }; + return Err(Error::NoPassThroughDeployments { + model: model.to_owned(), + }); + } + } + + // A deployment named by its id is served even while it cools down + // (`get_deployment_by_model_id` returns before the cooldown filter). + let settings = &snapshot.settings; + let cooling = if resolution.kind.is_single() { + Vec::new() + } else { + // `async_get_cooldown_deployments` reads every deployment's, as one throttled batch. + self.health + .active_cooldowns(&snapshot.registry.ids()) + .await? + }; + // Health-check filtering runs before the cooldown filter, except for a deployment named + // by its id and the healthy count a retry decision reads. + let checked = if resolution.kind.is_single() || inputs.count_only { + all.clone() + } else { + let when_all_unhealthy = if request.health_check_probe { + WhenAllUnhealthy::Nothing + } else { + WhenAllUnhealthy::KeepAll + }; + self.health + .health_check_filter(settings, all.clone(), when_all_unhealthy) + .await + }; + let mut healthy: Vec> = checked + .iter() + .filter(|d| !cooling.iter().any(|(id, _)| *id == d.id)) + .cloned() + .collect(); + // The safety net when health-check routing drives cooldowns (an allowed_fails_policy): + // never on a listed pass-through pick or the healthy count. + if healthy.is_empty() + && !listed + && !inputs.count_only + && settings.enable_health_check_routing + && settings.allowed_fails_policy.is_some() + { + healthy = checked; + } + if resolution.kind.is_single() + && let Some(deployment) = all.first() + { + // `_core_async_get_healthy_deployments`: a deployment named by id is checked alone. + if deployment.model_info.blocked { + return Err(Error::Paused { + model: deployment.params.model.clone(), + }); + } + if request.pass_through.is_some() && !uses_pass_through(deployment) { + return Err(Error::NotPassThrough { + id: deployment.id.clone(), + }); + } + } + healthy.retain(|d| !d.model_info.blocked); + // `continuation_deployments`: a mid-stream continuation never lands on a deployment the host + // named as unable to take its prefill. + if let Some(excluded) = request + .body + .get("continuation_excluded_ids") + .and_then(serde_json::Value::as_array) + { + healthy.retain(|d| !excluded.iter().any(|id| id.as_str() == Some(d.id.as_str()))); + } + let request_order = request.target_order.filter(|_| !inputs.fallback_hop); + // `async_callback_filter_deployments` runs after the cooldown and blocked filters on every + // async candidate list but a deployment named by id and a listed pass-through pick. + if settings.callback_filters + && !inputs.count_only + && !resolution.kind.is_single() + && request.pass_through != Some(PassThrough::Listed) + { + let mut ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + ctx.excluded_ids = &inputs.excluded_ids; + ctx.skipped_ids = &inputs.skipped_ids; + ctx.target_order = inputs.target_order.or(request_order); + healthy = self.host.callback_filters(&ctx, healthy).await?; + } + if inputs.count_only { + return Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }); + } + + if listed { + if settings.enable_pre_call_checks && !resolution.kind.is_single() { + healthy = self + .pre_call_checks( + snapshot, + op, + request, + &resolution.group, + name, + &all, + healthy, + ) + .await?; + } + // Only a priority group narrows a listed pass-through pick by order. + let priority = resolution.kind == ResolveKind::RoutingGroup + && registry.routing_groups().iter().any(|g| { + g.group_name == resolution.group && crate::routing_groups::is_priority(g) + }); + if priority { + let order = request + .target_order + .or_else(|| healthy.iter().filter_map(|d| d.order()).min()); + if let Some(order) = order { + healthy.retain(|d| d.order() == Some(order)); + } + } + return Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }); + } + + if !settings.plugins.is_empty() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + healthy = self + .host + .select(&SelectStage::Plugins, &ctx, healthy) + .await?; + } + + if settings.enable_pre_call_checks && !resolution.kind.is_single() { + healthy = self + .pre_call_checks( + snapshot, + op, + request, + &resolution.group, + name, + &all, + healthy, + ) + .await?; + } + + if request.filter_tags && !healthy.is_empty() { + let mut ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + ctx.pool = &all; + healthy = self.host.select(&SelectStage::Tags, &ctx, healthy).await?; + } + + if !resolution.kind.is_single() { + healthy = crate::routing_plugins::narrow(healthy, plugin_models.as_deref())?; + } + + // `_without_declared_text_only_deployments`: an image request skips the deployments that + // declare no vision, unless that leaves nothing. + if request.has_image && !resolution.kind.is_single() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let text_only = self + .host + .select(&SelectStage::TextOnly, &ctx, healthy.clone()) + .await?; + let vision: Vec> = healthy + .iter() + .filter(|d| !text_only.iter().any(|t| t.id == d.id)) + .cloned() + .collect(); + if !vision.is_empty() { + healthy = vision; + } + } + + // `_without_decisions_deployments_refusing_media`: a Decisions call carrying audio or + // videos skips the deployments that refuse them, unless that leaves nothing. + if request.decisions_media && !resolution.kind.is_single() && !healthy.is_empty() { + let ctx = self.context( + snapshot, + op, + request, + &resolution.group, + name, + StrategyScope::Default, + ); + let refusing = self + .host + .select(&SelectStage::MediaRefusing, &ctx, healthy.clone()) + .await?; + let taking: Vec> = healthy + .iter() + .filter(|d| !refusing.iter().any(|r| r.id == d.id)) + .cloned() + .collect(); + if !taking.is_empty() { + healthy = taking; + } + } + + if let Some(pinned) = request.pinned_deployment_id.as_deref() + && healthy.iter().any(|d| d.id == pinned) + { + healthy.retain(|d| d.id == pinned); + } + + let order = inputs + .target_order + .or(request_order) + .or_else(|| healthy.iter().filter_map(|d| d.order()).min()); + if let Some(order) = order { + healthy.retain(|d| d.order() == Some(order)); + } + + healthy.retain(|d| { + !request.excluded_ids.contains(&d.id) && !inputs.excluded_ids.contains(&d.id) + }); + + if !inputs.skipped_ids.is_empty() || !request.retry_skipped_ids.is_empty() { + let kept: Vec> = healthy + .iter() + .filter(|d| { + !inputs.skipped_ids.contains(&d.id) + && !request.retry_skipped_ids.contains(&d.id) + }) + .cloned() + .collect(); + if !kept.is_empty() { + healthy = kept; + } + } + + if request.pass_through == Some(PassThrough::Healthy) + && !resolution.kind.is_single() + && !healthy.is_empty() + { + healthy.retain(|d| uses_pass_through(d)); + if healthy.is_empty() { + return Err(Error::NoPassThroughDeployments { + model: name.to_owned(), + }); + } + } + + Ok(Candidates { + name: name.to_owned(), + model_group: resolution.group, + kind: resolution.kind, + all, + healthy, + wildcard_request, + }) + } + + /// `_pre_call_checks` over `healthy`, with the host's facts. + #[allow(clippy::too_many_arguments)] + async fn pre_call_checks( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + model_group: &str, + name: &str, + pool: &[Arc], + healthy: Vec>, + ) -> Result>, Error> { + let mut ctx = self.context( + snapshot, + op, + request, + model_group, + name, + StrategyScope::Default, + ); + ctx.pool = pool; + let facts = self.host.pre_call_facts(&ctx, &healthy).await?; + crate::precall::filter(model_group, healthy, pool, &facts) + } + + /// Pick one of `candidates.healthy` with the group's strategy. + pub(crate) async fn pick( + &self, + snapshot: &RouterStateSnapshot, + op: &Op, + request: &RouterRequest, + candidates: &Candidates, + ) -> Result, Error> { + let no_deployment = || Error::NoHealthyDeployments { + model: candidates.failed_model(&snapshot.registry), + }; + // A deployment named by its id is returned as is, before any strategy runs. + if candidates.kind.is_single() { + return candidates + .healthy + .first() + .cloned() + .ok_or_else(no_deployment); + } + let (strategy, scope) = routing_context( + &snapshot.registry, + &snapshot.settings.routing_strategy, + request.routing_strategy.as_deref(), + &candidates.kind, + &candidates.model_group, + ); + if strategy == RoutingStrategy::Unset { + return Err(no_deployment()); + } + if strategy == RoutingStrategy::SimpleShuffle { + let weights = crate::weights::validate(request.router_weights.as_ref()) + .map_err(|_| Error::InvalidRouterWeights)?; + return simple_shuffle( + &candidates.healthy, + &candidates.model_group, + weights.as_ref(), + self.rng.as_ref(), + )? + .ok_or_else(no_deployment); + } + let ctx = self.context( + snapshot, + op, + request, + &candidates.model_group, + &candidates.name, + scope, + ); + self.host + .select( + &SelectStage::Strategy(strategy), + &ctx, + candidates.healthy.clone(), + ) + .await? + .into_iter() + .next() + .ok_or_else(no_deployment) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::registry::DeploymentSpec; + + struct Fixed(f64); + + impl Randomness for Fixed { + fn next_f64(&self) -> f64 { + self.0 + } + } + + fn deployment(id: &str, params: serde_json::Value) -> Arc { + let spec: DeploymentSpec = serde_json::from_value(json!({ + "model_name": "g", "litellm_params": params, "model_info": {"id": id}, + })) + .unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + } + + #[rstest] + #[case::weight_low_draw(0.1, "a")] + #[case::weight_high_draw(0.9, "b")] + fn shuffle_uses_weight_before_rpm(#[case] draw: f64, #[case] expected: &str) { + let deployments = [ + deployment("a", json!({"model": "openai/a", "weight": 1, "rpm": 1000})), + deployment("b", json!({"model": "openai/b", "weight": 3})), + ]; + let picked = simple_shuffle(&deployments, "g", None, &Fixed(draw)) + .unwrap() + .unwrap(); + assert_eq!(picked.id, expected); + } + + #[rstest] + fn shuffle_prefers_request_weights() { + let deployments = [ + deployment("a", json!({"model": "openai/a", "weight": 100})), + deployment("b", json!({"model": "openai/b"})), + ]; + let weights = BTreeMap::from([("g".to_owned(), BTreeMap::from([("b".to_owned(), 1.0)]))]); + let picked = simple_shuffle(&deployments, "g", Some(&weights), &Fixed(0.99)) + .unwrap() + .unwrap(); + assert_eq!(picked.id, "b"); + } +} diff --git a/litellm-rust/crates/router-core/src/settings.rs b/litellm-rust/crates/router-core/src/settings.rs new file mode 100644 index 00000000000..f55b059d71c --- /dev/null +++ b/litellm-rust/crates/router-core/src/settings.rs @@ -0,0 +1,133 @@ +//! [`Settings`]: one typed, frozen view of the configuration a caller may read, built from the +//! static config and the current dynamic snapshot. + +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use crate::{ + config::{ + AllowedFailsPolicy, DEFAULT_COOLDOWN_TIME, PluginSpec, RetryPolicy, RouterConfig, + RouterDynamicConfig, RoutingStrategy, + }, + error::AttemptError, +}; + +#[derive(Clone, Debug)] +pub struct Settings { + pub routing_strategy: RoutingStrategy, + pub plugins: Arc<[PluginSpec]>, + pub cooldown_time: Duration, + pub disable_cooldowns: bool, + pub allowed_fails: Option, + pub allowed_fails_policy: Option, + pub retry_policy: Option, + pub model_group_retry_policy: Arc>, + pub num_retries: u32, + pub max_fallbacks: u32, + pub retry_after: Duration, + pub timeout: Option, + pub drop_params: Option, + pub enable_pre_call_checks: bool, + pub routing_plugins: bool, + pub callback_filters: bool, + pub enable_health_check_routing: bool, + pub health_check_staleness: Duration, + pub background_health_check_model_groups: Option>, + pub enable_weighted_failover: bool, + pub fallback_on_unsupported_params: bool, + pub provider_default_deployment_ids: Arc<[String]>, + pub dynamic: Arc, +} + +/// `DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER`. +pub const DEFAULT_HEALTH_CHECK_STALENESS: Duration = Duration::from_secs(300 * 2); + +fn seconds(value: f64) -> Duration { + Duration::try_from_secs_f64(value).unwrap_or_default() +} + +impl Settings { + pub fn new(config: &RouterConfig, dynamic: Arc) -> Self { + Self { + routing_strategy: config.routing_strategy.clone(), + plugins: config.plugins.clone().into(), + cooldown_time: config.cooldown_time.map_or(DEFAULT_COOLDOWN_TIME, seconds), + disable_cooldowns: config.disable_cooldowns, + allowed_fails: dynamic.allowed_fails, + allowed_fails_policy: config.allowed_fails_policy.clone(), + retry_policy: config.retry_policy.clone(), + model_group_retry_policy: Arc::new(config.model_group_retry_policy.clone()), + num_retries: dynamic.num_retries.unwrap_or(0), + max_fallbacks: dynamic.max_fallbacks, + retry_after: seconds(config.retry_after), + timeout: config.request_timeout.or(config.timeout).map(seconds), + drop_params: config.drop_params, + enable_pre_call_checks: config.enable_pre_call_checks, + routing_plugins: config.routing_plugins, + callback_filters: config.callback_filters, + enable_health_check_routing: config.enable_health_check_routing, + health_check_staleness: config + .health_check_staleness_threshold + .map_or(DEFAULT_HEALTH_CHECK_STALENESS, seconds), + background_health_check_model_groups: config + .background_health_check_model_groups + .clone() + .map(Into::into), + enable_weighted_failover: config.enable_weighted_failover, + fallback_on_unsupported_params: config.fallback_on_unsupported_params.unwrap_or(true), + provider_default_deployment_ids: config.provider_default_deployment_ids.clone().into(), + dynamic, + } + } + + /// `allowed_fails_for`: the policy's count for this failure. + pub fn allowed_fails_for(&self, failure: &AttemptError) -> Option { + self.allowed_fails_policy + .as_ref() + .and_then(|policy| policy.allowed_fails_for(failure)) + } + + /// How long an allowed-fails counter lives: `settings().cooldown_time or + /// DEFAULT_COOLDOWN_TIME_SECONDS`, so a router cooldown of 0 still counts over the default. + pub fn fails_window(&self) -> Duration { + if self.cooldown_time.is_zero() { + DEFAULT_COOLDOWN_TIME + } else { + self.cooldown_time + } + } + + /// `_is_allowed_fails_set_on_router`. + pub fn allowed_fails_set(&self) -> bool { + self.allowed_fails.is_some() && !self.dynamic.allowed_fails_is_default + } + + /// Python's `allowed_fails` default when set without a value (`litellm.allowed_fails`). + pub fn effective_allowed_fails(&self) -> u32 { + self.allowed_fails.unwrap_or(3) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::passed_to_the_router(Some(3), false, true)] + #[case::the_process_default(Some(3), true, false)] + #[case::none(None, false, false)] + fn allowed_fails_counts_as_set_unless_it_is_the_process_default( + #[case] allowed_fails: Option, + #[case] allowed_fails_is_default: bool, + #[case] expected: bool, + ) { + let dynamic = RouterDynamicConfig { + allowed_fails, + allowed_fails_is_default, + ..RouterDynamicConfig::default() + }; + let settings = Settings::new(&RouterConfig::default(), Arc::new(dynamic)); + assert_eq!(settings.allowed_fails_set(), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/strategy.rs b/litellm-rust/crates/router-core/src/strategy.rs new file mode 100644 index 00000000000..0e1f3b49fa8 --- /dev/null +++ b/litellm-rust/crates/router-core/src/strategy.rs @@ -0,0 +1,110 @@ +//! Which routing strategy picks for a request, and whose state it reads: a port of +//! `Router._get_routing_context`. A per-request `routing_strategy` override wins, then the +//! routing group the resolved model belongs to (called as the group, or as a member of a +//! non-priority group), then the router's own strategy. + +use crate::{ + config::RoutingStrategy, + registry::{Registry, ResolveKind}, +}; + +/// The per-request overrides `_OVERRIDABLE_ROUTING_STRATEGIES` honours: simple-shuffle and every +/// strategy with a default selector. Anything else is ignored. +pub const OVERRIDABLE: [&str; 6] = [ + "simple-shuffle", + "least-busy", + "usage-based-routing", + "usage-based-routing-v2", + "latency-based-routing", + "cost-based-routing", +]; + +/// Whose selector state a strategy pick reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StrategyScope { + /// The router's own strategy (the implicit `"default"` group). + Default, + /// A routing group's own selector. + Group(String), + /// A per-request override: the default selector when it names the router's strategy, else + /// one selector per strategy kept for the router's life. + Override, +} + +impl StrategyScope { + /// The scope as the host step spells it. + pub fn wire(&self) -> (&'static str, Option<&str>) { + match self { + Self::Default => ("default", None), + Self::Group(name) => ("group", Some(name)), + Self::Override => ("override", None), + } + } +} + +/// A request's `routing_strategy` override, when it is one Python honours. +pub fn request_override(raw: Option<&str>) -> Option { + let raw = raw?; + OVERRIDABLE + .contains(&raw) + .then(|| raw.parse().unwrap_or_default()) +} + +/// The strategy and selector scope for a pick of `model_group` (the resolved group, after +/// aliases), reached through a resolution of `kind`. +pub fn routing_context( + registry: &Registry, + default: &RoutingStrategy, + override_strategy: Option<&str>, + kind: &ResolveKind, + model_group: &str, +) -> (RoutingStrategy, StrategyScope) { + if let Some(strategy) = request_override(override_strategy) { + return (strategy, StrategyScope::Override); + } + let group = match kind { + ResolveKind::RoutingGroup => registry + .routing_groups() + .iter() + .find(|group| group.group_name == model_group), + ResolveKind::Alias | ResolveKind::Group => registry.member_routing_group(model_group), + ResolveKind::SpecificDeployment + | ResolveKind::Id + | ResolveKind::Default + | ResolveKind::Wildcard => None, + }; + match group { + None => (default.clone(), StrategyScope::Default), + Some(group) if is_priority(&group.routing_strategy) => { + (RoutingStrategy::SimpleShuffle, StrategyScope::Default) + } + Some(group) => ( + group.routing_strategy.clone(), + StrategyScope::Group(group.group_name.clone()), + ), + } +} + +fn is_priority(strategy: &RoutingStrategy) -> bool { + matches!(strategy, RoutingStrategy::Other(name) if name == "priority") +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::honoured(Some("least-busy"), Some(RoutingStrategy::LeastBusy))] + #[case::shuffle(Some("simple-shuffle"), Some(RoutingStrategy::SimpleShuffle))] + #[case::lar1_ignored(Some("lar1"), None)] + #[case::unknown_ignored(Some("bogus"), None)] + #[case::unset(None, None)] + fn only_overridable_strategies_override( + #[case] raw: Option<&str>, + #[case] expected: Option, + ) { + assert_eq!(request_override(raw), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/weights.rs b/litellm-rust/crates/router-core/src/weights.rs new file mode 100644 index 00000000000..3e5d76844ab --- /dev/null +++ b/litellm-rust/crates/router-core/src/weights.rs @@ -0,0 +1,81 @@ +//! `_router_weights` as `validate_router_weights` reads it (`litellm/types/router_weights.py`): +//! model group to deployment id to a finite weight of at least 0 (an int or a float, never a +//! bool or a string), with non-blank ids and at least one positive weight in every non-empty group. + +use std::collections::BTreeMap; + +use serde_json::Value; + +/// Validated request weights. +pub type RouterWeights = BTreeMap>; + +/// The request's weights didn't validate; the host raises the validator's own error. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InvalidRouterWeights; + +fn identifier(text: &str) -> bool { + text.chars().any(|c| !c.is_whitespace()) +} + +fn weight(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_f64().filter(|w| w.is_finite() && *w >= 0.0), + _ => None, + } +} + +/// `validate_router_weights(raw)`: `None` when the request passed none. +pub fn validate(raw: Option<&Value>) -> Result, InvalidRouterWeights> { + let groups = match raw { + None | Some(Value::Null) => return Ok(None), + Some(Value::Object(groups)) => groups, + Some(_) => return Err(InvalidRouterWeights), + }; + let mut validated = RouterWeights::new(); + for (group, weights) in groups { + let Value::Object(weights) = weights else { + return Err(InvalidRouterWeights); + }; + if !identifier(group) { + return Err(InvalidRouterWeights); + } + let mut entries = BTreeMap::new(); + for (id, value) in weights { + if !identifier(id) { + return Err(InvalidRouterWeights); + } + entries.insert(id.clone(), weight(value).ok_or(InvalidRouterWeights)?); + } + if !entries.is_empty() && !entries.values().any(|w| *w > 0.0) { + return Err(InvalidRouterWeights); + } + validated.insert(group.clone(), entries); + } + Ok(Some(validated)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::all_zero(json!({"g": {"a": 0.0, "b": 0.0}}))] + #[case::negative(json!({"g": {"a": -1.0}}))] + #[case::text(json!({"g": {"a": "1"}}))] + #[case::boolean(json!({"g": {"a": true}}))] + #[case::blank_id(json!({"g": {" ": 1.0}}))] + #[case::not_a_mapping(json!(["g"]))] + fn invalid_weights_are_refused(#[case] raw: Value) { + assert_eq!(validate(Some(&raw)), Err(InvalidRouterWeights)); + } + + #[rstest] + #[case::empty_group(json!({"g": {}}))] + #[case::int_weight(json!({"g": {"a": 1, "b": 0}}))] + fn valid_weights_are_kept(#[case] raw: Value) { + assert!(validate(Some(&raw)).unwrap().is_some()); + } +} diff --git a/litellm-rust/crates/router-core/tests/call_loop.rs b/litellm-rust/crates/router-core/tests/call_loop.rs new file mode 100644 index 00000000000..9f41371f255 --- /dev/null +++ b/litellm-rust/crates/router-core/tests/call_loop.rs @@ -0,0 +1,1757 @@ +//! A call through the router end to end: selection, retries, fallbacks and cooldowns against a +//! scripted host. + +use std::{collections::VecDeque, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use litellm_router_core::{ + Attempt, AttemptError, Deployment, Error, FailureKind, FailureReport, Host, + InMemoryRouterCache, Op, Output, Payload, Router, RouterCache, RouterConfig, + RouterDynamicConfig, RouterEvent, RouterRequest, RoutingContext, SeededRandomness, SelectStage, + SuccessReport, + call::StreamOrigin, + fallback::Fallbacks, + health::{UsageMetric, current_minute, usage_key}, + host::UncountedRetries, + select::PassThrough, +}; +use parking_lot::Mutex; +use rstest::{fixture, rstest}; +use serde_json::json; + +#[derive(Default)] +struct ScriptedHost { + /// Failures handed out in order, keyed by model group; a group with none left succeeds. + script: Mutex)>>, + events: Mutex>, + cache: InMemoryRouterCache, + /// Answer every success as a stream whose outcome the host reports when it ends. + deferred: bool, + /// [`Output::tracks_usage`] on every success. + tracks_usage: Option, + /// Refuse every fallback chain with a dict target. + refuses_dict_targets: bool, + /// Every fallback check raises this exception class, as a host's sync check does. + fallback_check_raises: Option<&'static str>, + /// Failures the tag stage raises in turn, before it answers. + pick_failures: Mutex>, + /// The next attempt's failure, whatever group it serves. + first_failure: Mutex>, +} + +impl ScriptedHost { + fn failing(group: &str, failures: Vec) -> Self { + Self::scripted(vec![(group, failures)]) + } + + fn scripted(groups: Vec<(&str, Vec)>) -> Self { + Self { + script: Mutex::new( + groups + .into_iter() + .map(|(group, failures)| (group.to_owned(), failures.into())) + .collect(), + ), + ..Self::default() + } + } + + fn selected(&self) -> Vec { + self.events + .lock() + .iter() + .filter_map(|event| match event { + RouterEvent::Selected { deployment_id, .. } => Some(deployment_id.clone()), + _ => None, + }) + .collect() + } + + fn retries(&self) -> usize { + self.events + .lock() + .iter() + .filter(|event| matches!(event, RouterEvent::Retry { .. })) + .count() + } +} + +#[async_trait] +impl Host for ScriptedHost { + async fn invoke(&self, attempt: &Attempt) -> Result { + if let Some(failure) = self.first_failure.lock().take() { + return Err(failure); + } + let next = self + .script + .lock() + .iter_mut() + .find(|(group, _)| *group == attempt.model_group) + .and_then(|(_, failures)| failures.pop_front()); + match next { + Some(failure) => Err(failure), + None => Ok(Output { + value: Payload(Arc::new(attempt.deployment.id.clone())), + total_tokens: 10, + blocked: None, + deferred: self.deferred, + tracks_usage: self.tracks_usage, + }), + } + } + + async fn select( + &self, + _stage: &SelectStage, + _ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + match self.pick_failures.lock().pop_front() { + Some(failure) => Err(failure.into()), + None => Ok(candidates), + } + } + + async fn allow_fallback( + &self, + _request: &RouterRequest, + _target: &str, + ) -> Result { + match self.fallback_check_raises { + Some(class) => Err(AttemptError::host_raised(&[class])), + None => Ok(true), + } + } + + async fn refuses_fallbacks( + &self, + _level: u32, + targets: &[serde_json::Map], + ) -> bool { + self.refuses_dict_targets && !targets.is_empty() + } + + async fn sleep(&self, _duration: Duration) {} + + fn observe(&self, event: &RouterEvent) { + self.events.lock().push(event.clone()); + } + + fn cache(&self) -> Arc { + Arc::new(self.cache.clone()) + } +} + +#[fixture] +fn config() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}}, + ] + })) + .unwrap() +} + +fn router(config: RouterConfig, dynamic: serde_json::Value, host: Arc) -> Router { + let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap(); + Router::builder(config, dynamic, host) + .randomness(Arc::new(SeededRandomness::new(7))) + .build() + .unwrap() +} + +fn request() -> RouterRequest { + RouterRequest { + model: "primary".to_owned(), + ..RouterRequest::default() + } +} + +/// `primary` at order 1, its second deployment at order 2 when `tiered`, and two other groups. +fn tiers(tiered: bool) -> RouterConfig { + let second = if tiered { "primary" } else { "unused" }; + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/p1", "order": 1}, "model_info": {"id": "p1"}}, + {"model_name": second, "litellm_params": {"model": "openai/p2", "order": 2}, "model_info": {"id": "p2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/b1"}, "model_info": {"id": "b1"}}, + {"model_name": "context", "litellm_params": {"model": "anthropic/c1"}, "model_info": {"id": "c1"}}, + ] + })) + .unwrap() +} + +fn failure(kind: FailureKind, status: u16) -> AttemptError { + AttemptError::new(kind, Some(status)) +} + +fn context_window() -> AttemptError { + failure(FailureKind::ContextWindowExceeded, 400) +} + +fn server_error() -> AttemptError { + failure(FailureKind::InternalServer, 500) +} + +fn agentic_followup() -> AttemptError { + let mut failed = server_error(); + failed.agentic_followup = true; + failed +} + +#[rstest] +#[tokio::test] +async fn a_rate_limit_retries_in_the_same_group(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router(config, json!({"num_retries": 2}), host.clone()); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + ( + response.model_group.as_str(), + response.attempted_retries, + response.attempted_fallbacks + ), + ("primary", 1, 0) + ); +} + +#[rstest] +#[tokio::test] +async fn a_bad_request_falls_back_without_retrying(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::BadRequest, Some(400))], + )); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let retried = host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::Retry { .. })); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks, + retried + ), + ("b1", 1, false) + ); +} + +#[rstest] +#[tokio::test] +async fn a_rate_limit_cools_the_deployment_down(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router(config, json!({}), host.clone()); + let error = router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + let cooled = router + .health() + .active_cooldowns(&["a1".to_owned(), "a2".to_owned()]) + .await + .unwrap(); + assert_eq!( + (error.attempt().map(|failure| failure.kind), cooled.len()), + (Some(FailureKind::RateLimit), 1) + ); +} + +#[rstest] +#[case::now(false, (Some(1.0), (10, 1)))] +#[case::deferred(true, (None, (10, 1)))] +#[tokio::test] +async fn a_deferred_attempt_counts_its_request_and_tokens_but_no_success( + #[case] deferred: bool, + #[case] expected: (Option, (u64, u64)), +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a", "rpm": 100}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + let successes = host + .cache + .get("a1:successes") + .and_then(|count| count.as_f64()); + let usage = router.health().usage(&deployment).await.unwrap(); + assert_eq!((successes, usage), expected); +} + +/// `litellm_params`, `model_info` and a limit only the host knows of all count; a realtime +/// session (`Some(false)` on a deferred answer) counts nothing, whatever limits it has. +#[rstest] +#[case::in_params(json!({"rpm": 100}), json!({}), false, None, (10, 1))] +#[case::in_model_info(json!({}), json!({"rpm": 100}), false, None, (10, 1))] +#[case::in_the_catalog(json!({}), json!({}), false, Some(true), (10, 1))] +#[case::no_limit(json!({}), json!({}), false, None, (0, 0))] +#[case::a_stream_in_model_info(json!({}), json!({"rpm": 100}), true, None, (10, 1))] +#[case::a_realtime_session(json!({"rpm": 100}), json!({}), true, Some(false), (0, 0))] +#[tokio::test] +async fn usage_counts_for_a_deployment_that_tracks_it( + #[case] params: serde_json::Value, + #[case] info: serde_json::Value, + #[case] deferred: bool, + #[case] tracks_usage: Option, + #[case] expected: (u64, u64), +) { + let mut litellm_params = json!({"model": "openai/a"}); + litellm_params + .as_object_mut() + .unwrap() + .extend(params.as_object().unwrap().clone()); + let mut model_info = json!({"id": "a1"}); + model_info + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": litellm_params, "model_info": model_info}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred, + tracks_usage, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + assert_eq!(router.health().usage(&deployment).await.unwrap(), expected); +} + +/// A stream end's report of a wildcard deployment counts under the model the route filled, the +/// key its request was counted under, not the configured `*`. +#[rstest] +#[tokio::test] +async fn a_reported_success_keeps_the_filled_wildcard_model() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "rpm": 100}, "model_info": {"id": "w1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let report = SuccessReport { + usage_model: Some("openai/gpt-4o".to_owned()), + ..SuccessReport::default() + }; + router.report_success("w1", 7.0, 0.0, report).await.unwrap(); + let minute = current_minute(std::time::SystemTime::now()); + let key = |model: &str| usage_key("w1", model, UsageMetric::Tpm, &minute); + assert_eq!( + ( + host.cache.get(&key("openai/gpt-4o")), + host.cache.get(&key("openai/*")), + ), + (Some(json!(7)), None), + ); +} + +/// `async_deployment_callback_on_failure`'s request is written only when it runs: a failure the +/// host says ran no failure callback (`recorded`) writes none, nor does a sync call's. +#[rstest] +#[case::the_first_logged_failure(false, false, 1, Some(1.0))] +#[case::a_later_failure_on_the_shared_logging_object(true, false, 0, None)] +#[case::a_sync_call(false, true, 0, Some(1.0))] +#[tokio::test] +async fn a_failed_attempt_counts_its_request_only_when_its_callback_ran( + #[case] recorded: bool, + #[case] uncounted_request: bool, + #[case] requests: u64, + #[case] fails: Option, +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let mut failure = AttemptError::new(FailureKind::InternalServer, Some(500)); + failure.recorded = recorded; + failure.uncounted_request = uncounted_request; + let host = Arc::new(ScriptedHost::failing("primary", vec![failure])); + let router = router(config, json!({"num_retries": 0}), host.clone()); + let _ = router.call(Op::from("acompletion"), request()).await; + let deployment = router.registry().deployment("a1").unwrap(); + let usage = router.health().usage(&deployment).await.unwrap(); + let counted = host.cache.get("a1:fails").and_then(|count| count.as_f64()); + assert_eq!((usage.1, counted), (requests, fails)); +} + +#[rstest] +#[case::counted(true, 2)] +#[case::uncounted(false, 1)] +#[tokio::test] +async fn a_reported_failure_counts_its_request_when_asked( + #[case] counts_request: bool, + #[case] requests: u64, +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a", "rpm": 100}, "model_info": {"id": "a1"}}, + ] + })) + .unwrap(); + let host = Arc::new(ScriptedHost { + deferred: true, + ..ScriptedHost::default() + }); + let router = router(config, json!({}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let report = FailureReport { + counts_request, + ..FailureReport::default() + }; + router + .report_failure( + "a1", + &AttemptError::new(FailureKind::RateLimit, Some(429)), + report, + ) + .await + .unwrap(); + let deployment = router.registry().deployment("a1").unwrap(); + let fails = host.cache.get("a1:fails").and_then(|count| count.as_f64()); + let usage = router.health().usage(&deployment).await.unwrap(); + assert_eq!((fails, usage.1), (Some(1.0), requests)); +} + +#[rstest] +#[tokio::test] +async fn a_raising_retry_policy_lookup_ends_the_hop_and_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![AttemptError::new(FailureKind::RateLimit, Some(429))], + )); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let request = RouterRequest { + retry_policy_raises: vec!["primary".to_owned()], + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + let events = host.events.lock(); + assert!( + !events + .iter() + .any(|e| matches!(e, RouterEvent::Retry { .. })) + ); + assert!(events.iter().any(|e| matches!( + e, + RouterEvent::RetryPolicyRaised { level: 0, model, .. } if model == "primary" + ))); + assert_eq!(response.model_group, "backup"); +} + +#[rstest] +#[tokio::test] +async fn a_num_retries_that_raises_ends_the_hop_and_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let request = RouterRequest { + uncounted_retries: Some(UncountedRetries::Raises), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + let events = host.events.lock(); + assert!( + !events + .iter() + .any(|e| matches!(e, RouterEvent::Retry { .. })) + ); + assert!(events.iter().any(|e| matches!( + e, + RouterEvent::RetryCountRaised { level: 0, model, .. } if model == "primary" + ))); + assert_eq!(response.model_group, "backup"); +} + +#[rstest] +#[case::negative(UncountedRetries::Negative)] +#[case::not_positive(UncountedRetries::NotPositive)] +#[tokio::test] +async fn a_num_retries_below_one_leaves_the_failure_standing( + config: RouterConfig, + #[case] uncounted: UncountedRetries, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let request = RouterRequest { + uncounted_retries: Some(uncounted), + ..request() + }; + let refused = router.call(Op::from("acompletion"), request).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.status == Some(500))); + assert_eq!(host.retries(), 0); + assert!( + !host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::RetryCountRaised { .. })) + ); +} + +#[rstest] +#[tokio::test] +async fn a_retry_policy_that_grants_retries_outranks_a_num_retries_that_raises( + config: RouterConfig, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + RouterConfig { + retry_policy: serde_json::from_value(json!({"DefaultRetries": 1})).unwrap(), + ..config + }, + json!({}), + host.clone(), + ); + let request = RouterRequest { + uncounted_retries: Some(UncountedRetries::Raises), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + assert_eq!(response.model_group, "primary"); + assert_eq!(host.retries(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_name_nothing_serves_routes_to_the_default_deployment(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router( + config, + json!({"default_deployment": { + "model_name": "dflt", "litellm_params": {"model": "openai/x"}, "model_info": {"id": "dd"} + }}), + host.clone(), + ); + let request = RouterRequest { + model: "other".to_owned(), + ..RouterRequest::default() + }; + let response = router + .call(Op::from("acompletion"), request.clone()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.model_group.as_str() + ), + ("dd", "other") + ); + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.params.model, "other"); +} + +#[rstest] +#[tokio::test] +async fn a_stale_member_map_naming_a_dropped_group_raises_on_every_pick(mut config: RouterConfig) { + config.routing_group_members = Some([("primary".to_owned(), "gone".to_owned())].into()); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let refused = router.select(&Op::from("acompletion"), &request()).await; + assert!(matches!(refused, Err(Error::MissingRoutingGroup { group }) if group == "gone")); + let other = RouterRequest { + model: "backup".to_owned(), + ..RouterRequest::default() + }; + assert!( + router + .select(&Op::from("acompletion"), &other) + .await + .is_ok() + ); +} + +#[rstest] +#[tokio::test] +async fn written_indexes_replace_the_ones_the_deployments_build(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let index: litellm_router_core::registry::IndexOverrides = serde_json::from_value(json!({ + "model_names": ["backup"], + "model_name": {"primary": [1], "backup": [2]}, + })) + .unwrap(); + router + .apply(litellm_router_core::ChangeSet { + index: Some(index), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + for _ in 0..4 { + let picked = router + .select(&Op::from("acompletion"), &request()) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } + router + .apply(litellm_router_core::ChangeSet { + index: Some(litellm_router_core::registry::IndexOverrides::default()), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + assert_eq!(router.registry().group("primary").unwrap().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_retry_policy_retries_a_pick_that_found_no_deployment(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let mut config = config; + config.retry_policy = serde_json::from_value(json!({"DefaultRetries": 2})).unwrap(); + let router = router(config, json!({"num_retries": 0}), host.clone()); + let request = RouterRequest { + model: "nothing".to_owned(), + ..RouterRequest::default() + }; + let refused = router.call(Op::from("acompletion"), request).await; + assert!(matches!(refused, Err(Error::UnknownModel { .. }))); + let events = host.events.lock(); + let retries = events + .iter() + .filter(|event| matches!(event, RouterEvent::Retry { .. })) + .count(); + assert_eq!(retries, 2); + assert!(events.iter().any(|event| matches!( + event, + RouterEvent::RetriesExhausted { + picked: true, + level: 0, + .. + } + ))); +} + +#[rstest] +#[tokio::test] +async fn a_default_deployment_python_cannot_copy_raises_for_a_name_nothing_serves( + config: RouterConfig, +) { + let router = router( + config, + json!({"default_deployment_raises": true}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + model: "other".to_owned(), + ..RouterRequest::default() + }; + let refused = router.select(&Op::from("acompletion"), &request).await; + assert!(matches!(refused, Err(Error::DefaultDeploymentRaises { model }) if model == "other")); + assert!( + router + .select(&Op::from("acompletion"), &self::request()) + .await + .is_ok() + ); +} + +#[rstest] +#[tokio::test] +async fn a_written_index_position_past_the_list_raises_for_that_name(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let index: litellm_router_core::registry::IndexOverrides = + serde_json::from_value(json!({"model_name": {"primary": [0, 9], "backup": [2]}})).unwrap(); + router + .apply(litellm_router_core::ChangeSet { + index: Some(index), + ..litellm_router_core::ChangeSet::default() + }) + .unwrap(); + let refused = router.select(&Op::from("acompletion"), &request()).await; + assert!(matches!(refused, Err(Error::IndexOutOfRange { model }) if model == "primary")); + let backup = RouterRequest { + model: "backup".to_owned(), + ..RouterRequest::default() + }; + assert!( + router + .select(&Op::from("acompletion"), &backup) + .await + .is_ok() + ); +} + +#[rstest] +#[case::no_list_takes_the_order_tier(json!(null), true, false, "p2")] +#[case::unmatched_list_takes_the_order_tier(json!([{"unrelated": ["context"]}]), true, false, "p2")] +#[case::unmatched_list_falls_through_to_regular(json!([{"unrelated": ["context"]}]), false, false, "b1")] +#[case::unmatched_list_leaves_the_client_side_list(json!([{"unrelated": ["context"]}]), false, true, "b1")] +#[case::matched_list_beats_the_order_tier(json!([{"primary": ["context"]}]), true, false, "c1")] +#[case::matched_star_beats_the_order_tier(json!([{"*": ["context"]}]), true, false, "c1")] +#[case::matched_list_beats_the_client_side_list(json!([{"primary": ["context"]}]), false, true, "c1")] +#[case::matched_list_beats_both(json!([{"primary": ["context"]}]), true, true, "c1")] +#[tokio::test] +async fn a_context_window_error_takes_its_own_chain_only_when_the_group_matches( + #[case] context_window_fallbacks: serde_json::Value, + #[case] tiered: bool, + #[case] client_side: bool, + #[case] expected: &str, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![context_window()])); + let router = router( + tiers(tiered), + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": context_window_fallbacks, + }), + host.clone(), + ); + let call = RouterRequest { + fallbacks: client_side.then(|| Some(serde_json::from_value(json!(["backup"])).unwrap())), + ..request() + }; + let response = router.call(Op::from("acompletion"), call).await.unwrap(); + assert_eq!(response.deployment_id, expected); +} + +#[rstest] +#[case::context_window_keeps_its_own_error( + context_window(), + FailureKind::ContextWindowExceeded, + false +)] +#[case::any_other_raises_the_last_hop(server_error(), FailureKind::ServiceUnavailable, true)] +#[tokio::test] +async fn a_failed_order_chain_raises_by_the_error_that_started_it( + #[case] first: AttemptError, + #[case] raised: FailureKind, + #[case] last_hop: bool, +) { + let host = Arc::new(ScriptedHost::scripted(vec![ + ("primary", vec![first, failure(FailureKind::RateLimit, 429)]), + ( + "backup", + vec![failure(FailureKind::ServiceUnavailable, 503)], + ), + ])); + let router = router( + tiers(true), + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": [{"unrelated": ["context"]}], + }), + host.clone(), + ); + let refused = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.kind == raised)); + assert_eq!(host.selected(), ["p1", "p2", "b1"]); + assert!( + host.events + .lock() + .contains(&RouterEvent::OrderChainFailed { level: 0, last_hop }) + ); +} + +#[rstest] +#[case::context_window_skips_it(context_window(), "backup")] +#[case::a_rate_limit_takes_it(failure(FailureKind::RateLimit, 429), "primary")] +#[tokio::test] +async fn weighted_failover_serves_every_failure_but_a_context_window_error( + #[case] first: AttemptError, + #[case] served_by: &str, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![first])); + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}}, + ], + "enable_weighted_failover": true, + })) + .unwrap(); + let router = router( + config, + json!({ + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": [{"unrelated": ["backup"]}], + }), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.model_group, served_by); +} + +#[rstest] +#[tokio::test] +async fn a_failed_agentic_followup_is_not_retried_but_still_falls_back(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router( + config, + json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); + assert_eq!(host.retries(), 0); + assert_eq!(host.selected().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_failed_agentic_followup_skips_the_retry_policy_lookup(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let call = RouterRequest { + retry_policy_raises: vec!["primary".to_owned()], + ..request() + }; + let refused = router.call(Op::from("acompletion"), call).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert!( + !host + .events + .lock() + .iter() + .any(|event| matches!(event, RouterEvent::RetryPolicyRaised { .. })) + ); +} + +#[rstest] +#[tokio::test] +async fn a_retry_that_fails_its_agentic_followup_ends_the_retries(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![server_error(), agentic_followup()], + )); + let router = router(config, json!({"num_retries": 3}), host.clone()); + let refused = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (1, 2)); +} + +fn stream_origin() -> StreamOrigin { + StreamOrigin { + group: "primary".to_owned(), + ..StreamOrigin::default() + } +} + +fn stream_request() -> RouterRequest { + RouterRequest { + num_retries: Some(3), + ..request() + } +} + +#[rstest] +#[tokio::test] +async fn a_stream_that_failed_its_agentic_followup_is_not_retried(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let refused = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + agentic_followup(), + ) + .await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (0, 0)); +} + +#[rstest] +#[tokio::test] +async fn a_stream_retry_that_fails_its_agentic_followup_ends_the_retries(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![agentic_followup()])); + let router = router(config, json!({}), host.clone()); + let refused = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + server_error(), + ) + .await; + assert!(matches!(refused, Err(Error::Attempt(failed)) if failed.agentic_followup)); + assert_eq!((host.retries(), host.selected().len()), (1, 1)); +} + +#[rstest] +#[tokio::test] +async fn a_stream_retry_is_reported_before_its_attempt(config: RouterConfig) { + let host = Arc::new(ScriptedHost::default()); + let router = router(config, json!({}), host.clone()); + let response = router + .retry_stream( + Op::from("acompletion"), + stream_request(), + stream_origin(), + server_error(), + ) + .await + .unwrap(); + assert_eq!((response.attempted_retries, host.retries()), (1, 1)); +} + +fn fallback_list(value: serde_json::Value) -> Fallbacks { + serde_json::from_value(value).unwrap() +} + +#[rstest] +#[case::names(json!(["backup"]))] +#[case::keyed(json!([{"primary": ["backup"]}]))] +#[case::star(json!([{"*": ["backup"]}]))] +fn a_request_fallback_list_can_take_over_a_stream( + config: RouterConfig, + #[case] list: serde_json::Value, +) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + fallbacks: Some(Some(fallback_list(list))), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(policy.can_fall_back); + assert!(policy.refusal_fallback); +} + +#[rstest] +fn a_request_content_policy_list_can_take_over_a_stream(config: RouterConfig) { + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + content_policy_fallbacks: Some(Some(fallback_list(json!([{"primary": ["backup"]}])))), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(policy.can_fall_back); + assert!(policy.refusal_fallback); +} + +#[rstest] +#[case::cleared(None)] +#[case::another_group(Some(json!([{"other": ["backup"]}])))] +fn a_request_fallback_list_replaces_the_router_list_for_a_stream( + config: RouterConfig, + #[case] list: Option, +) { + let router = router( + config, + json!({"fallbacks": [{"primary": ["backup"]}], "default_fallbacks": ["backup"]}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + fallbacks: Some(list.map(fallback_list)), + ..request() + }; + let policy = router.stream_policy(&request, &stream_origin(), None); + assert!(!policy.can_fall_back); +} + +#[rstest] +#[case::readable(vec![], 5, true)] +#[case::malformed(vec!["primary".to_owned()], 1, false)] +fn a_group_policy_that_does_not_read_governs_no_stream_retry( + mut config: RouterConfig, + #[case] retry_policy_raises: Vec, + #[case] budget: u32, + #[case] applies: bool, +) { + config.retry_policy = serde_json::from_value(json!({"InternalServerErrorRetries": 5})).unwrap(); + let router = router( + config, + json!({"num_retries": 1}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + retry_policy_raises, + ..request() + }; + let policy = router.stream_policy( + &request, + &stream_origin(), + Some(FailureKind::InternalServer), + ); + assert_eq!(policy.retry_budget, Some((budget, applies))); +} + +#[rstest] +#[tokio::test] +async fn a_level_reports_its_fallback_handling_before_any_hop(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![context_window()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let events = host.events.lock(); + let falling_back = RouterEvent::FallingBack { + level: 0, + model: "primary".to_owned(), + }; + let reported = events.iter().position(|e| *e == falling_back).unwrap(); + let hopped = events + .iter() + .position(|e| matches!(e, RouterEvent::Fallback { .. })) + .unwrap(); + assert!(reported < hopped); +} + +#[rstest] +#[tokio::test] +async fn the_order_path_reports_no_fallback_handling() { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(tiers(true), json!({"num_retries": 0}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert!( + !host + .events + .lock() + .iter() + .any(|e| matches!(e, RouterEvent::FallingBack { .. })) + ); +} + +fn fallbacks_failed(host: &ScriptedHost) -> Vec<(u32, bool, bool)> { + host.events + .lock() + .iter() + .filter_map(|event| match event { + RouterEvent::FallbacksFailed { + level, + chain, + last_hop, + } => Some((*level, *chain, *last_hop)), + _ => None, + }) + .collect() +} + +#[rstest] +#[tokio::test] +async fn a_chain_that_fails_reports_every_level_innermost_first(config: RouterConfig) { + let host = Arc::new(ScriptedHost::scripted(vec![ + ("primary", vec![server_error()]), + ("backup", vec![server_error()]), + ])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + assert_eq!( + fallbacks_failed(&host), + vec![(1, true, false), (0, true, true)] + ); +} + +#[rstest] +#[tokio::test] +async fn a_chain_past_max_fallbacks_reports_the_levels_own_failure(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "max_fallbacks": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap_err(); + assert_eq!(fallbacks_failed(&host), vec![(0, true, false)]); +} + +#[rstest] +#[tokio::test] +async fn a_served_fallback_and_a_disabled_walk_report_no_failure(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let dynamic = json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}); + let served = router(config.clone(), dynamic.clone(), host.clone()); + served + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(fallbacks_failed(&host), vec![]); + + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let disabled = router(config, dynamic, host.clone()); + let request = RouterRequest { + disable_fallbacks: true, + ..request() + }; + disabled + .call(Op::from("acompletion"), request) + .await + .unwrap_err(); + assert_eq!(fallbacks_failed(&host), vec![]); +} + +#[tokio::test] +async fn the_order_path_reports_no_failed_fallbacks() { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(tiers(true), json!({"num_retries": 0}), host.clone()); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(fallbacks_failed(&host), vec![]); +} + +fn refusing_host(group: &str) -> Arc { + Arc::new(ScriptedHost { + refuses_dict_targets: true, + ..ScriptedHost::failing(group, vec![server_error()]) + }) +} + +#[rstest] +#[case::dict_target_first(json!([{"model": "backup", "api_key": "x"}, "backup"]))] +#[case::dict_target_last(json!(["backup", {"model": "backup", "api_key": "x"}]))] +#[tokio::test] +async fn a_refused_chain_tries_none_of_its_targets( + config: RouterConfig, + #[case] chain: serde_json::Value, +) { + let host = refusing_host("primary"); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": chain}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("InternalServerError"))); + assert_eq!(host.selected().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_chain_of_names_is_never_refused(config: RouterConfig) { + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + refusing_host("primary"), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +#[rstest] +#[tokio::test] +async fn a_refused_chain_on_the_order_path_raises_the_refusal() { + let host = refusing_host("primary"); + let router = router( + tiers(true), + json!({"num_retries": 0, "fallbacks": [{"primary": [{"model": "backup", "api_key": "x"}]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("ValueError"))); + assert_eq!(host.selected(), ["p1"]); +} + +#[rstest] +#[tokio::test] +async fn a_refused_chain_fails_only_its_own_level(mut config: RouterConfig) { + let host = Arc::new(ScriptedHost { + refuses_dict_targets: true, + ..ScriptedHost::scripted(vec![ + ("primary", vec![server_error()]), + ("backup", vec![server_error()]), + ]) + }); + let dynamic = json!({"num_retries": 0, "fallbacks": [ + {"primary": ["backup", "third"]}, + {"backup": [{"model": "third", "api_key": "x"}]}, + ]}); + config.model_list.push( + serde_json::from_value(json!({ + "model_name": "third", "litellm_params": {"model": "openai/t"}, "model_info": {"id": "t1"}, + })) + .unwrap(), + ); + let response = router(config, dynamic, host.clone()) + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!(response.deployment_id, "t1"); +} + +fn shared_public_name() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "backup", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "b1"}}, + {"model_name": "t1-pub", "litellm_params": {"model": "openai/m"}, + "model_info": {"id": "p1", "team_id": "T1", "team_public_model_name": "pub"}}, + {"model_name": "t2-pub", "litellm_params": {"model": "openai/m"}, + "model_info": {"id": "p2", "team_id": "T2", "team_public_model_name": "pub"}}, + ] + })) + .unwrap() +} + +fn proxy_admin_of_pub() -> RouterRequest { + RouterRequest { + model: "pub".to_owned(), + proxy_admin: true, + admin_model: Some("pub".to_owned()), + ..RouterRequest::default() + } +} + +#[rstest] +#[tokio::test] +async fn a_name_several_teams_share_falls_back() { + let router = router( + shared_public_name(), + json!({"fallbacks": [{"pub": ["backup"]}]}), + Arc::new(ScriptedHost::default()), + ); + let response = router + .call(Op::from("acompletion"), proxy_admin_of_pub()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 1) + ); +} + +#[rstest] +#[tokio::test] +async fn a_name_several_teams_share_raises_its_refusal_when_the_fallback_fails() { + let router = router( + shared_public_name(), + json!({"fallbacks": [{"pub": ["backup"]}]}), + Arc::new(ScriptedHost::failing("backup", vec![server_error()])), + ); + let raised = router + .call(Op::from("acompletion"), proxy_admin_of_pub()) + .await; + assert!(matches!(raised, Err(Error::MultipleTeams { model }) if model == "pub")); +} + +/// A group whose second order level belongs to another team. +fn team_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/p1", "order": 1}, + "model_info": {"id": "p1", "team_id": "t1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/p2", "order": 2}, + "model_info": {"id": "p2", "team_id": "t2"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::listed_for_the_team(Some("t1"), false)] +#[case::listed_for_no_team(None, true)] +#[tokio::test] +async fn a_failed_level_lists_its_order_levels_for_the_metadata_team( + #[case] listing_team_id: Option<&str>, + #[case] walks_the_order_chain: bool, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(team_tiers(), json!({"num_retries": 0}), host.clone()); + let request = RouterRequest { + team_id: Some("t1".to_owned()), + listing_team_id: listing_team_id.map(str::to_owned), + ..request() + }; + let raised = router.call(Op::from("acompletion"), request).await; + assert_eq!( + matches!(raised, Err(Error::NoHealthyDeployments { .. })), + walks_the_order_chain + ); + assert_eq!(host.selected(), ["p1"]); +} + +fn tagged_request() -> RouterRequest { + RouterRequest { + filter_tags: true, + ..request() + } +} + +#[rstest] +#[tokio::test] +async fn a_failed_pick_retries_like_a_failed_attempt(config: RouterConfig) { + let host = Arc::new(ScriptedHost { + pick_failures: Mutex::new(VecDeque::from([failure(FailureKind::RateLimit, 429)])), + ..ScriptedHost::default() + }); + let response = router(config, json!({"num_retries": 1}), host.clone()) + .call(Op::from("acompletion"), tagged_request()) + .await + .unwrap(); + assert_eq!( + (response.model_group.as_str(), response.attempted_retries), + ("primary", 1) + ); +} + +#[rstest] +#[tokio::test] +async fn a_failed_pick_falls_back_with_no_deployment_to_blame(config: RouterConfig) { + let host = Arc::new(ScriptedHost { + pick_failures: Mutex::new(VecDeque::from([failure(FailureKind::RateLimit, 429)])), + ..ScriptedHost::default() + }); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), tagged_request()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 1) + ); + assert_eq!(host.selected(), ["b1"]); +} + +#[rstest] +#[tokio::test] +async fn a_fallback_check_that_raises_ends_the_chain_with_the_levels_own_error( + config: RouterConfig, +) { + let host = Arc::new(ScriptedHost { + fallback_check_raises: Some("TypeError"), + ..ScriptedHost::failing("primary", vec![failure(FailureKind::RateLimit, 429)]) + }); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.kind == FailureKind::RateLimit)); + assert_eq!(host.selected().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_fallback_check_that_raises_on_the_order_path_raises_its_own_error() { + let host = Arc::new(ScriptedHost { + fallback_check_raises: Some("TypeError"), + ..ScriptedHost::scripted(vec![("primary", vec![server_error(), server_error()])]) + }); + let router = router( + tiers(true), + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + let raised = router.call(Op::from("acompletion"), request()).await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a("TypeError"))); +} + +#[rstest] +#[case::an_int(json!(5))] +#[case::none(json!(null))] +#[tokio::test] +async fn a_dict_target_with_a_non_string_model_is_a_hop_that_attempts_nothing( + config: RouterConfig, + #[case] model: serde_json::Value, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": [{"model": model}, "backup"]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + ( + response.deployment_id.as_str(), + response.attempted_fallbacks + ), + ("b1", 2) + ); + assert_eq!(host.selected().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_dict_target_without_a_model_serves_the_group_that_failed_again(config: RouterConfig) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router( + config, + json!({"num_retries": 0, "fallbacks": [{"primary": [{"temperature": 0.5}]}]}), + host.clone(), + ); + let response = router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + assert_eq!( + (response.model_group.as_str(), response.attempted_fallbacks), + ("primary", 1) + ); +} + +fn weighted_failover() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + ], + "enable_weighted_failover": true, + })) + .unwrap() +} + +#[rstest] +#[case::a_bug_propagates(AttemptError::host_raised(&["TypeError", "Exception"]), "TypeError")] +#[case::a_provider_error_defers(failure(FailureKind::RateLimit, 429), "InternalServerError")] +#[case::no_deployment_defers( + AttemptError::host_raised(&["RouterRateLimitError", "ValueError"]), + "InternalServerError" +)] +#[tokio::test] +async fn weighted_failover_swallows_only_api_and_rate_limit_errors( + #[case] hop_failure: AttemptError, + #[case] raised_class: &str, +) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![server_error(), hop_failure], + )); + let raised = router(weighted_failover(), json!({"num_retries": 0}), host.clone()) + .call(Op::from("acompletion"), request()) + .await; + assert!(matches!(raised, Err(Error::Attempt(failed)) if failed.is_a(raised_class))); + assert_eq!(host.selected().len(), 2); +} + +fn wildcard_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "order": 1}, "model_info": {"id": "w1"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "order": 2}, "model_info": {"id": "w2"}}, + ] + })) + .unwrap() +} + +fn alias_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "gg", "litellm_params": {"model": "openai/a", "order": 1}, "model_info": {"id": "w1"}}, + {"model_name": "gg", "litellm_params": {"model": "openai/b", "order": 2}, "model_info": {"id": "w2"}}, + ] + })) + .unwrap() +} + +fn team_public_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "t1a", "litellm_params": {"model": "openai/a", "order": 1}, + "model_info": {"id": "w1", "team_id": "T1", "team_public_model_name": "tp"}}, + {"model_name": "t1b", "litellm_params": {"model": "openai/b", "order": 2}, + "model_info": {"id": "w2", "team_id": "T1", "team_public_model_name": "tp"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::a_wildcard(wildcard_tiers(), json!({"num_retries": 0}), "openai/gpt-4o", None)] +#[case::an_alias( + alias_tiers(), + json!({"num_retries": 0, "model_group_alias": {"al": "gg"}}), + "al", + None +)] +#[case::a_team_public_name(team_public_tiers(), json!({"num_retries": 0}), "tp", Some("T1"))] +#[tokio::test] +async fn order_levels_walk_through_any_name_the_model_list_lists( + #[case] config: RouterConfig, + #[case] dynamic: serde_json::Value, + #[case] model: &str, + #[case] team: Option<&str>, +) { + let host = Arc::new(ScriptedHost::default()); + *host.first_failure.lock() = Some(failure(FailureKind::RateLimit, 429)); + let request = RouterRequest { + model: model.to_owned(), + team_id: team.map(str::to_owned), + listing_team_id: team.map(str::to_owned), + ..RouterRequest::default() + }; + let response = router(config, dynamic, host.clone()) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "w2"); +} + +#[rstest] +#[tokio::test] +async fn a_team_wildcard_public_name_routes_for_its_team() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "t1-w", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw1", "team_id": "T1", "team_public_model_name": "tw-*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "gp"}}, + ] + })) + .unwrap(); + let request = RouterRequest { + model: "tw-gpt-4o".to_owned(), + team_id: Some("T1".to_owned()), + ..RouterRequest::default() + }; + let response = router(config, json!({}), Arc::new(ScriptedHost::default())) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "tw1"); +} + +#[rstest] +#[tokio::test] +async fn a_paused_deployment_named_by_id_falls_back() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, + "model_info": {"id": "a1", "blocked": true}}, + {"model_name": "backup", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "b1"}}, + ] + })) + .unwrap(); + let request = RouterRequest { + model: "a1".to_owned(), + ..RouterRequest::default() + }; + let response = router( + config, + json!({"fallbacks": [{"a1": ["backup"]}]}), + Arc::new(ScriptedHost::default()), + ) + .call(Op::from("acompletion"), request) + .await + .unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +/// `primary` with `a1`'s `model_info` as given, and an unremarkable `a2`. +fn with_first_info(info: serde_json::Value) -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": info}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[tokio::test] +async fn a_web_search_request_skips_the_deployments_that_declare_no_support() { + let config = with_first_info(json!({"id": "a1", "supports_web_search": false})); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + web_search: true, + ..request() + }; + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn an_endpoint_a_deployment_refuses_routes_to_the_others() { + let config = with_first_info(json!({ + "id": "a1", + "enforce_supported_endpoints": true, + "supported_endpoints": ["/v1/embeddings"], + })); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + endpoint: Some("/v1/chat/completions".to_owned()), + ..request() + }; + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn an_endpoint_every_deployment_refuses_fails_the_pick() { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [{ + "model_name": "primary", + "litellm_params": {"model": "openai/a"}, + "model_info": { + "id": "a1", + "enforce_supported_endpoints": true, + "supported_endpoints": ["/v1/embeddings"], + }, + }] + })) + .unwrap(); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let request = RouterRequest { + endpoint: Some("/v1/chat/completions".to_owned()), + ..request() + }; + let refused = router + .call(Op::from("acompletion"), request) + .await + .unwrap_err(); + assert!(matches!( + refused, + Error::EndpointNotSupported { model, message } + if model == "primary" && message + == "/v1/chat/completions is not supported by model=primary. \ + Supported endpoints: /v1/embeddings" + )); +} + +#[rstest] +#[tokio::test] +async fn an_order_in_model_info_narrows_the_pick() { + let mut config = with_first_info(json!({"id": "a1", "order": 2})); + config.model_list[1] + .model_info + .insert("order".to_owned(), json!("1")); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + for _ in 0..16 { + let picked = router + .select(&Op::from("acompletion"), &request()) + .await + .unwrap(); + assert_eq!(picked.id, "a2"); + } +} + +#[rstest] +#[tokio::test] +async fn a_fallback_hop_drops_the_order_the_request_named() { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![failure(FailureKind::InternalServer, 500)], + )); + let router = router( + tiers(false), + json!({"num_retries": 0, "fallbacks": [{"primary": ["backup"]}]}), + host, + ); + let request = RouterRequest { + target_order: Some(1), + ..request() + }; + let response = router.call(Op::from("acompletion"), request).await.unwrap(); + assert_eq!(response.deployment_id, "b1"); +} + +#[rstest] +#[tokio::test] +async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config: RouterConfig) { + let router = router( + config, + json!({"model_alias_map": {"al": "primary"}}), + Arc::new(ScriptedHost::default()), + ); + let request = RouterRequest { + model: "al".to_owned(), + pass_through: Some(PassThrough::Listed), + ..RouterRequest::default() + }; + let refused = router + .select(&Op::from("acompletion"), &request) + .await + .unwrap_err(); + assert!(matches!( + refused, + Error::NoPassThroughDeployments { model } if model == "primary" + )); +} diff --git a/litellm-rust/crates/secret-value/src/secret_value.rs b/litellm-rust/crates/secret-value/src/secret_value.rs index 27e4227080d..33c559b80ec 100644 --- a/litellm-rust/crates/secret-value/src/secret_value.rs +++ b/litellm-rust/crates/secret-value/src/secret_value.rs @@ -16,6 +16,18 @@ impl SecretValue { } } +impl From for SecretValue { + fn from(value: String) -> Self { + Self::new(value) + } +} + +impl From<&str> for SecretValue { + fn from(value: &str) -> Self { + Self::new(value) + } +} + impl PartialEq for SecretValue { fn eq(&self, other: &Self) -> bool { subtle::ConstantTimeEq::ct_eq(self.0.as_bytes(), other.0.as_bytes()).into() diff --git a/litellm-rust/crates/secrets-aws/src/auth.rs b/litellm-rust/crates/secrets-aws/src/auth.rs index 66f4f51bda9..6bec8ad96a1 100644 --- a/litellm-rust/crates/secrets-aws/src/auth.rs +++ b/litellm-rust/crates/secrets-aws/src/auth.rs @@ -39,30 +39,15 @@ impl Credentials { Self { auth, config: AwsAuthConfig { - access_key_id: context - .access_key_id - .as_ref() - .map(|value| value.expose().to_owned()), - secret_access_key: context - .secret_access_key - .as_ref() - .map(|value| value.expose().to_owned()), - session_token: context - .session_token - .as_ref() - .map(|value| value.expose().to_owned()), + access_key_id: context.access_key_id.as_ref().cloned(), + secret_access_key: context.secret_access_key.as_ref().cloned(), + session_token: context.session_token.as_ref().cloned(), region_name: region(settings, environment.as_ref()).ok(), role_name: settings.aws_role_name.clone(), session_name: settings.aws_session_name.clone(), - external_id: settings - .aws_external_id - .as_ref() - .map(|v| v.expose().to_owned()), + external_id: settings.aws_external_id.as_ref().cloned(), profile_name: settings.aws_profile_name.clone(), - web_identity_token: settings - .aws_web_identity_token - .as_ref() - .map(|v| v.expose().to_owned()), + web_identity_token: settings.aws_web_identity_token.as_ref().cloned(), sts_endpoint: settings.aws_sts_endpoint.clone(), }, environment, diff --git a/litellm/__init__.py b/litellm/__init__.py index baee4e9f7ba..8e5dd68a186 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -8,7 +8,7 @@ _apply_process_defaults() import threading import os -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import ( Any, Callable, @@ -688,6 +688,7 @@ darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads streamlake_models: Final[Set[str]] = set() +wally_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() morph_models: Set = set() @@ -984,6 +985,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje kyma_models.add(key) elif value.get("litellm_provider") == "streamlake": streamlake_models.add(key) + elif value.get("litellm_provider") == "wally": + wally_models.add(key) elif value.get("litellm_provider") == "zerogpu": zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": @@ -1071,7 +1074,6 @@ _populate_provider_model_sets(model_cost) # Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting azure_llms = { "gpt-35-turbo": "azure/gpt-35-turbo", - "gpt-35-turbo-16k": "azure/gpt-35-turbo-16k", "gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct", "azure/gpt-41": "gpt-4.1", "azure/gpt-41-mini": "gpt-4.1-mini", @@ -1166,6 +1168,7 @@ model_list = list( | openinfer_models | kyma_models | streamlake_models + | wally_models | zerogpu_models | v0_models | morph_models @@ -1299,6 +1302,7 @@ def _build_models_by_provider() -> dict: "openinfer": openinfer_models, "kyma": kyma_models, "streamlake": streamlake_models, + "wally": wally_models, "zerogpu": zerogpu_models, "v0": v0_models, "morph": morph_models, @@ -1562,7 +1566,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, + acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret, arealtime_calls, + arealtime_translation_calls as arealtime_translation_calls, ) from .responses.main import _aresponses_websocket @@ -1754,6 +1760,9 @@ if TYPE_CHECKING: from .llms.typesafe.decisions.transformation import ( TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, ) + from .llms.parallel_ai.decisions.transformation import ( + ParallelAIDecisionsConfig as ParallelAIDecisionsConfig, + ) from .llms.openrouter.decisions.transformation import ( OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, ) @@ -1787,6 +1796,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.empiriolabs.decisions.transformation import ( + EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig, + ) from .llms.eu_jev.decisions.transformation import ( EuJevDecisionsConfig as EuJevDecisionsConfig, ) @@ -1808,6 +1820,9 @@ if TYPE_CHECKING: from .llms.vertex_ai.rerank.transformation import ( VertexAIRerankConfig as VertexAIRerankConfig, ) + from .llms.together_ai.chat.transformation import ( + TogetherAIChatConfig as TogetherAIChatConfig, + ) from .llms.fireworks_ai.rerank.transformation import ( FireworksAIRerankConfig as FireworksAIRerankConfig, ) @@ -1853,9 +1868,6 @@ if TYPE_CHECKING: BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig, ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig - from .llms.together_ai.chat.transformation import ( - TogetherAIChatConfig as TogetherAIChatConfig, - ) from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig as VertexGeminiConfig, @@ -2014,6 +2026,9 @@ if TYPE_CHECKING: from .llms.xai.responses.transformation import ( XAIResponsesAPIConfig as XAIResponsesAPIConfig, ) + from .llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig as VertexAIInteractionsConfig, + ) from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, ) @@ -2041,9 +2056,6 @@ if TYPE_CHECKING: from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) - from .llms.vertex_ai.interactions.transformation import ( - VertexAIInteractionsConfig as VertexAIInteractionsConfig, - ) from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 40d3377bf66..f2f7869876d 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]: internal_parent_request_id.reset(parent_token) +@contextmanager +def billed_separately() -> Generator[None]: + """A call that writes its own spend row even inside a sub-call whose parent bills the model response.""" + token: Final = is_internal_call.set(False) + parent_token: Final = internal_parent_request_id.set(None) + try: + yield + finally: + is_internal_call.reset(token) + internal_parent_request_id.reset(parent_token) + + @contextmanager def post_response_phase() -> Generator[None]: """Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns.""" diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index c9949613303..bf903e23bc0 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -163,6 +163,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMRerankConfig", "PerplexityDecisionsConfig", "TypeSafeDecisionsConfig", + "ParallelAIDecisionsConfig", "NaceDecisionsConfig", "OpenRouterDecisionsConfig", "FireworksAIDecisionsConfig", @@ -175,6 +176,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "EmpirioLabsDecisionsConfig", "EuJevDecisionsConfig", "OpenAIDecisionsConfig", "AzureDecisionsConfig", @@ -740,6 +742,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"), "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), @@ -757,6 +760,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"), "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), diff --git a/litellm/_v2/AGENTS.md b/litellm/_v2/AGENTS.md deleted file mode 100644 index 55998814a7f..00000000000 --- a/litellm/_v2/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -Everything here is experimental and should not be documented - -Use this directory to explore alternative APIs where the Rust migration makes backward compatibility difficult. The gateway can use them for performance, but keep them behind the v2 flag to avoid breaking SDK users diff --git a/litellm/_v2/__init__.py b/litellm/_v2/__init__.py deleted file mode 100644 index 79156cf0f80..00000000000 --- a/litellm/_v2/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from litellm._v2.cache import Cache - -__all__ = ("Cache",) diff --git a/litellm/_v2/cache/AGENTS.md b/litellm/_v2/cache/AGENTS.md deleted file mode 100644 index 308419b7253..00000000000 --- a/litellm/_v2/cache/AGENTS.md +++ /dev/null @@ -1,11 +0,0 @@ -# Python v2 cache - -Keep `litellm._v2.cache.Cache` import-compatible when reorganizing this package. This package owns Python cache factories and the adapter between the existing `BaseCache` interface and `NativeCacheHandle` - -Construct native handles at this boundary and inject the adapter through the existing cache facade's `_backend` parameter. Keep the facade's `type`, namespace, and TTL consistent with the configured native backend - -Keep storage implementation in the Rust storage crates and response-cache policy in `litellm-cache-response` and core. Do not duplicate cache-key generation, freshness rules, response encoding, or inference orchestration here - -Preserve synchronous and asynchronous cache operations, including TTL forwarding and lifecycle methods. Validate Python values before passing them to typed native interfaces. Keep native extension imports lazy so importing the package does not require loading the extension - -Extend the existing v2 cache tests in `tests/test_litellm_rust/test_v2.py` for behavioral changes, following that directory's `AGENTS.md`. Test observable cache behavior rather than package layout or implementation structure diff --git a/litellm/_v2/cache/__init__.py b/litellm/_v2/cache/__init__.py deleted file mode 100644 index 5a3860b7a6e..00000000000 --- a/litellm/_v2/cache/__init__.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING, Final - -from pydantic import TypeAdapter - -from litellm.caching.base_cache import BaseCache -from litellm.caching.caching import Cache as CacheFacade -from litellm.types.caching import LiteLLMCacheType - -if TYPE_CHECKING: - from litellm.rust_bridge._native import NativeCacheHandle - -_DURATION: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) - - -class NativeBackend(BaseCache): - def __init__(self, handle: NativeCacheHandle) -> None: - self.native_handle = handle - - def get_cache(self, key: str, **kwargs: object) -> object: - return self.native_handle.get(key) - - async def async_get_cache(self, key: str, **kwargs: object) -> object: - return await self.native_handle.async_get(key) - - def set_cache(self, key: str, value: object, **kwargs: object) -> None: - self.native_handle.set(key, value, ttl=_DURATION.validate_python(kwargs.get("ttl"))) - - async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: - await self.native_handle.async_set(key, value, ttl=_DURATION.validate_python(kwargs.get("ttl"))) - - async def async_set_cache_pipeline(self, cache_list: Sequence[tuple[str, object]], **kwargs: object) -> None: - await self.native_handle.async_set_many(cache_list, ttl=_DURATION.validate_python(kwargs.get("ttl"))) - - async def batch_cache_write(self, key: str, value: object, **kwargs: object) -> None: - await self.async_set_cache(key, value, **kwargs) - - def flush_cache(self) -> None: - self.native_handle.flush() - - async def async_flush_cache(self) -> None: - await self.native_handle.async_flush() - - async def ping(self) -> bool: - return await self.native_handle.ping() - - async def disconnect(self) -> None: - await self.native_handle.disconnect() - - async def delete_cache_keys(self, keys: Sequence[str]) -> None: - await self.native_handle.delete(keys) - - async def test_connection(self) -> dict[str, str]: - return {"status": "success" if await self.ping() else "failed"} - - -class Cache: - @staticmethod - def memory(*, ttl: float = 600, capacity: int = 200, max_entry_bytes: int = 4194304) -> CacheFacade: - from litellm.rust_bridge._native import NativeCacheHandle - - handle: Final = NativeCacheHandle.memory(ttl=ttl, capacity=capacity, max_entry_bytes=max_entry_bytes) - return CacheFacade(type=LiteLLMCacheType.LOCAL, ttl=ttl, _backend=NativeBackend(handle)) - - @staticmethod - def redis(url: str, *, namespace: str, ttl: float = 600, max_entry_bytes: int = 4194304) -> CacheFacade: - from litellm.rust_bridge._native import NativeCacheHandle - - handle: Final = NativeCacheHandle.redis(url, namespace=namespace, ttl=ttl, max_entry_bytes=max_entry_bytes) - return CacheFacade(type=LiteLLMCacheType.REDIS, namespace=namespace, ttl=ttl, _backend=NativeBackend(handle)) diff --git a/litellm/accounting/__init__.py b/litellm/accounting/__init__.py index d4e56f61279..e69de29bb2d 100644 --- a/litellm/accounting/__init__.py +++ b/litellm/accounting/__init__.py @@ -1,62 +0,0 @@ -"""Public accounting contracts. Importing these values does not enable accounting.""" - -from litellm.accounting.measurements import measurements_for -from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter -from litellm.accounting.types import ( - AdmissionDecision, - AggregateQuery, - AggregateRow, - CostEvent, - DeliveryState, - DeliveryTarget, - Dimension, - DimensionFilter, - DimensionName, - EventKind, - Granularity, - InboundUsageEvent, - Measurement, - Metric, - MetricName, - Multiplier, - NamedCost, - OutboundUsageEvent, - Outcome, - PricedQuantity, - PricingSnapshot, - Rate, - Unpriced, - validate_aggregate_rows, -) - -__all__ = [ - "AdmissionDecision", - "AggregateQuery", - "AggregateRow", - "BudgetEnforcer", - "CostAggregator", - "CostCompiler", - "CostEvent", - "DeliveryState", - "DeliveryTarget", - "Dimension", - "DimensionFilter", - "DimensionName", - "EventKind", - "Granularity", - "InboundUsageEvent", - "Measurement", - "Metric", - "MetricName", - "Multiplier", - "NamedCost", - "OutboundUsageEvent", - "Outcome", - "PricedQuantity", - "PricingSnapshot", - "Rate", - "Unpriced", - "UsageMeter", - "measurements_for", - "validate_aggregate_rows", -] diff --git a/litellm/accounting/application.py b/litellm/accounting/application.py new file mode 100644 index 00000000000..1e5f23e2997 --- /dev/null +++ b/litellm/accounting/application.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING, Final + +from litellm.accounting.delivery_projection import entity_transactions, project_deliveries +from litellm.accounting.projection import Contribution +from litellm.proxy.db.budget_window_spend_writer import commit_window_spend_updates +from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction +from litellm.proxy.db.model_usage_rollup import queue_model_usage_upsert +from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import PrismaClient, ProxyLogging + + +async def apply_deliveries( + transaction: LedgerTransaction, + deliveries: tuple[DeliveryProjection, ...], + *, + prisma_client: "PrismaClient", + writer: "DBSpendUpdateWriter", + proxy_logging_obj: "ProxyLogging", +) -> tuple[Contribution, ...]: + from litellm.proxy.guardrails.usage_tracking import UsageUnitIncrement, UsageUnitKey, upsert_usage_unit_row + + contributions: Final = project_deliveries(deliveries) + await writer.apply_accounting_entities( + prisma_client, proxy_logging_obj, transaction, entity_transactions(contributions) + ) + for table in sorted(DAILY_SPEND_TABLES.values(), key=lambda item: item.name): + rows = tuple( + {**dict(item.key), **dict(item.values), **dict(item.increments)} + for item in contributions + if item.table == table.name + ) + merged = merge_by_conflict_key(table, rows) + for offset in range(0, len(merged), 100): + sql, params = build_bulk_upsert(table, merged[offset : offset + 100]) + async with db_span("upsert_daily_spend", table.name): + await transaction.execute_raw(sql, *params) + async with transaction.batch_() as batcher: + for item in contributions: + if item.table == "LiteLLM_DailyModelUsage": + queue_model_usage_upsert(batcher, dict(item.key), dict(item.increments)) + for item in contributions: + if item.table == "LiteLLM_DailyGuardrailUsageUnits": + key = dict(item.key) + counters = dict(item.increments) + await upsert_usage_unit_row( + prisma_client, + UsageUnitKey(key["guardrail_id"], key["date"], key["team_id"], key["api_key"], key["usage_unit"]), + UsageUnitIncrement(int(counters["units"]), float(counters["cost"]), int(counters["untracked_units"])), + outer_transaction=transaction, + ) + windows: Final = tuple(_window(item) for item in contributions if item.table == "LiteLLM_BudgetWindowSpend") + await commit_window_spend_updates(prisma_client, windows, outer_transaction=transaction) + return contributions + + +def _window(item: Contribution) -> WindowSpendTransaction: + key: Final = dict(item.key) + values: Final = dict(item.values) + return WindowSpendTransaction( + entity_type=key["entity_type"], + entity_id=key["entity_id"], + window_duration=key["window_duration"], + window_start=values["window_start"] or "", + spend=float(dict(item.increments)["spend"]), + started_at=values["started_at"], + ) diff --git a/litellm/accounting/delivery_projection.py b/litellm/accounting/delivery_projection.py new file mode 100644 index 00000000000..da63542d5b8 --- /dev/null +++ b/litellm/accounting/delivery_projection.py @@ -0,0 +1,78 @@ +from collections.abc import Iterable +from itertools import chain, groupby +from typing import TYPE_CHECKING, Final +from urllib.parse import quote + +from litellm.accounting.projection import Contribution, project +from litellm.repositories.accounting_delivery_repository import DeliveryProjection + + +def _destination(contribution: Contribution) -> tuple[str, tuple[tuple[str, str], ...], str]: + return ( + contribution.table, + contribution.key, + (dict(contribution.values).get("window_start") or "") + if contribution.table == "LiteLLM_BudgetWindowSpend" + else "", + ) + + +def _sum(contributions: tuple[Contribution, ...]) -> Contribution: + first: Final = contributions[0] + increments: Final = tuple( + (column, sum(dict(item.increments).get(column, 0) for item in contributions)) + for column in dict.fromkeys( + column for column, _ in chain.from_iterable(item.increments for item in contributions) + ) + ) + starts: Final = tuple( + value + for column, value in chain.from_iterable(item.values for item in contributions) + if column == "started_at" and value + ) + values: Final = tuple( + (column, min(starts) if column == "started_at" and starts else value) for column, value in first.values + ) + return Contribution(first.table, first.key, increments, values) + + +def project_deliveries(deliveries: Iterable[DeliveryProjection]) -> tuple[Contribution, ...]: + ordered: Final = sorted(chain.from_iterable(project(delivery.row) for delivery in deliveries), key=_destination) + return tuple(_sum(tuple(group)) for _, group in groupby(ordered, key=_destination)) + + +def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpdateTransactions": + return { + "user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_UserTable")), + "end_user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_EndUserTable")), + "key_list_transactions": dict(_entity_totals(contributions, "LiteLLM_VerificationToken")), + "team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamTable")), + "team_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamMembership")), + "org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationTable")), + "org_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationMembership")), + "project_list_transactions": dict(_entity_totals(contributions, "LiteLLM_ProjectTable")), + "tag_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TagTable")), + "agent_list_transactions": dict(_entity_totals(contributions, "LiteLLM_AgentsTable")), + "model_access_group_list_transactions": dict( + _entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable") + ), + } + + +def _entity_totals(contributions: tuple[Contribution, ...], table: str) -> tuple[tuple[str, float], ...]: + return tuple( + (_entity_key(item), float(dict(item.increments)["spend"])) for item in contributions if item.table == table + ) + + +def _entity_key(item: Contribution) -> str: + fields: Final = dict(item.key) + if item.table == "LiteLLM_TeamMembership": + return f"team_id::{fields['team_id']}::user_id::{fields['user_id']}" + if item.table == "LiteLLM_OrganizationMembership": + return f"org_id::{quote(fields['organization_id'], safe='')}::user_id::{quote(fields['user_id'], safe='')}" + return item.key[0][1] + + +if TYPE_CHECKING: + from litellm.proxy._types import DBSpendUpdateTransactions diff --git a/litellm/accounting/measurements.py b/litellm/accounting/measurements.py deleted file mode 100644 index 83011a280da..00000000000 --- a/litellm/accounting/measurements.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Final - -from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent - - -def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]: - """Return the complete contribution set for one event, ready for one apply call.""" - metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics - occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at - return tuple( - Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric) - for metric in metrics or () - ) diff --git a/litellm/accounting/metrics.py b/litellm/accounting/metrics.py new file mode 100644 index 00000000000..ade715d5d14 --- /dev/null +++ b/litellm/accounting/metrics.py @@ -0,0 +1,30 @@ +from typing import Final + +from prometheus_client import Gauge, Histogram + +from litellm.repositories.accounting_delivery_repository import DeliveryMetrics + +_PENDING: Final = Gauge("litellm_accounting_pending", "Pending SpendLogs deliveries", multiprocess_mode="livemax") +_PARKED: Final = Gauge("litellm_accounting_parked", "Parked SpendLogs deliveries", multiprocess_mode="livemax") +_OLDEST: Final = Gauge( + "litellm_accounting_oldest_pending_seconds", + "Age of the oldest pending SpendLogs delivery", + multiprocess_mode="livemax", +) +_BATCH_TIME: Final = Histogram("litellm_accounting_batch_seconds", "SpendLogs delivery worker batch time") +_BATCH_COUNT: Final = Gauge( + "litellm_accounting_batch_deliveries", "Retained batch deliveries", multiprocess_mode="livemax" +) +_TABLE_BYTES: Final = Gauge( + "litellm_accounting_delivery_table_bytes", "Delivery table size in bytes", multiprocess_mode="livemax" +) + + +def record_deliveries(metrics: DeliveryMetrics, seconds: float) -> None: + _PENDING.set(metrics.pending_count) + _PARKED.set(metrics.parked_count) + _OLDEST.set(max(0.0, metrics.oldest_pending_seconds)) + _BATCH_TIME.observe(seconds) + + _BATCH_COUNT.set(metrics.batch_count) + _TABLE_BYTES.set(metrics.table_bytes) diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py new file mode 100644 index 00000000000..dd52b49f101 --- /dev/null +++ b/litellm/accounting/projection.py @@ -0,0 +1,326 @@ +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from itertools import chain +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + MODEL_INSIGHTS_DEFAULT_TASK, + MODEL_INSIGHTS_TASK_TAG_PREFIX, +) +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry +from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids +from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks +from litellm.proxy.db.model_usage_rollup import date_from_start_time +from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING +from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens +from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + +_FIELDS: Final = TypeAdapter(dict[str, object]) +_STRINGS: Final = TypeAdapter(tuple[str, ...]) +_VALUES: Final = TypeAdapter(tuple[object, ...]) +_ROWS: Final = TypeAdapter(tuple[dict[str, object], ...]) +_TASK_TYPES: Final = frozenset(load_model_insight_tasks()) +_MAX_INDEXED_TEXT_BYTES: Final = 2000 +_MAX_COUNTER: Final = 2**40 + + +@dataclass(frozen=True, slots=True) +class Contribution: + table: str + key: tuple[tuple[str, str], ...] + increments: tuple[tuple[str, int | float], ...] + values: tuple[tuple[str, str | None], ...] = () + + +def _fields(value: object) -> Mapping[str, object]: + if isinstance(value, str): + return _FIELDS.validate_json(value) + return _FIELDS.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({}) + + +def _text(value: object) -> str: + return "" if value is None else str(value) + + +def _number(value: object) -> int | float: + return value if isinstance(value, (int, float)) else 0 + + +def _tags(row: Mapping[str, object]) -> tuple[str, ...]: + value: Final = row.get("request_tags") + if value is None: + return () + parsed: Final = _VALUES.validate_json(value) if isinstance(value, str) else _VALUES.validate_python(value) + return tuple(tag for tag in parsed if isinstance(tag, str)) + + +def _date(row: Mapping[str, object]) -> str | None: + start: Final = row.get("startTime") + return date_from_start_time(start) if isinstance(start, (datetime, str)) else None + + +def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contribution: + return Contribution(table, key, (("spend", spend),)) + + +def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: + spend: Final = _number(row.get("spend")) + entities: Final = _fields(metadata.get("accounting_entities")) + identities: Final = ( + ("LiteLLM_VerificationToken", "token", entities.get("api_key")), + ("LiteLLM_UserTable", "user_id", entities.get("user")), + ("LiteLLM_UserTable", "user_id", entities.get("proxy_budget_name")), + ("LiteLLM_TeamTable", "team_id", entities.get("team_id")), + ("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")), + ("LiteLLM_EndUserTable", "user_id", entities.get("end_user")), + ("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")), + ("LiteLLM_ProjectTable", "project_id", entities.get("project_id")), + ) + groups: Final = _STRINGS.validate_python(metadata.get("accounting_model_access_groups", ())) + return ( + *( + _spend(table, ((column, identity),), spend) + for table, column, identity in identities + if isinstance(identity, str) + ), + *(_spend("LiteLLM_TagTable", (("tag_name", tag),), spend) for tag in _tags(row) if tag), + *(_spend("LiteLLM_ModelAccessGroupBudgetTable", (("access_group_name", group),), spend) for group in groups), + *_membership_contributions(entities, spend), + ) + + +def _membership_contributions(row: Mapping[str, object], spend: float) -> tuple[Contribution, ...]: + user: Final = row.get("user") + if not isinstance(user, str): + return () + owners: Final = ( + ("LiteLLM_TeamMembership", "team_id", row.get("team_id")), + ("LiteLLM_OrganizationMembership", "organization_id", row.get("organization_id")), + ) + return tuple( + _spend(table, ((column, identity), ("user_id", user)), spend) + for table, column, identity in owners + if isinstance(identity, str) + ) + + +def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[tuple[str, int | float], ...]: + usage: Final = _fields(metadata.get("usage_object")) + savings: Final = _fields(metadata.get("accounting_savings")) + internal: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) + caller: Final = not internal and metadata.get(BILLED_RETRY_ATTEMPT_METADATA_KEY) is not True + success: Final = metadata.get("status") != "failure" + duration: Final = row.get("request_duration_ms") + timed: Final = not internal and success and isinstance(duration, int) and duration >= 0 + return ( + ("spend", _number(row.get("spend"))), + ("prompt_tokens", _number(row.get("prompt_tokens"))), + ("completion_tokens", _number(row.get("completion_tokens"))), + ("api_requests", int(caller)), + ("successful_requests", int(caller and success)), + ("failed_requests", int(caller and not success)), + ("cache_read_input_tokens", extract_cache_read_tokens(usage)), + ("cache_creation_input_tokens", extract_cache_creation_tokens(usage)), + ("compression_saved_tokens", extract_compression_saved_tokens(metadata)), + ("compression_savings_spend", _number(savings.get("compression"))), + ("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))), + ("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))), + ("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))), + ("total_response_time_ms", _number(duration) if timed else 0), + ("timed_requests", int(timed)), + ) + + +def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: + date: Final = _date(row) + if date is None or "api_key" not in row: + return () + if "mcp_namespaced_tool_name" not in row and ( + "model" not in row or "custom_llm_provider" not in row or "model_group" not in row + ): + return () + counters: Final = _daily_counters(row, metadata) + endpoint: Final = ROUTE_ENDPOINT_MAPPING.get(_text(row.get("call_type"))) + shared_key: Final = ( + ("date", date), + ("api_key", _text(row.get("api_key"))), + ("model", _text(row.get("model"))), + ("custom_llm_provider", _text(row.get("custom_llm_provider"))), + ("mcp_namespaced_tool_name", _text(row.get("mcp_namespaced_tool_name"))), + ("endpoint", _text(endpoint)), + ) + # The legacy daily organization writer uses the charged org argument, not + # SpendLogs' normalized empty-string identity for a request with no org. + daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")} + return tuple(_daily_rows(daily_row, shared_key, counters)) + + +def _daily_rows( + row: Mapping[str, object], + shared_key: tuple[tuple[str, str], ...], + counters: tuple[tuple[str, int | float], ...], +) -> Iterator[Contribution]: + for entity, table in DAILY_SPEND_TABLES.items(): + for identity in daily_spend_entity_ids(row, entity): + yield Contribution( + table.name, + ((table.entity_id_column, _text(identity)), *shared_key), + counters, + ( + ("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))), + *((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()), + ), + ) + + +def _window_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: + windows: Final = _ROWS.validate_python(metadata.get("accounting_windows", ())) + return tuple( + Contribution( + "LiteLLM_BudgetWindowSpend", + tuple((column, _text(window[column])) for column in ("entity_type", "entity_id", "window_duration")), + (("spend", _number(window.get("spend", row.get("spend")))),), + (("window_start", _text(window["window_start"])), ("started_at", _text(row.get("startTime")))), + ) + for window in windows + ) + + +def project(row: Mapping[str, object]) -> tuple[Contribution, ...]: + metadata: Final = _fields(row.get("metadata")) + contributions: Final = ( + *_entity_contributions(row, metadata), + *_daily_contributions(row, metadata), + *_window_contributions(row, metadata), + *_model_contributions(row, metadata), + *_guardrail_contributions(row, metadata), + ) + return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution)) + + +def _indexable(contribution: Contribution) -> bool: + texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None) + return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts) + + +def _capped(contribution: Contribution) -> Contribution: + """Counters are summed into bigint columns in the charge's transaction, so one capped far below that range + can never overflow a day's total.""" + return replace( + contribution, + increments=tuple( + (name, max(-_MAX_COUNTER, min(_MAX_COUNTER, value)) if isinstance(value, int) else value) + for name, value in contribution.increments + ), + ) + + +def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: + date: Final = _date(row) + origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) + if date is None or (origin and origin != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN): + return () + model: Final = _text(row.get("model")) or "unknown" + caller: Final = not origin + success: Final = row.get("status") == "success" + return ( + Contribution( + "LiteLLM_DailyModelUsage", + ( + ("date", date), + ("model_group", _text(row.get("model_group")) or model), + ("model", model), + ("custom_llm_provider", _text(row.get("custom_llm_provider")) or "unknown"), + ("task_type", _model_task(row)), + ), + ( + ("spend", _number(row.get("spend"))), + ("prompt_tokens", _number(row.get("prompt_tokens"))), + ("completion_tokens", _number(row.get("completion_tokens"))), + ("request_count", int(caller)), + ("successful_requests", int(caller and success)), + ("failed_requests", int(caller and not success)), + ), + ), + ) + + +def _utc_date(row: Mapping[str, object]) -> str | None: + start: Final = row.get("startTime") + try: + parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start + except ValueError: + return None + if not isinstance(parsed, datetime): + return None + return ( + (parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed) + .astimezone(timezone.utc) + .date() + .isoformat() + ) + + +def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: + date: Final = _utc_date(row) + info: Final = metadata.get("guardrail_information") or metadata.get("standard_logging_guardrail_information") + if not row.get("request_id") or date is None or not isinstance(info, list): + return () + return tuple(chain.from_iterable(_guardrail_entry(row, date, entry) for entry in _ROWS.validate_python(info))) + + +def _guardrail_entry(row: Mapping[str, object], date: str, entry: Mapping[str, object]) -> tuple[Contribution, ...]: + identity: Final = _text(entry.get("guardrail_id") or entry.get("guardrail_name")) + usage: Final = entry.get("guardrail_usage") + if not identity or not isinstance(usage, Mapping): + return () + costs: Final = _guardrail_unit_costs(entry) + return tuple( + _guardrail_unit(row, (identity, date, unit), units, None if costs is None else costs.get(unit)) + for unit, units in _fields(usage).items() + if isinstance(units, int) and not isinstance(units, bool) and units > 0 + ) + + +def _guardrail_unit( + row: Mapping[str, object], identity: tuple[str, str, str], units: int, cost: float | None +) -> Contribution: + guardrail_id, date, unit = identity + return Contribution( + "LiteLLM_DailyGuardrailUsageUnits", + ( + ("guardrail_id", guardrail_id), + ("date", date), + ("team_id", _text(row.get("team_id"))), + ("api_key", _text(row.get("api_key"))), + ("usage_unit", unit), + ), + (("units", units), ("cost", cost or 0.0), ("untracked_units", units if cost is None else 0)), + ) + + +def _model_task(row: Mapping[str, object]) -> str: + return next( + ( + task + for tag in _tags(row) + if tag.startswith(MODEL_INSIGHTS_TASK_TAG_PREFIX) + and (task := tag.removeprefix(MODEL_INSIGHTS_TASK_TAG_PREFIX)) in _TASK_TYPES + ), + MODEL_INSIGHTS_DEFAULT_TASK, + ) + + +def _guardrail_unit_costs(entry: Mapping[str, object]) -> Mapping[str, float | None] | None: + try: + cost: Final = GuardrailCostByUnitEntry.model_validate(entry) + except ValidationError: + return None + return None if cost.guardrail_cost_in_spend is False else cost.guardrail_cost_by_unit diff --git a/litellm/accounting/protocols.py b/litellm/accounting/protocols.py deleted file mode 100644 index 0208e37fbdf..00000000000 --- a/litellm/accounting/protocols.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Contracts only. Implementations own persistence, pricing, projections and live budgets. - -The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``. -Usage envelopes retain the public event and request-scoped pricing context: quantities with units, -model/provider, requested and billed tier/region when known, and per-request/deployment overrides. -These inputs, including the effective cost-map rates, must survive request teardown. -Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot. -T5/T6 own envelope serialization; this public API does not introduce a second persisted schema. -""" - -from typing import Protocol - -from litellm.accounting.types import ( - AdmissionDecision, - AggregateQuery, - AggregateRow, - CostEvent, - Dimension, - InboundUsageEvent, - Measurement, - Metric, - OutboundUsageEvent, - Unpriced, -) - - -class UsageMeter(Protocol): - async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None: - """Acknowledge commit, no-op identical IDs and reject conflicting content. - - Persist the event and its request-scoped pricing context before acknowledging. - HTTP acknowledgement is a separate policy. Unknown usage remains None. - """ - ... - - -class CostCompiler(Protocol): - async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced: - """Reuse the committed charge or finalize one charge and its delivery intent. - - Resolve captured request-scoped context and rates by event identity. - Persist the resolved inputs in the required snapshot. Replay - reuses the committed result and never consumes free allowance again. - Missing pricing is Unpriced, never an inferred zero. - """ - ... - - -class CostAggregator(Protocol): - async def apply(self, measurements: tuple[Measurement, ...]) -> None: - """Apply one source event's complete contribution set per target/version. - - Each target commits all contributions and its receipt atomically. A receipt - for the first metric must not suppress the rest. Usage need not wait for cost. - """ - ... - - async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]: - """Return disjoint [start, end) rows per group using existing daily projections. - - Filters combine with AND, values within a filter with IN. DAY emits daily - buckets; None emits whole-interval totals. Reject unsupported metric/scope - combinations. Different groups may overlap, including non-additive tags. - """ - ... - - -class BudgetEnforcer(Protocol): - async def reserve( - self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...] - ) -> AdmissionDecision: - """Apply existing admission policies and estimates; duplicate IDs add no hold.""" - ... - - async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None: - """Settle the committed charge with atomic deduplication and hold mutation. - - Preserve scope/window attribution and retry unfinished settlement after - restart. SQL projection receipts do not provide live-counter atomicity. - """ - ... - - async def release(self, reservation_id: str) -> None: - """Idempotently release unused exposure at the request boundary, preserving charges.""" - ... diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py new file mode 100644 index 00000000000..d5652f79ed0 --- /dev/null +++ b/litellm/accounting/snapshot.py @@ -0,0 +1,161 @@ +from collections.abc import Callable, Mapping, Sequence +from datetime import datetime +from typing import TYPE_CHECKING, Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure +from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens +from litellm.proxy.spend_tracking.savings import SavingsSpend, compute_savings_spend, marks_gateway_injection + +if TYPE_CHECKING: + from litellm.router import Router + +_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def _mapping(value: object) -> Mapping[str, object] | None: + return _FIELDS.validate_python(value) if isinstance(value, Mapping) else None + + +_METADATA_FIELDS: Final = frozenset( + { + "usage_object", + "status", + "guardrail_information", + "standard_logging_guardrail_information", + "compression_savings", + "internal_call_origin", + "billed_retry_attempt", + } +) + + +_ACCOUNTING_FIELDS: Final = frozenset( + { + "request_id", + "call_type", + "api_key", + "spend", + "prompt_tokens", + "completion_tokens", + "startTime", + "model", + "model_group", + "custom_llm_provider", + "user", + "team_id", + "organization_id", + "end_user", + "agent_id", + "request_tags", + "status", + "request_duration_ms", + "mcp_namespaced_tool_name", + } +) + + +def build_accounting_payload( + payload: SpendLogsPayload, + *, + entities: Mapping[str, str | None], + model_access_groups: Sequence[str], + windows: Sequence[Mapping[str, object]], + router: "Callable[[], Router | None]", +) -> Mapping[str, object]: + metadata: Final = _FIELDS.validate_json(payload["metadata"]) + savings: Final = _savings_or_zero(payload, metadata, router) + accounting_metadata: Final = { + **{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS}, + "accounting_entities": entities, + "accounting_savings": savings._asdict(), + "accounting_model_access_groups": model_access_groups, + "accounting_windows": windows, + } + return _FIELDS.validate_python( + safe_json_structure( + { + **{ + key: value.isoformat() if isinstance(value, datetime) else value + for key, value in payload.items() + if key in _ACCOUNTING_FIELDS + }, + "metadata": accounting_metadata, + }, + non_finite_as_null=True, + ) + ) + + +def _savings_or_zero( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + """Savings are analytics computed from caller-supplied counts, so one that cannot be computed is stored as zero + rather than losing the charge it rides with.""" + try: + return _savings(payload, metadata, router) + except (ArithmeticError, ValueError, TypeError) as error: + verbose_proxy_logger.warning( + "Recording zero savings for request_id=%s, savings could not be computed: %s", payload["request_id"], error + ) + return SavingsSpend(compression=0.0, prompt_caching=0.0) + + +def _savings( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + return compute_savings_spend( + model=payload["model"], + custom_llm_provider=payload["custom_llm_provider"], + compression_saved_tokens=extract_compression_saved_tokens(metadata), + gateway_injected_cache=marks_gateway_injection(metadata, payload["model_id"]), + routing_decision=_mapping(metadata.get("routing_decision")), + model_id=payload["model_id"], + llm_router=router, + usage_object=_mapping(metadata.get("usage_object")), + cost_breakdown=_mapping(metadata.get("cost_breakdown")), + recorded_autorouter_savings=metadata.get("autorouter_savings"), + recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")), + billed_at=payload["endTime"], + ) + + +_GUARDRAIL_ENTRIES: Final = TypeAdapter(tuple[dict[str, object], ...]) +_GUARDRAIL_METADATA_FIELDS: Final = frozenset({"guardrail_information", "standard_logging_guardrail_information"}) +_GUARDRAIL_ACCOUNTING_FIELDS: Final = frozenset( + { + "guardrail_id", + "guardrail_name", + "guardrail_provider", + "guardrail_usage", + "guardrail_cost_by_unit", + "guardrail_cost_in_spend", + } +) +_GUARDRAIL_RESPONSE: Final = TypeAdapter(dict[str, object]) + + +def _accounting_guardrail_entries(value: object) -> object: + try: + entries: Final = _GUARDRAIL_ENTRIES.validate_python((value,) if isinstance(value, dict) else value) + except ValidationError: + return None + return [_accounting_guardrail_entry(entry) for entry in entries] + + +def _accounting_guardrail_entry(entry: Mapping[str, object]) -> Mapping[str, object]: + kept: Final = {key: item for key, item in entry.items() if key in _GUARDRAIL_ACCOUNTING_FIELDS} + response: Final = entry.get("guardrail_response") + if not isinstance(response, dict) or "tokens_saved" not in response: + return kept + return { + **kept, + "guardrail_response": {"tokens_saved": _GUARDRAIL_RESPONSE.validate_python(response)["tokens_saved"]}, + } + + +def _accounting_metadata_field(key: str, value: object) -> object: + return _accounting_guardrail_entries(value) if key in _GUARDRAIL_METADATA_FIELDS else value diff --git a/litellm/accounting/types.py b/litellm/accounting/types.py deleted file mode 100644 index 2ee6f60f756..00000000000 --- a/litellm/accounting/types.py +++ /dev/null @@ -1,428 +0,0 @@ -"""Accounting values use UTC instants, decimal strings on the wire and USD charges.""" - -from collections.abc import Mapping -from datetime import datetime, timezone -from decimal import Decimal -from enum import Enum -from typing import Annotated, Final, Literal, TypeAlias - -from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator -from typing_extensions import Self - -from litellm.types.utils import CallTypes - -_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True) -Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")] -Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)] - - -def _utc(value: datetime) -> datetime: - return value.astimezone(timezone.utc) - - -Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)] - - -class MetricName(str, Enum): - """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens.""" - - REQUESTS = "requests" - SUCCESSFUL_REQUESTS = "successful_requests" - FAILED_REQUESTS = "failed_requests" - PROMPT_TOKENS = "prompt_tokens" - COMPLETION_TOKENS = "completion_tokens" - REASONING_TOKENS = "reasoning_tokens" - TOTAL_TOKENS = "total_tokens" - CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens" - CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens" - CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr" - SPEND = "spend" - - -class DimensionName(str, Enum): - KEY = "key" - USER = "user" - TEAM = "team" - ORGANIZATION = "organization" - END_USER = "end_user" - MODEL = "model" - MODEL_GROUP = "model_group" - DEPLOYMENT = "deployment" - PROVIDER = "provider" - TOOL = "tool" - AGENT = "agent" - PROJECT = "project" - MODEL_ACCESS_GROUP = "model_access_group" - API_BASE = "api_base" - SERVICE_TIER = "service_tier" - TAG = "tag" - - -class Outcome(str, Enum): - SUCCEEDED = "succeeded" - FAILED = "failed" - CANCELLED = "cancelled" - - -class Granularity(str, Enum): - DAY = "day" - - -class EventKind(str, Enum): - INBOUND_USAGE = "inbound_usage" - OUTBOUND_USAGE = "outbound_usage" - COST = "cost" - - -class DeliveryState(str, Enum): - PENDING = "pending" - APPLIED = "applied" - PARKED = "parked" - - -class DeliveryTarget(str, Enum): - """One member per existing table the projector writes; each is applied and receipted independently.""" - - KEY_SPEND = "LiteLLM_VerificationToken" - USER_SPEND = "LiteLLM_UserTable" - TEAM_SPEND = "LiteLLM_TeamTable" - TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership" - ORGANIZATION_SPEND = "LiteLLM_OrganizationTable" - ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership" - END_USER_SPEND = "LiteLLM_EndUserTable" - AGENT_SPEND = "LiteLLM_AgentsTable" - TAG_SPEND = "LiteLLM_TagTable" - PROJECT_SPEND = "LiteLLM_ProjectTable" - MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable" - BUDGET_SPEND = "LiteLLM_BudgetTable" - BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend" - DAILY_USER_SPEND = "LiteLLM_DailyUserSpend" - DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend" - DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend" - DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend" - DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend" - DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend" - DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage" - DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend" - AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend" - - -_SUBSET_PAIRS: Final = ( - (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS), - (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS), -) - - -def _check_subsets(values: Mapping[str, Decimal]) -> None: - for part, whole in _SUBSET_PAIRS: - if part.value in values and whole.value in values and values[part.value] > values[whole.value]: - raise ValueError(f"{part.value} cannot exceed {whole.value}") - - -class Metric(BaseModel): - model_config = _CONFIG - - name: MetricName - value: Quantity - unit: Literal["count", "USD"] = "count" - - @model_validator(mode="after") - def validate_unit(self) -> Self: - if (self.name is MetricName.SPEND) != (self.unit == "USD"): - raise ValueError("SPEND requires USD; usage metrics require count") - if self.unit == "count" and self.value != self.value.to_integral_value(): - raise ValueError("Count metrics require integral values") - return self - - -class Dimension(BaseModel): - model_config = _CONFIG - - name: DimensionName - value: Identifier - - -_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP}) - - -def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]: - if len(frozenset(values)) != len(values): - raise ValueError("Duplicate dimension values") - single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS) - if len(frozenset(single_names)) != len(single_names): - raise ValueError("Only tag and model access group dimensions may have multiple values") - return values - - -Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)] -_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS)) - - -def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]: - if len(frozenset(value.name for value in values)) != len(values): - raise ValueError("Duplicate metric names") - return values - - -Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)] - - -def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None: - if end < start or (end == start and not allow_empty): - raise ValueError("End must follow start") - - -class InboundUsageEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - request_id: Identifier - call_type: CallTypes - started_at: Instant - ended_at: Instant - outcome: Outcome - dimensions: Dimensions - metrics: Metrics - - @model_validator(mode="after") - def validate_caller_activity(self) -> Self: - _interval(self.started_at, self.ended_at, allow_empty=True) - if any(metric.name not in _CALLER_METRICS for metric in self.metrics): - raise ValueError("Inbound events own only caller request metrics") - if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics): - raise ValueError("Inbound events count exactly one caller request") - for metric in self.metrics: - if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED): - raise ValueError("Successful request count must match outcome") - if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED): - raise ValueError("Failed request count must match outcome") - return self - - -class OutboundUsageEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - attempt_id: Identifier - request_id: Identifier | None - retry_of_attempt_id: Identifier | None - call_type: CallTypes - started_at: Instant - ended_at: Instant - outcome: Outcome - dimensions: Dimensions - metrics: Metrics | None - - @model_validator(mode="after") - def validate_attempt_usage(self) -> Self: - _interval(self.started_at, self.ended_at, allow_empty=True) - if self.retry_of_attempt_id == self.attempt_id: - raise ValueError("An attempt cannot retry itself") - if self.metrics is not None and any( - metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics - ): - raise ValueError("Outbound events own only attempt usage metrics") - if self.metrics is not None: - _check_subsets({metric.name.value: metric.value for metric in self.metrics}) - return self - - -PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"] - - -class Rate(BaseModel): - model_config = _CONFIG - - measure: Identifier - unit: PricingUnit - usd_per_unit: Quantity - - -class PricedQuantity(BaseModel): - """A pricing input in its billing unit. Only seconds may be fractional.""" - - model_config = _CONFIG - - measure: Identifier - unit: PricingUnit - value: Quantity - - @model_validator(mode="after") - def validate_discrete_quantity(self) -> Self: - if self.unit != "second" and self.value != self.value.to_integral_value(): - raise ValueError("Discrete priced quantities require integral values") - return self - - -class Multiplier(BaseModel): - """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5.""" - - model_config = _CONFIG - - name: Identifier - value: Identifier - factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)] - - -class NamedCost(BaseModel): - """A free-form extra charge by name, e.g. azure_model_router_flat_cost.""" - - model_config = _CONFIG - - name: Identifier - amount: Quantity - - -class PricingSnapshot(BaseModel): - """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change. - - Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload. - Retries reuse the committed charge and snapshot. - Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge. - Component sums and float tolerances belong to reconciliation, not this structural contract. - """ - - model_config = _CONFIG - - schema_version: Literal[1] = 1 - calculator_version: Identifier - model: Identifier - provider: Identifier | None - pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"] - priced_model_key: Identifier - multipliers: tuple[Multiplier, ...] = () - rates: tuple[Rate, ...] = () - usage: tuple[PricedQuantity, ...] = () - input_cost: Quantity | None = None - output_cost: Quantity | None = None - cache_read_cost: Quantity | None = None - cache_creation_cost: Quantity | None = None - reasoning_cost: Quantity | None = None - tool_usage_cost: Quantity | None = None - guardrail_cost: Quantity | None = None - additional_costs: tuple[NamedCost, ...] = () - original_cost: Quantity | None = None - discount_percent: Quantity | None = None - discount_amount: Quantity | None = None - margin_percent: Quantity | None = None - margin_fixed_amount: Quantity | None = None - margin_total_amount: Quantity | None = None - upstream_reported_cost: Quantity | None = None - total_cost: Quantity - - @model_validator(mode="after") - def validate_snapshot(self) -> Self: - rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates) - quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage) - if len(rate_keys) != len(self.rates): - raise ValueError("Duplicate rate measures and units") - if len(quantity_keys) != len(self.usage): - raise ValueError("Duplicate priced quantity measures and units") - if not rate_keys.issubset(quantity_keys): - raise ValueError("Every rate requires one priced quantity with the same measure and unit") - if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs): - raise ValueError("Duplicate additional cost names") - if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers): - raise ValueError("Duplicate multiplier names") - _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"}) - return self - - -class CostEvent(BaseModel): - model_config = _CONFIG - - event_id: Identifier - usage_event_id: Identifier - occurred_at: Instant - dimensions: Dimensions - charges: Metrics - snapshot: PricingSnapshot - - @model_validator(mode="after") - def validate_charge(self) -> Self: - if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND: - raise ValueError("A cost event requires one authoritative USD spend metric") - if self.event_id == self.usage_event_id: - raise ValueError("Cost and usage events require distinct identities") - if self.snapshot.total_cost != self.charges[0].value: - raise ValueError("Snapshot total must equal the authoritative spend") - return self - - -class Unpriced(BaseModel): - model_config = _CONFIG - - usage_event_id: Identifier - reason: Identifier - - -class Measurement(BaseModel): - model_config = _CONFIG - - source_event_id: Identifier - occurred_at: Instant - dimensions: Dimensions - metric: Metric - - -class DimensionFilter(BaseModel): - model_config = _CONFIG - - name: DimensionName - values: Annotated[tuple[Identifier, ...], Field(min_length=1)] - - -class AggregateQuery(BaseModel): - model_config = _CONFIG - - metric: MetricName - filters: tuple[DimensionFilter, ...] - group_by: tuple[DimensionName, ...] - start: Instant - end: Instant - granularity: Granularity | None - - @model_validator(mode="after") - def validate_query(self) -> Self: - _interval(self.start, self.end) - if len(frozenset(item.name for item in self.filters)) != len(self.filters): - raise ValueError("Use one IN filter per dimension; filters combine with AND") - if len(frozenset(self.group_by)) != len(self.group_by): - raise ValueError("Duplicate group-by dimensions") - if any( - (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0) - for instant in (self.start, self.end) - ): - raise ValueError("Initial reporting queries require UTC day boundaries") - return self - - -class AggregateRow(BaseModel): - model_config = _CONFIG - - start: Instant - end: Instant - dimensions: Dimensions - value: Quantity - - @model_validator(mode="after") - def validate_bounds(self) -> Self: - _interval(self.start, self.end) - return self - - -def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]: - """Validate [start, end) buckets per group; overlapping tag groups are not additive.""" - groups: Final = frozenset(frozenset(row.dimensions) for row in rows) - for group in groups: - ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start) - if any(left.end > right.start for left, right in zip(ordered, ordered[1:])): - raise ValueError("Aggregate buckets overlap within a dimension group") - return rows - - -class AdmissionDecision(BaseModel): - model_config = _CONFIG - - allowed: bool - reason: Identifier | None = None diff --git a/litellm/accounting/worker.py b/litellm/accounting/worker.py new file mode 100644 index 00000000000..df71f2da31c --- /dev/null +++ b/litellm/accounting/worker.py @@ -0,0 +1,102 @@ +import os +import time +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Protocol, + cast, # noqa: TID251 # generated Prisma transaction signatures cannot express the borrowed transaction protocol +) + +from litellm._logging import verbose_proxy_logger +from litellm.accounting.application import apply_deliveries +from litellm.accounting.metrics import record_deliveries +from litellm.accounting.projection import Contribution +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerDatabase, + LedgerTransaction, +) +from litellm.types.integrations.slack_alerting import AlertType + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import PrismaClient, ProxyLogging + + +class _AccountingAlerts(Protocol): + async def alerting_handler( + self, message: str, level: Literal["Low", "Medium", "High"], alert_type: AlertType + ) -> None: ... + + +_JOB: Final = "accounting_delivery" + + +async def flush_accounting_deliveries( + writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging" +) -> None: + acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False) + if acquired is False: + return + applied: tuple[Contribution, ...] = () # rebind-ok: keep committed contributions for cache invalidation + + async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None: + nonlocal applied + applied = await apply_deliveries( # rebind-ok: invalidate caches only after the outer transaction commits + transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj + ) + + try: + database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction + LedgerDatabase, prisma_client + ) + repository: Final = AccountingDeliveryRepository(database) + started: Final = time.monotonic() + batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "10000")) + + async def run_batch(*, retry_only: bool) -> bool: + result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only) + if result.applied: + await writer.invalidate_accounting_entities(applied, proxy_logging_obj) + if result.parked: + verbose_proxy_logger.error( + "Accounting delivery parked %d requests after isolated failures", len(result.parked) + ) + await _alert( + proxy_logging_obj, + message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.", + ) + return bool(result.applied or result.failed) + + await run_batch(retry_only=False) + retry_deadline: Final = time.monotonic() + 5 + for _ in range(batch_size): + if time.monotonic() >= retry_deadline or not await run_batch(retry_only=True): + break + await repository.cleanup_applied( + retention_days=int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1")), + batch_size=batch_size * 2, + ) + metrics: Final = await repository.metrics() + record_deliveries(metrics, time.monotonic() - started) + if metrics.batch_count >= int( + os.environ.get("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1000000") + ) or metrics.table_bytes >= int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1073741824")): + await _alert( + proxy_logging_obj, + message=f"Accounting delivery storage requires review. Retained batches={metrics.batch_count}, table bytes={metrics.table_bytes}.", + ) + if metrics.oldest_pending_seconds > 300: + await _alert( + proxy_logging_obj, + message=f"Accounting deliveries are delayed. Pending={metrics.pending_count}, oldest={metrics.oldest_pending_seconds:.0f}s.", + ) + finally: + if acquired: + await writer.pod_lock_manager.release_lock(cronjob_id=_JOB) + + +async def _alert(alerts: _AccountingAlerts, message: str) -> None: + await alerts.alerting_handler(message=message, level="High", alert_type=AlertType.failed_tracking_spend) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index e284845b2ff..c03f7cda990 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -198,7 +198,6 @@ def _load_beta_headers_config() -> dict: if _BETA_HEADERS_CONFIG is not None: return _BETA_HEADERS_CONFIG - # Get the URL from environment or use default from litellm import anthropic_beta_headers_url _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index a0c0a90be5f..d272a235758 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -10,6 +10,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils import batch_utils as core_batch_utils from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo @@ -38,8 +39,8 @@ class BatchCostUsageResult: completion_cost: float = 0.0 -_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"}) _DECODED_BATCH_LINE: Final = TypeAdapter(object) +batch_cost_is_final: Final = core_batch_utils.batch_cost_is_final def _uses_native_vertex_output( @@ -54,26 +55,6 @@ def _uses_native_vertex_output( return first_row is not None and is_native_vertex_batch_output_row(first_row) -_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"}) - - -def batch_cost_is_final(batch: Batch) -> bool: - """Whether this retrieve of the batch is the one to account its cost from. - - A batch still in flight has nothing to price, and a "completed" batch can report - no output_file_id for a moment before the output populates; pricing either records - $0 under the batch's single spend row and pins it there. Final means a completed - batch whose output file has arrived or whose counts prove no line succeeded, or - any other terminal status (failed, cancelled, expired). - """ - if batch.status not in _TERMINAL_BATCH_STATUSES: - return False - if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None: - return True - request_counts: Final = batch.request_counts - return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0 - - async def calculate_batch_cost_and_usage( file_content_dictionary: Sequence[dict[str, object]], custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"], diff --git a/litellm/blog_posts.json b/litellm/blog_posts.json index fa768b3ec57..adcb3911652 100644 --- a/litellm/blog_posts.json +++ b/litellm/blog_posts.json @@ -1,10 +1,588 @@ { "posts": [ { - "title": "Realtime WebRTC HTTP Endpoints", - "description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.", - "date": "2026-03-12", - "url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints" + "title": "Release cadence changes and an LTS version pilot", + "description": "Starting the week of November 2, 2026, release candidates and stable releases ship on Tuesday mornings, and LiteLLM is piloting a long-term support line on 1.108 for one month.", + "date": "2026-11-02", + "url": "https://docs.litellm.ai/blog/release-cadence-and-lts-pilot", + "tags": [ + "release", + "support" + ] + }, + { + "title": "How to build an agent with grounding on LiteLLM", + "description": "Build a grounded chat agent in four steps with LiteLLM's AI Gateway. Connect your evidence, call a grounding guardrail, and show its verdict in chat.", + "date": "2026-10-31", + "url": "https://docs.litellm.ai/blog/grounded-agent", + "tags": [ + "engineering", + "guardrails", + "agents" + ] + }, + { + "title": "Introducing LiteLLM’s Unified Cost Accounting Engine", + "description": "Consistent cost reporting, independently scalable accounting, and spend analysis by team, user, model, or custom dimensions.", + "date": "2026-10-31", + "url": "https://docs.litellm.ai/blog/cost-infrastructure-makeover", + "tags": [ + "engineering", + "reliability" + ] + }, + { + "title": "What we are changing after October 21", + "description": "Why the controls built after March did not stop the v1.107.0 release, and the identity, credential and detection changes we are making in response.", + "date": "2026-10-30", + "url": "https://docs.litellm.ai/blog/security-changes-after-oct-21-2026", + "tags": [ + "security", + "incident-report" + ] + }, + { + "title": "Use Hex to analyze LiteLLM costs, then set budgets and routing", + "description": "Connect Hex to LiteLLM’s Postgres data, import a cost dashboard, then use the LiteLLM UI to set a budget or change model routing.", + "date": "2026-10-30", + "url": "https://docs.litellm.ai/blog/hex-litellm-roi", + "tags": [ + "observability", + "cost tracking", + "ai-gateway" + ] + }, + { + "title": "Introducing the LiteLLM Private Registry", + "description": "LiteLLM is committing to a private registry to provide early access to fixes for the most severe security vulnerabilities, ahead of public release.", + "date": "2026-10-27", + "url": "https://docs.litellm.ai/blog/litellm-private-registry", + "tags": [ + "security" + ] + }, + { + "title": "Morph open-weight models on LiteLLM", + "description": "Use Morph's Kimi, GLM, and DeepSeek models through LiteLLM with tool calling, structured output, streaming, and prompt-cache cost tracking.", + "date": "2026-10-27", + "url": "https://docs.litellm.ai/blog/morph-provider", + "tags": [] + }, + { + "title": "Abliterated models on LiteLLM", + "description": "Call abliterated models from Refuseless and Abliteration through LiteLLM, with gateway access controls, streaming, tool calls, and cached-token pricing.", + "date": "2026-10-26", + "url": "https://docs.litellm.ai/blog/abliterated-models", + "tags": [ + "engineering" + ] + }, + { + "title": "GPT-6.1 Terra API Pricing and Day 0 Support on LiteLLM", + "description": "GPT-6.1 Terra costs $1 per 1M input tokens and $5 per 1M output tokens, half the price of GPT-6.1 Sol. LiteLLM supports it on day 0.", + "date": "2026-10-26", + "url": "https://docs.litellm.ai/blog/gpt-6-1-terra", + "tags": [ + "openai", + "gpt-6.1", + "gpt-6.1-terra", + "pricing", + "day 0 support", + "card_title:GPT-6.1 Terra", + "card_summary:Half the price of GPT-6.1 Sol at $1 / $5 per 1M tokens, 50.3% on Terminal-Bench 4.0, 1.05M context." + ] + }, + { + "title": "Rust Migration: /v1/messages end-to-end in Rust by Dec 1", + "description": "By December 1st, the proxy will handle Anthropic messages requests 20x faster", + "date": "2026-10-26", + "url": "https://docs.litellm.ai/blog/rust-migration-rerouted", + "tags": [ + "rust", + "rust-migration", + "ai-gateway", + "engineering" + ] + }, + { + "title": "OpenCode Go is now a LiteLLM provider", + "description": "Use your OpenCode Go subscription through LiteLLM. Connect once, route models to the right API, preserve sessions, and track estimated usage.", + "date": "2026-10-24", + "url": "https://docs.litellm.ai/blog/opencode-go", + "tags": [ + "providers", + "tutorial" + ] + }, + { + "title": "The attack chain behind the v1.107.0 LiteLLM release", + "description": "How an internal key committed to a public repository in May led to three stolen GitHub tokens and the compromised litellm v1.107.0 release.", + "date": "2026-10-23", + "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain", + "tags": [ + "security", + "incident-report" + ] + }, + { + "title": "Tokenize text across vLLM and Watsonx", + "description": "Choose token counting for prompt budgets or /tokenize to inspect tokens. Use one request format across vLLM and Watsonx through LiteLLM.", + "date": "2026-10-23", + "url": "https://docs.litellm.ai/blog/unified-tokenize", + "tags": [ + "engineering" + ] + }, + { + "title": "October Townhall Updates: 426 Bug Fixes, 88.4% Coverage, and the Decisions API", + "description": "A recap of the October LiteLLM town hall: security changes that need action before you upgrade, 426 bug fixes, test coverage at 88.4%, the Decisions API, Lens, Moyai, and LiteLLM Relay.", + "date": "2026-10-23", + "url": "https://docs.litellm.ai/blog/october-townhall-updates", + "tags": [ + "townhall", + "security", + "reliability", + "product" + ] + }, + { + "title": "Contained: Oct 21, 2026 LiteLLM release v1.107.0 Compromised", + "description": "LiteLLM release v1.107.0 was compromised by a malicious actor.", + "date": "2026-10-21", + "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026", + "tags": [ + "security", + "incident-report" + ] + }, + { + "title": "JevRank: decision models through LiteLLM’s rerank API", + "description": "Preview JevRank for low-cost reranking with Perplexity and TypeSafe decision models. Explore SciFact results, reproducible data and LiteLLM setup.", + "date": "2026-10-20", + "url": "https://docs.litellm.ai/blog/jevrank", + "tags": [ + "engineering", + "reranking", + "decisions" + ] + }, + { + "title": "Cache-Aware Routing for Self-Hosted Models", + "description": "Route each agent session to the same vLLM pod on Kubernetes to reuse cached prompt prefixes.", + "date": "2026-10-20", + "url": "https://docs.litellm.ai/blog/cache-aware-routing", + "tags": [ + "routing", + "kubernetes", + "vllm", + "ai-gateway" + ] + }, + { + "title": "Day 0 Support: TypeSafe Jev 1.14: Adds images, box questions, and long context", + "description": "TypeSafe Jev 1.14 reads images, answers box questions with coordinates and takes long-context requests. LiteLLM serves all three on /v1/systemone and /v1/decisions behind TypeSafe's opt-in headers.", + "date": "2026-10-15", + "url": "https://docs.litellm.ai/blog/typesafe_jev_images", + "tags": [ + "typesafe", + "jev", + "decisions", + "day 0 support", + "card_title:Jev 1.14", + "card_summary:Jev 1.14 reads PNG and JPEG images, answers box questions with coordinates, and takes long-context requests on LiteLLM." + ] + }, + { + "title": "Decision models on LiteLLM: /v1/decisions and /v1/systemone", + "description": "Decision models return probabilities instead of text, which makes routing, triage and guardrail checks cheap and deterministic. LiteLLM serves nine providers of them behind one request shape.", + "date": "2026-10-14", + "url": "https://docs.litellm.ai/blog/decision-models", + "tags": [ + "openai", + "typesafe", + "decisions", + "day 0 support", + "card_title:Decision Models", + "card_summary:Typed yes/no, choice and score answers as probabilities, from 9 providers on /v1/decisions and /v1/systemone." + ] + }, + { + "title": "Workshop: Build a self-hosted cloud agent.", + "description": "Build a self-hosted cloud agent with the LiteLLM team. A two-week remote workshop, October 19–30, to build a working v0 and deploy an internal beta on your infrastructure.", + "date": "2026-10-13", + "url": "https://docs.litellm.ai/blog/cloud-agent-workshop", + "tags": [ + "agents", + "infrastructure", + "announcement" + ] + }, + { + "title": "Gemini 4 Argon API Pricing and Day 0 Support on LiteLLM", + "description": "Gemini 4 Argon costs $2.00 per 1M input tokens and $10.00 per 1M output tokens until Nov 30. LiteLLM supports it on day 0 on AI Studio and Google Cloud.", + "date": "2026-10-13", + "url": "https://docs.litellm.ai/blog/gemini-4-argon", + "tags": [ + "gemini", + "gemini 4", + "gemini-4-argon", + "pricing", + "day 0 support", + "card_title:Gemini 4 Argon", + "card_summary:Google's new flagship: 77.9% on DeepSWE v1.1, 1M context, thinking levels, $2 / $10 per 1M tokens until Nov 30." + ] + }, + { + "title": "October Townhall: Product + Roadmap Updates", + "description": "Join the LiteLLM October townhall on Friday, 23 October at 7:30 AM PT to learn about LiteLLM's product updates and roadmap.", + "date": "2026-10-10", + "url": "https://docs.litellm.ai/blog/october-townhall-announcement", + "tags": [ + "announcement", + "townhall" + ] + }, + { + "title": "Introducing Microsoft 365 Copilot in LiteLLM", + "description": "Use Microsoft 365 Copilot from your applications through LiteLLM, with access to your Microsoft 365 work data.", + "date": "2026-10-09", + "url": "https://docs.litellm.ai/blog/microsoft-365-copilot", + "tags": [ + "announcement", + "ai-gateway" + ] + }, + { + "title": "Introducing litellm-core: the first step toward a leaner SDK", + "description": "Ongoing work to reduce LiteLLM's dependencies and installation footprint, improve runtime loading, and make imports faster", + "date": "2026-10-08", + "url": "https://docs.litellm.ai/blog/introducing-litellm-core", + "tags": [] + }, + { + "title": "Claude Haiku 5.5 API Pricing and Day 0 Support on LiteLLM", + "description": "Claude Haiku 5.5 costs $0.10 per 1M input tokens and $0.50 per 1M output tokens. LiteLLM supports it on day 0 on Anthropic, AWS, Google Cloud and Azure.", + "date": "2026-10-07", + "url": "https://docs.litellm.ai/blog/claude-haiku-5-5", + "tags": [ + "anthropic", + "claude", + "claude haiku", + "haiku 5.5", + "pricing", + "day 0 support", + "card_title:Claude Haiku 5.5", + "card_summary:90% cheaper than Haiku 4.5 on short prompts, 1M context, 72.4% on OSWorld 2.1, $0.10 / $0.50 per 1M tokens." + ] + }, + { + "title": "Open Sourcing Moyai: Self-Hosted Cloud Coding Agent", + "description": "Run cloud coding agents on your own infrastructure with Moyai. Choose your agent and models through LiteLLM, track costs, and follow the setup guide.", + "date": "2026-10-07", + "url": "https://docs.litellm.ai/blog/moyai-open-source", + "tags": [ + "agents", + "open-source", + "infrastructure" + ] + }, + { + "title": "Day 0 Support: Nano Banana 2.1", + "description": "Day 0 support for Gemini Nano Banana 2.1 on LiteLLM, on Google AI Studio and Gemini Enterprise Agent Platform, with image output at half Nano Banana 2's price.", + "date": "2026-10-06", + "url": "https://docs.litellm.ai/blog/nano_banana_2_1", + "tags": [ + "gemini", + "gemini enterprise agent platform", + "nano banana", + "image generation", + "day 0 support", + "card_title:Nano Banana 2.1", + "card_summary:Gemini image model with image output at half Nano Banana 2's price: $30 / MTok image out, $1.50 text in." + ] + }, + { + "title": "How LiteLLM Lens finds repeated failures across 1,000s of agent traces", + "description": "Inside Lens: parallel trace review, Python tools for large traces, and investigations that connect repeated failures to source evidence.", + "date": "2026-10-05", + "url": "https://docs.litellm.ai/blog/lens-failure-patterns", + "tags": [ + "lens", + "agent-tracing", + "engineering" + ] + }, + { + "title": "How we cut time to first byte by 94% for long prompts", + "description": "A 440k-token benchmark went from 553 ms to 35 ms median time to first byte. We removed unnecessary prompt-cache routing work before the model call.", + "date": "2026-10-03", + "url": "https://docs.litellm.ai/blog/long-prompt-latency", + "tags": [ + "performance", + "proxy", + "engineering", + "ai-gateway" + ] + }, + { + "title": "How we built our own internal Devin in 2 days", + "description": "How we built Moyai Devin with Render, Modal, Hermes, Temporal, and LiteLLM: durable sessions, parallel agents, Slack, and shared organization connections.", + "date": "2026-10-03", + "url": "https://docs.litellm.ai/blog/internal-devin-two-days", + "tags": [ + "engineering", + "agents", + "infrastructure", + "slack" + ] + }, + { + "title": "Adding Self-hosted Auto Router Classifiers: Laya & Nimble", + "description": "Use Laya or Bespoke Nimble to classify Auto Router requests on your own infrastructure. Control where classification runs, which model you serve, and how you provision it.", + "date": "2026-10-02", + "url": "https://docs.litellm.ai/blog/laya-nimble-classifiers", + "tags": [ + "auto-router", + "product", + "ai-gateway" + ] + }, + { + "title": "Launching LiteLLM Lens", + "description": "LiteLLM Lens turns the traces flowing through your gateway into findings your agents can act on. Built for agent swarms generating 200K+ traces.", + "date": "2026-10-01", + "url": "https://docs.litellm.ai/blog/litellm-lens-launch", + "tags": [ + "lens", + "agent-tracing" + ] + }, + { + "title": "How we cut LiteLLM's Redis round trips per request by 64%", + "description": "A governed request to the LiteLLM AI Gateway waited on Redis 22 times. It now waits 8 times: one pipeline per Redis backend before the model call, one after.", + "date": "2026-10-01", + "url": "https://docs.litellm.ai/blog/redis-request-pipelines", + "tags": [ + "performance", + "redis", + "proxy", + "engineering", + "ai-gateway" + ] + }, + { + "title": "How we made the LiteLLM Usage page 120x faster", + "description": "From six minutes to 3.2 seconds: how moving aggregation into Postgres made LiteLLM's Usage page roughly 120x faster in our benchmark.", + "date": "2026-09-30", + "url": "https://docs.litellm.ai/blog/usage-page-bounded-loading", + "tags": [ + "performance", + "admin-ui", + "postgres", + "engineering", + "ai-gateway" + ] + }, + { + "title": "Day 0 Support: GPT-6.1 Sol", + "description": "Day 0 support for GPT-6.1 Sol on LiteLLM, with cached input at half GPT-6 Sol's price.", + "date": "2026-09-29", + "url": "https://docs.litellm.ai/blog/gpt_6_1_sol", + "tags": [ + "openai", + "gpt-6", + "gpt-6.1-sol", + "completion", + "day 0 support", + "card_title:GPT-6.1 Sol", + "card_summary:GPT-6 Sol successor with cached input at half GPT-6 Sol's price, same request family and params." + ] + }, + { + "title": "Learn the LiteLLM gateway with a guided course", + "description": "Follow a request through the LiteLLM gateway, Router, and SDK. A guided course for teams deploying the gateway and contributors making code changes.", + "date": "2026-09-28", + "url": "https://docs.litellm.ai/blog/litellm-gateway-course", + "tags": [ + "proxy", + "product" + ] + }, + { + "title": "Day 0 Support: Claude Sonnet 5.5", + "description": "Day 0 support for Claude Sonnet 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.", + "date": "2026-09-28", + "url": "https://docs.litellm.ai/blog/claude_sonnet_5_5", + "tags": [ + "anthropic", + "claude", + "sonnet 5.5", + "day 0 support", + "card_title:Claude Sonnet 5.5", + "card_summary:Same $2 / $10 price as Sonnet 5, 30%+ faster, 70.6% on Terminal-Bench 4.0 (Sonnet 5: 10.3%), 1M context." + ] + }, + { + "title": "Introducing the LiteLLM ROI Calculator", + "description": "Compare each engineer's LiteLLM gateway spend with the estimated effort of their merged pull requests. Open source and self-hosted.", + "date": "2026-09-26", + "url": "https://docs.litellm.ai/blog/litellm-roi-calculator", + "tags": [ + "cost", + "product", + "announcement" + ] + }, + { + "title": "Introducing LiteAgents", + "description": "Switch agent harnesses without rewriting your agent. Keep your tools and model configuration, use native harness controls, and add Temporal when you need durable runs.", + "date": "2026-09-25", + "url": "https://docs.litellm.ai/blog/liteagents-sdk", + "tags": [ + "agents", + "sdk" + ] + }, + { + "title": "September Townhall Updates: 583 Bug Fixes, OCR on Rust, and 83.5% Coverage", + "description": "A recap of the September LiteLLM town hall: security and stability updates, OCR running on Rust by default, test coverage past the 80% target, and new Fusion and liteagents launches.", + "date": "2026-09-24", + "url": "https://docs.litellm.ai/blog/september-townhall-updates", + "tags": [ + "townhall", + "reliability", + "product" + ] + }, + { + "title": "Day 0 support: Gemini 3.8 Flash TTS and Flash-Lite TTS", + "description": "Day 0 support for Gemini 3.8 Flash TTS and Flash-Lite TTS on LiteLLM's /v1/audio/speech, with launch pricing tracked through 2026.", + "date": "2026-09-23", + "url": "https://docs.litellm.ai/blog/gemini_3_8_flash_tts", + "tags": [ + "gemini", + "gemini-3.8-flash-tts", + "text to speech", + "day 0 support", + "card_title:Gemini 3.8 Flash TTS and Flash-Lite TTS", + "card_summary:Gemini speech models on /v1/audio/speech, Flash and Flash-Lite, with launch pricing tracked through 2026." + ] + }, + { + "title": "Introducing LiteAdmin MCP", + "description": "Give your agent tools to create keys, add models, and manage budgets with LiteAdmin MCP. Use the same connector through LiteAdmin, our Slack admin agent.", + "date": "2026-09-23", + "url": "https://docs.litellm.ai/blog/litellm-admin-mcp", + "tags": [ + "mcp", + "agents", + "ai-gateway" + ] + }, + { + "title": "Day 0 Support: GPT-6 Sol and GPT-6 Luna", + "description": "Day 0 support for GPT-6 Sol and GPT-6 Luna on LiteLLM, at half the price of their GPT-5.6 counterparts.", + "date": "2026-09-22", + "url": "https://docs.litellm.ai/blog/gpt_6_sol_luna", + "tags": [ + "openai", + "gpt-6", + "gpt-6-sol", + "gpt-6-luna", + "completion", + "day 0 support", + "card_title:GPT-6 Sol and GPT-6 Luna", + "card_summary:Two GPT-6 variants at half the price of their GPT-5.6 counterparts, routed as the GPT-5 request family." + ] + }, + { + "title": "Day 0 Support: Claude Opus 5.5", + "description": "Day 0 support for Claude Opus 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.", + "date": "2026-09-22", + "url": "https://docs.litellm.ai/blog/claude_opus_5_5", + "tags": [ + "anthropic", + "claude", + "opus 5.5", + "day 0 support", + "card_title:Claude Opus 5.5", + "card_summary:20% cheaper than Opus 5 at $4 / $20 per 1M, 30% faster output, Terminal-Bench 4.0 66.4%, thinking always on." + ] + }, + { + "title": "Getting Started with Fireworks AI on LiteLLM", + "description": "A beginner tutorial: run an open model on Fireworks AI behind a LiteLLM gateway, call it with the OpenAI SDK you already use, then add a second model and a fallback.", + "date": "2026-09-22", + "url": "https://docs.litellm.ai/blog/fireworks-getting-started", + "tags": [ + "tutorial", + "providers", + "fireworks", + "getting-started" + ] + }, + { + "title": "Day 0 Support: Xiaomi MiMo V2.6", + "description": "Day 0 support for Xiaomi MiMo V2.6 Pro and Flash on LiteLLM, priced on the native route for the first time.", + "date": "2026-09-22", + "url": "https://docs.litellm.ai/blog/mimo_v2_6", + "tags": [ + "xiaomi", + "mimo", + "mimo-v2.6", + "day 0 support", + "card_title:Xiaomi MiMo V2.6", + "card_summary:Xiaomi's Pro ($0.435 / $0.87 per 1M) and Flash ($0.14 / $0.28), same rates as V2.5, priced on the native route." + ] + }, + { + "title": "Claude Code server-side auto mode through LiteLLM", + "description": "Anthropic is moving Claude Code auto mode's safety classifier server-side. LiteLLM's native /v1/messages route now forwards the safeguards contract to the Anthropic API, Bedrock InvokeModel, Bedrock Mantle and Vertex AI. Here is the contract, what changed, which releases carry it, and how to verify it.", + "date": "2026-09-21", + "url": "https://docs.litellm.ai/blog/claude-code-server-side-auto-mode", + "tags": [ + "announcement", + "claude-code", + "anthropic", + "ai-gateway" + ] + }, + { + "title": "Day 0 Support: Grok 4.7", + "description": "Day 0 support for Grok 4.7 on LiteLLM, at the same price as Grok 4.6.", + "date": "2026-09-21", + "url": "https://docs.litellm.ai/blog/grok_4_7", + "tags": [ + "xai", + "grok", + "grok-4.7", + "day 0 support", + "card_title:Grok 4.7", + "card_summary:SpaceXAI's update to Grok 4.6 at the same price and limits, better at checking its own work and long context." + ] + }, + { + "title": "JEV Classifier: 5.43x as Fast as Haiku, 96% Lower Cost", + "description": "JEV classified requests 5.43x as fast as Haiku by median latency in our AI Gateway benchmark. Explore the setup, cost savings and methodology.", + "date": "2026-09-20", + "url": "https://docs.litellm.ai/blog/jev-auto-router-benchmark", + "tags": [ + "routing", + "complexity-router", + "engineering", + "ai-gateway" + ] + }, + { + "title": "TypeSafe Jev on LiteLLM", + "description": "TypeSafe AI's Jev lands in LiteLLM v1.103.0-rc: call it through the proxy with logging and cost tracking.", + "date": "2026-09-20", + "url": "https://docs.litellm.ai/blog/typesafe_jev", + "tags": [ + "typesafe", + "jev", + "product" + ] } ] } diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index c82d25c9425..af084a464ee 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -620,7 +620,7 @@ class DualCache(BaseCache): increment_list: list["RedisPipelineIncrementOperation"], local_only: bool = False, parent_otel_span: Span | None = None, - **kwargs, + **kwargs: object, # kwargs-ok: accepted and unused, as the cache interface's other calls take them ) -> list[float] | None: result: list[float] | None = None try: diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 76d17ea9693..f6879c313de 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,7 +11,7 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse @@ -20,15 +20,15 @@ if TYPE_CHECKING: class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): - model: str - messages: list - optional_params: dict - litellm_params: dict - headers: dict - model_response: "ModelResponse" - logging_obj: "LiteLLMLoggingObj" - custom_llm_provider: str - encoding: object + model: ReadOnly[str] + messages: ReadOnly[list] + optional_params: ReadOnly[dict[str, object]] + litellm_params: ReadOnly[dict[str, object]] + headers: ReadOnly[dict] + model_response: ReadOnly["ModelResponse"] + logging_obj: ReadOnly["LiteLLMLoggingObj"] + custom_llm_provider: ReadOnly[str] + encoding: ReadOnly[object] def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: @@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) request_data: Final = self.transformation_handler.transform_request( model=model, @@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) try: request_data: Final = self.transformation_handler.transform_request( diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 0a500173c5b..8e6341805db 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -34,7 +34,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( replay_items, requested_phase, ) -from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_chat_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, @@ -1166,10 +1166,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): optional_params = self._extract_extra_body_params(optional_params) provider: Final = _served_provider(litellm_params) base_model: Final = litellm_params.get("base_model") - supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or ( - isinstance(base_model, str) - and bool(base_model) - and supports_openai_prompt_cache_breakpoint(base_model, provider) + model_info: Final = routed_model_info(litellm_params.get("model_info")) + named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,) + supports_prompt_cache_breakpoint: Final = any( + supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( messages, @@ -1180,11 +1180,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): include_namespace=True, replay_openai_history=provider == "openai", ) - # OpenAI's Responses API rejects an empty input. For a system-only - # request, carry the system message as a system-role input item instead - # of instructions, mirroring how non-string system content is already - # handled in convert_chat_completion_messages_to_responses_api. - is_system_only_request: Final = not converted_input_items and converted_instructions is not None + # Keep the system text in input instead of instructions when the Responses API needs it there. + # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions. + typed_optional_params: Final = cast("Mapping[str, object]", optional_params) # cast-ok: untyped dict + system_goes_in_input: Final = converted_instructions is not None and ( + not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object" + ) target_input_items: Final = ( _without_audio_input_parts(converted_input_items) if _drops_audio_input(model, litellm_params) @@ -1196,12 +1197,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "type": "message", "role": "system", "content": [{"type": "input_text", "text": converted_instructions}], - } + }, + *target_input_items, ] - if is_system_only_request + if system_goes_in_input else target_input_items ) - instructions: Final = None if is_system_only_request else converted_instructions + instructions: Final = None if system_goes_in_input else converted_instructions # Build responses API request using the reverse transformation logic responses_api_request: Final = ResponsesAPIOptionalRequestParams() @@ -1235,10 +1237,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): api_model: Final = model - from litellm.types.utils import CallTypes - - setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) - sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params) request_data: Final = { diff --git a/litellm/constants.py b/litellm/constants.py index 14a2a9f9eca..50e090eabaf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -135,6 +135,7 @@ ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10 +DEFAULT_MAX_REQUEST_HEADER_SIZE: Final = 16 * 1024 budget_reservation_disabled_info_emitted = False DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1) DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512) @@ -451,6 +452,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset( {"response.completed", "response.incomplete", "response.failed"} ) REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged" +REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally" # SSL/TLS cipher configuration for faster handshakes # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones @@ -971,7 +973,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "languages", + "keywords", "response_format", + "stream", "timestamp_granularities", ] @@ -1088,6 +1093,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api.scx.ai/v1", "https://api.prisminference.com/v1", "https://api.reka.ai/v1", + "https://inference.runanywhere.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", "https://api.openinfer.ai/v1", "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", @@ -1175,6 +1181,7 @@ openai_compatible_providers: Final[list[str]] = [ "sail", "reka", "streamlake", + "wally", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) @@ -1666,6 +1673,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" @@ -2063,6 +2073,7 @@ SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000 +SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS: Final = 30 SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE: Final = 100 # Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated # callers from forcing a DB query per request for unknown names, while bounding diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3c028278b2d..3910a554905 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -23,9 +23,14 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount -from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type +from litellm.litellm_core_utils.decisions_pricing import ( + decisions_pricing_fields, + decisions_request_price, + is_decisions_call_type, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -44,6 +49,9 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( generic_cost_per_token, get_batch_cost_rates, get_billable_input_tokens, + get_cost_per_unit, + get_fedramp_uplift_multiplier, + get_modality_token_cost_key, get_regional_uplift_multiplier, get_service_tier_cost_key, get_token_type_cost_breakdown, @@ -320,12 +328,41 @@ def _get_additional_costs( return None -def _transcription_usage_has_token_details( +_TranscriptionRateKey: TypeAlias = Literal[ + "input_cost_per_second", + "output_cost_per_second", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + "input_cost_per_token", + "output_cost_per_token", +] +_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", +) +_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = ( + "input_cost_per_second", + "output_cost_per_second", +) + + +def _transcription_uses_token_pricing( + model: str, + custom_llm_provider: str | None, usage_block: Usage | None, ) -> bool: if usage_block is None: return False + model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + if ( + _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None + and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None + ): + return False + prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0 completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0 prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None) @@ -339,13 +376,38 @@ def _transcription_usage_has_token_details( return (prompt_tokens_val > 0) or (completion_tokens_val > 0) -def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool: - if model is None or custom_llm_provider != LlmProviders.OPENAI.value: +def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool: + if pricing_model is None or custom_llm_provider is None: return False + transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider) + transcription_metadata: Final = ( + as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None + ) + return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1 + + +def _transcription_duration_for_cost( + completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None +) -> float: + hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {} + measured_duration: Final = normalized_audio_duration_seconds( + hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration")) + ) + if measured_duration is not None and measured_duration > 0: + return measured_duration + provider_duration: Final = _get_transcription_usage_duration(completion_response) + if ( + provider_duration is not None + and provider_duration > 0 + and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model)) + ): + return provider_duration + if model is None or custom_llm_provider != LlmProviders.OPENAI.value: + return 0.0 transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI ) - return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) + return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"] @@ -383,6 +445,14 @@ def _per_second_pricing_cost( model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None + return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds) + + +def _per_second_pricing_cost_from_info( + model_info: ModelInfoBase, + response_time_ms: float | None, + audio_seconds: float = 0.0, +) -> tuple[float, float] | None: if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): return None cost_per_second: Final = model_info.get("cost_per_second") @@ -405,7 +475,7 @@ def _per_second_pricing_cost( ) verbose_logger.debug( "For model=%s - cost_per_second: %s; response time: %s", - model, + model_info.get("key"), resolved_cost_per_second, response_time_ms, ) @@ -428,6 +498,7 @@ def cost_per_token( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, ### NUMBER OF QUERIES ### number_of_queries: int | None = None, ### USAGE OBJECT ### @@ -539,6 +610,26 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] + if requested_model_info_override is not None: + per_second: Final = _per_second_pricing_cost_from_info( + requested_model_info_override, response_time_ms, audio_transcription_file_duration + ) + if per_second is not None: + return per_second + if custom_llm_provider == "anthropic": + return anthropic_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier, model_info=requested_model_info_override + ) + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider=custom_llm_provider or "", + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + model_info=requested_model_info_override, + ) + model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -605,6 +696,9 @@ def cost_per_token( decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) + request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides) + if request_price is not None: + return request_price, 0.0 if decisions_overrides: return generic_cost_per_token( model=model, @@ -694,7 +788,7 @@ def cost_per_token( vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": - if _transcription_usage_has_token_details(usage_block): + if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block): return generic_cost_per_token( model=model, usage=usage_block, @@ -1040,20 +1134,19 @@ def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_nam return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name) -def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool: - rule_info: Final = unmapped_model_generalization(model, custom_llm_provider) - return rule_info is not None and _cost_map_entry_prices_anything(rule_info) +def declared_cost_map_pricing(key: str, model: str, custom_llm_provider: str | None) -> Mapping[str, object]: + entry: Final = _raw_cost_map_entry(key) + if entry is None: + return unmapped_model_generalization(model, custom_llm_provider) or MappingProxyType({}) + fill_missing: Final = mapped_model_fill_missing(key, entry) + return entry if fill_missing is None else MappingProxyType({**fill_missing, **entry}) def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool: key: Final = cached_cost_map_key(model, custom_llm_provider) - if key is None: - return False - entry: Final = _raw_cost_map_entry(key) - if entry is None: - return _rule_declares_rates(model, custom_llm_provider) - fill_missing: Final = mapped_model_fill_missing(key, entry) - return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry}) + return key is not None and _cost_map_entry_prices_anything( + declared_cost_map_pricing(key, model, custom_llm_provider) + ) def _served_or_priced_requested_model( @@ -1276,6 +1369,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: ) +def _get_transcription_usage_duration(completion_response: object) -> float | None: + response_fields: Final = as_str_mapping(completion_response) + usage_object: Final = ( + response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage") + ) + usage_fields: Final = as_str_mapping(usage_object) + usage_type: Final = ( + usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type") + ) + if usage_type != "duration": + return None + seconds: Final = ( + usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds") + ) + return normalized_audio_duration_seconds(seconds) + + def _extraction_usage(usage: ExtractionUsage) -> Usage: return Usage( prompt_tokens=usage.input_tokens, @@ -1669,6 +1779,7 @@ def completion_cost( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, optional_params: dict | None = None, custom_pricing: bool | None = None, base_model: str | None = None, @@ -1722,6 +1833,7 @@ def completion_cost( quality=quality, n=n, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, custom_cost_per_second=custom_cost_per_second, optional_params=optional_params, custom_pricing=custom_pricing, @@ -2001,20 +2113,8 @@ def completion_cost( prompt, ) elif call_type in _TRANSCRIPTION_CALL_TYPES: - # Check _hidden_params first (duration stored there to - # avoid polluting the response body), then fall back to - # the response attribute (for verbose_json responses that - # naturally include duration from the provider). - _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) or ( - completion_response.usage.seconds - if isinstance(completion_response, TranscriptionResponse) - and isinstance(completion_response.usage, TranscriptionUsageDurationObject) - and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 + audio_transcription_file_duration = _transcription_duration_for_cost( + completion_response, model, requested_model, custom_llm_provider ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -2124,6 +2224,7 @@ def completion_cost( litellm_logging_obj=litellm_logging_obj, custom_pricing_model=selected_model if custom_pricing else None, base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None), + deployment_models=(base_model, requested_model), ) elif call_type == _MCP_CALL_TYPE: from litellm.proxy.mcp_server.cost_calculator import ( @@ -2207,6 +2308,7 @@ def completion_cost( region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, prompt_characters=prompt_characters, completion_characters=completion_characters, cache_creation_input_tokens=cache_creation_input_tokens, @@ -2352,18 +2454,20 @@ def completion_cost( raise e -def get_response_cost_from_hidden_params( - hidden_params: dict | BaseModel, -) -> float | None: - if isinstance(hidden_params, BaseModel): - _hidden_params_dict = hidden_params.model_dump() - else: - _hidden_params_dict = hidden_params +_ADDITIONAL_HEADERS: Final = TypeAdapter(Mapping[str, object]) - additional_headers: Final = _hidden_params_dict.get("additional_headers", {}) - if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers: - return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"]) - return None + +def get_response_cost_from_hidden_params( + hidden_params: Mapping[str, object] | BaseModel, +) -> float | None: + params: Final[Mapping[str, object]] = ( + hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params + ) + additional_headers: Final = params.get("additional_headers") + if not isinstance(additional_headers, Mapping): + return None + headers: Final = _ADDITIONAL_HEADERS.validate_python(additional_headers) + return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost")) def _billable_header_cost(value: object) -> float | None: @@ -2439,6 +2543,7 @@ def response_cost_calculator( region_name: str | None = None, custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, ) -> float: """ Returns @@ -2495,6 +2600,7 @@ def response_cost_calculator( custom_pricing=custom_pricing, base_model=base_model, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, @@ -3089,8 +3195,12 @@ def _batch_rate( "input_cost_per_video_token_batches", ], fallback: float, + *, + prompt_tokens: int = 0, ) -> float: - rate: Final = model_info.get(key) + context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens) + context_rate: Final = get_cost_per_unit(model_info, context_key, None) + rate: Final = model_info.get(key) if context_rate is None else context_rate return fallback if rate is None else rate @@ -3208,7 +3318,9 @@ def batch_cost_calculator( text_tokens: Final = usage.completion_tokens - image_tokens total_completion_cost = text_tokens * text_rate + image_tokens * image_rate - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) + uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_fedramp_uplift_multiplier( + model_info, data_residency + ) if uplift != 1.0: total_prompt_cost *= uplift total_completion_cost *= uplift @@ -3217,15 +3329,35 @@ def batch_cost_calculator( return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift +def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float | None: + return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None) + + +def _cached_audio_batch_fallback( + model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int +) -> float: + if batch_cache_read_rate is not None: + return batch_cache_read_rate + standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens) + if standard_audio_rate is not None: + return standard_audio_rate + return 0.0 if cache_read_rate is None else cache_read_rate + + def _batch_prompt_cost( usage: Usage, model_info: ModelInfo, input_rate: float, - cache_read_rate: float | None, + batch_cache_read_rate: float | None, cache_creation_rate: float | None, cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: details: Final = parse_prompt_tokens_details(usage) + cache_read_rate: Final = ( + _standard_rate(model_info, "cache_read_input_token_cost", usage.prompt_tokens) + if batch_cache_read_rate is None + else batch_cache_read_rate + ) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) @@ -3240,12 +3372,17 @@ def _batch_prompt_cost( ) return ( text_tokens * input_rate - + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + + audio_tokens + * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + cached_audio_tokens - * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + * _batch_rate( + model_info, + "cache_read_input_audio_token_cost_batches", + _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens), + ) + ( calculate_cache_writing_cost( written_tokens, @@ -3402,8 +3539,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): def create_logging_realtime_object( usage: Usage, results: OpenAIRealtimeStreamList ) -> LiteLLMRealtimeStreamLoggingObject: + translated_input_seconds, _ = _translation_session_seconds(results) return LiteLLMRealtimeStreamLoggingObject( - usage=usage, + usage=usage.model_copy(update={"seconds": translated_input_seconds}) + if translated_input_seconds > 0 + else usage, results=results, ) @@ -3488,6 +3628,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed" class _LiveResponsePayload(BaseModel): @@ -3710,6 +3851,7 @@ def handle_realtime_stream_cost_calculation( litellm_logging_obj: LitellmLoggingObject | None = None, custom_pricing_model: str | None = None, base_pricing_model: str | None = None, + deployment_models: Sequence[str | None] = (), ) -> float: """ Handles the cost calculation for realtime stream responses. @@ -3754,6 +3896,13 @@ def handle_realtime_stream_cost_calculation( if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) else 0.0 ) + translation_cost: Final = handle_realtime_translation_cost_calculation( + results=results, + custom_llm_provider=custom_llm_provider, + litellm_model_name=litellm_model_name, + deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models), + session_model=received_model, + ) live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"}) live_session_started: Final = any(result["type"] == "session.started" for result in results) live_session_closed: Final = any(result["type"] == "session.closed" for result in results) @@ -3810,12 +3959,19 @@ def handle_realtime_stream_cost_calculation( input_cost: Final = input_cost_per_token + live_response_input_cost output_cost: Final = output_cost_per_token + live_response_output_cost total_cost: Final = ( - input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost + input_cost + + output_cost + + transcription_cost + + translation_cost + + live_voice_cost + + audio_duration_cost + + text_message_cost ) additional_costs: Final = tuple( (cost_name, cost) for cost_name, cost in ( ("transcription_cost", transcription_cost), + ("translation_cost", translation_cost), ("live_voice_cost", live_voice_cost), ("audio_duration_cost", audio_duration_cost), ("text_message_cost", text_message_cost), @@ -3836,6 +3992,55 @@ def handle_realtime_stream_cost_calculation( return total_cost +def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]: + usage_events: Final = tuple( + usage + for result in results + if (event := as_str_mapping(result)) is not None + and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + and (usage := as_str_mapping(event.get("usage"))) is not None + ) + input_seconds: Final = sum( + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None + ) + output_seconds: Final = sum( + output_duration + for usage in usage_events + if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None + ) + return float(input_seconds), float(output_seconds) + + +def _translation_per_second_rates( + models: Sequence[str | None], custom_llm_provider: str +) -> tuple[float | None, float | None]: + model_infos: Final = tuple( + _lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None + ) + return ( + _first_declared_transcription_rate(("input_cost_per_second",), model_infos), + _first_declared_transcription_rate(("output_cost_per_second",), model_infos), + ) + + +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, + deployment_pricing_models: Sequence[str | None] = (), + session_model: str | None = None, +) -> float: + input_seconds, output_seconds = _translation_session_seconds(results) + if input_seconds <= 0 and output_seconds <= 0: + return 0.0 + input_rate, output_rate = _translation_per_second_rates( + (*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider + ) + return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0) + + def handle_realtime_transcription_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, @@ -3912,17 +4117,16 @@ def _get_transcription_model_name_from_results( ) -_TranscriptionRateKey: TypeAlias = Literal[ - "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" -] - - -def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None: +def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None: match key: case "input_cost_per_second": return info.get("input_cost_per_second") + case "output_cost_per_second": + return info.get("output_cost_per_second") case "input_cost_per_audio_token": return info.get("input_cost_per_audio_token") + case "output_cost_per_audio_token": + return info.get("output_cost_per_audio_token") case "input_cost_per_token": return info.get("input_cost_per_token") case "output_cost_per_token": @@ -3931,7 +4135,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non assert_never(key) -def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: +def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: @@ -3949,15 +4153,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript ) -def _transcription_rate( - keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None -) -> float: - rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base)) - return next((rate for rate in rates if rate is not None), 0.0) +def _first_declared_transcription_rate( + keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None] +) -> float | None: + rates: Final = (_declared_transcription_rate(info, keys) for info in infos) + return next((rate for rate in rates if rate is not None), None) + + +def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float: + return _first_declared_transcription_rate(keys, infos) or 0.0 def _transcription_usage_cost( - usage: dict, + usage: Mapping[str, object], model_info: ModelInfo | None, override_info: ModelInfo | None = None, ) -> float: @@ -3966,13 +4174,13 @@ def _transcription_usage_cost( usage_type: Final = usage.get("type") if usage_type == "duration": - seconds: Final = usage.get("seconds") or 0.0 + seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0 return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info) if usage_type == "tokens": - input_token_details: Final = usage.get("input_token_details") or {} - audio_tokens: Final = input_token_details.get("audio_tokens") or 0 - text_tokens: Final = input_token_details.get("text_tokens") or 0 - output_tokens: Final = usage.get("output_tokens") or 0 + input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {} + audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0 + text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0 + output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0 audio_cost: Final = float(audio_tokens) * _transcription_rate( ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info ) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 871fa8227ea..9c0e51e14ac 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -13,6 +13,7 @@ from typing_extensions import assert_never import litellm from litellm.cost_calculator import cost_map_key_prices_anything from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -555,7 +556,7 @@ def prepare_call( resolved_api_base, static_api_key = provider_config.resolve_connection( dynamic_api_base or call_api_base, dynamic_api_key or api_key ) - except ValueError as error: + except (ValueError, httpx.InvalidURL) as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error if resolved_api_base is None: raise litellm.BadRequestError( @@ -613,6 +614,9 @@ def prepare_call( litellm_params={ "litellm_call_id": kwargs.get("litellm_call_id"), "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + "input_cost_per_query": ( + kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None + ), }, custom_llm_provider=provider, ) @@ -787,7 +791,9 @@ def _formatted_response( case DecisionsRequestBody(): return ir_to_systemone_response(caller_response, caller_request) case OpenAIDecisionRequestBody(): - return ir_to_openai_response(caller_response, caller_request, call.requested_model) + return ir_to_openai_response( + caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields + ) case _: assert_never(call.request) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 12425c45b01..6662cf4a2c2 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -867,6 +867,7 @@ class BadGatewayError(openai.APIStatusError): class InternalServerError(openai.InternalServerError): response: httpx.Response # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX response request: httpx.Request # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX request + provider_status_code: int | None def __init__( self, @@ -878,8 +879,10 @@ class InternalServerError(openai.InternalServerError): max_retries: int | None = None, num_retries: int | None = None, body: object | None = None, + provider_status_code: int | None = None, ): self.status_code = 500 + self.provider_status_code = provider_status_code self.message = f"litellm.InternalServerError: {message}" self.llm_provider = llm_provider self.model = model diff --git a/litellm/images/main.py b/litellm/images/main.py index 165543aadba..79bb07fc394 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -28,8 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.custom_llm import CustomLLM -from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params +from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() @@ -68,6 +67,10 @@ from litellm.utils import ( ) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) +_OPTIONAL_OBJECT_DICT_ADAPTER: Final = TypeAdapter[dict[str, object] | None](dict[str, object] | None) +_OPTIONAL_STRING_DICT_ADAPTER: Final = TypeAdapter[dict[str, str] | None](dict[str, str] | None) +_OBJECT_DICT_ADAPTER: Final = TypeAdapter[dict[str, object]](dict[str, object]) +_STRING_DICT_ADAPTER: Final = TypeAdapter[dict[str, str]](dict[str, str]) _PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE: Final = ( LlmProviders.DASHSCOPE, LlmProviders.QWENCLOUD, @@ -94,8 +97,8 @@ def _chatgpt_image_params(kwargs: Mapping[str, object]) -> GenericLiteLLMParams: from litellm.responses.utils import ResponsesAPIRequestUtils headers: Final = ResponsesAPIRequestUtils.merge_client_forwarded_headers( - TypeAdapter[dict[str, object] | None](dict[str, object] | None).validate_python(kwargs.get("extra_headers")), - TypeAdapter[dict[str, str] | None](dict[str, str] | None).validate_python(kwargs.get("headers")), + _OPTIONAL_OBJECT_DICT_ADAPTER.validate_python(kwargs.get("extra_headers")), + _OPTIONAL_STRING_DICT_ADAPTER.validate_python(kwargs.get("headers")), ) return GenericLiteLLMParams.model_validate({**kwargs, "extra_headers": headers}) @@ -168,7 +171,7 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse: custom_llm_provider=provider_hints.get("custom_llm_provider"), api_base=provider_hints.get("api_base"), litellm_params=( - _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python(kwargs)) + _chatgpt_image_params(_OBJECT_DICT_ADAPTER.validate_python(kwargs)) if provider_hints.get("custom_llm_provider") == "chatgpt" or (isinstance(model, str) and model.startswith("chatgpt/")) else None @@ -300,7 +303,7 @@ def image_generation( headers.update(extra_headers) model_response: ImageResponse = litellm.utils.ImageResponse() chatgpt_params: Final = ( - _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python({**kwargs, "api_key": api_key})) + _chatgpt_image_params(_OBJECT_DICT_ADAPTER.validate_python({**kwargs, "api_key": api_key})) if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/")) else None ) @@ -583,7 +586,7 @@ def image_generation( _chatgpt_image_headers( model, chatgpt_params.model_copy(update={"api_key": dynamic_api_key or api_key}), - TypeAdapter(dict[str, str]).validate_python(headers), + _STRING_DICT_ADAPTER.validate_python(headers), ) ) if chatgpt_params is not None @@ -608,10 +611,7 @@ def image_generation( ) elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider # Get the Custom Handler - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -915,10 +915,7 @@ def image_edit( # Check for custom provider if custom_llm_provider in litellm._custom_providers: - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -995,10 +992,15 @@ def image_edit( or custom_llm_provider == "azure" or custom_llm_provider in litellm.openai_compatible_providers ): + dropped_form_params: Final = header_auth_params(custom_llm_provider) image_edit_request_params.update( flatten_form_field_values( - non_default_params, - extra_body if isinstance(extra_body, dict) else None, + {key: value for key, value in non_default_params.items() if key not in dropped_form_params}, + ( + {key: value for key, value in extra_body.items() if key not in dropped_form_params} + if isinstance(extra_body, dict) + else None + ), ) ) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 01be455e229..4e473c2f3f0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -9,24 +9,29 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ +import bisect import copy import itertools import os import re from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast from urllib.parse import urlparse from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.prompt_templates.common_utils import ( + PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_as_cache_control, with_prompt_cache_breakpoint, + without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.factory import ( tool_call_ids_answered_by_client, @@ -85,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object]) _ATTRIBUTE_VALUE: Final = TypeAdapter(object) +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues +CostMap: TypeAlias = Mapping[str, Mapping[str, object]] + + +def routed_model_info(value: object) -> Mapping[str, object] | None: + try: + return _MODEL_INFO_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap: + import litellm + + if model_cost is not None: + return model_cost + return cast(CostMap, litellm.model_cost) # cast-ok: the live map holds catalog entries + + +def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None: + declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") + return declared if isinstance(declared, bool) else None + + +def routed_cost_map() -> CostMap: + from litellm.utils import catalog_with_runtime_registrations + + return catalog_with_runtime_registrations() + + +def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None: + return None if model_info is None else routed_cost_map() def _validated_object_mapping(value: object) -> dict[object, object] | None: @@ -295,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object: return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) +def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params) + + def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]: if not isinstance(value, (list, tuple)): return () @@ -314,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non return provider -def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: - return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True +def supports_openai_prompt_cache_breakpoint( + model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True -def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: +def openai_prompt_cache_breakpoint_support( + model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool | None: + declared: Final = declared_prompt_cache_breakpoint(model_info) + if declared is not None: + return declared + model_cost: Final = _catalog_unless_unrouted(model_info) hosted_flag: Final = ( None if custom_llm_provider is None - else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model) + else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost) ) if hosted_flag is not None: return hosted_flag - model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) + model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost) if model_map_flag is not None: return model_map_flag fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) @@ -347,13 +398,18 @@ class _DeclaredBaseModels(BaseModel): return tuple(name for name in candidates if isinstance(name, str) and name) +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info) + + def deployment_prompt_cache_breakpoint_support( model: str, custom_llm_provider: str, litellm_params: object ) -> bool | None: """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first.""" declared: Final = _DeclaredBaseModels.model_validate(litellm_params) + model_info: Final = routed_model_info(declared.model_info) supports: Final = ( - openai_prompt_cache_breakpoint_support(name, custom_llm_provider) + openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info) for name in dict.fromkeys((*declared.names(), model)) ) return next((support for support in supports if support is not None), None) @@ -370,11 +426,10 @@ def _named_model_breakpoint_support(model: str) -> bool | None: return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION -def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: - import litellm - - entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) - flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) +def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None: + cost_map: Final = _live_cost_map_unless(model_cost) + entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1])) + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping)) return next((flag is True for flag in flags if flag is not None), None) @@ -390,7 +445,7 @@ def _hosted_openai_dialect_flag( model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None], - model_cost: Mapping[str, Mapping[str, object]] | None = None, + model_cost: CostMap | None = None, ) -> bool | None: """ Explicit opt-in for an OpenAI-shaped deployment served by another provider. @@ -407,13 +462,7 @@ def _hosted_openai_dialect_flag( row of the same name, so an exact entry only speaks for a deployment when it is keyed for that deployment's provider. """ - import litellm - - cost_map: Final = ( - cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries - if model_cost is None - else model_cost - ) + cost_map: Final = _live_cost_map_unless(model_cost) exact_entry: Final = cost_map.get(model) exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)): @@ -423,10 +472,16 @@ def _hosted_openai_dialect_flag( return None if isinstance(exact_entry, Mapping) and exact_provider == provider: return _hosted_entry_flag(exact_entry, lambda: provider) + entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider)) + flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) + return next((flag for flag in flags if flag is not None), None) + + +def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]: from litellm.utils import get_potential_model_names names: Final = get_potential_model_names(model, provider) - candidates: Final = ( + return ( names["combined_model_name"], names["region_free_combined_model_name"], names["split_model"], @@ -434,9 +489,60 @@ def _hosted_openai_dialect_flag( names["stripped_model_name"], names["provider_prefixed_model_name"], ) - entries: Final = (cost_map.get(candidate) for candidate in candidates) - flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) - return next((flag for flag in flags if flag is not None), None) + + +def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool: + entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider))) + return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries) + + +_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse")) +_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle")) + + +def _declared_hosted_flag( + declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object +) -> bool | None: + """ + A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the + request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes + a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep + the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass + the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost + map, or the deployment's own ``litellm_provider``, lists under that provider. + """ + if provider is None or provider == "openai": + return None + return ( + declared + and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT + and ( + provider in _PROVIDERS_HOSTING_OPENAIS_API + or deployment_provider == provider + or _cost_map_lists_model_for_provider(model, provider, model_cost) + ) + and _sends_openai_shaped_chat(model, provider) + ) + + +@lru_cache(maxsize=512) +def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + from litellm.llms.openai.openai import OpenAIConfig + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + try: + served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=served_model, provider=LlmProviders(provider) + ) + except (BadRequestError, ValueError): + return False + return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig)) def targets_openai_api(api_base: object) -> bool: @@ -456,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool: return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS) +def _carries_cache_control(block: object) -> bool: + return _attribute_or_key(block, "cache_control") is not None + + def _attribute_or_key(value: object, key: str) -> object | None: if hasattr(value, key): return _ATTRIBUTE_VALUE.validate_python(getattr(value, key)) @@ -535,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig) +def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool: + return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages( + model, custom_llm_provider + ) + + def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None: provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP) return provider if isinstance(provider, str) else None @@ -578,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES +def _carries_prompt_cache_breakpoint(block: object) -> bool: + return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None + + def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: return message.get("role") != "assistant" +def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and any( + _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or () + ) + + +def _part_without_prompt_cache_breakpoint(part: object) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return { + key: value + for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() + if key != "prompt_cache_breakpoint" + } + + +def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues: + if not _tool_result_carries_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + return cast( # cast-ok: the same tool message with its part marks removed + AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]} + ) + + +def _created_cache_control(original: object, translated: object) -> bool: + return ( + _attribute_or_key(original, "cache_control") is None + and _attribute_or_key(translated, "cache_control") is not None + ) + + +def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]: + if translated is original: + return () + original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or [] + translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or [] + return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts)) + + +def _part_without_cache_control(part: object) -> object: + return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"} + + +def _message_dropping_created_cache_controls_between( + message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int +) -> AllMessageValues: + if not any(created): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before) + return cast( # cast-ok: the same message with its middle translated marks removed + AllMessageValues, + { + **message, + "content": [ + _part_without_cache_control(part) + if is_created and first_dropped <= ordinal < first_kept_after + else part + for part, is_created, ordinal in zip(parts, created, ordinals) + ], + }, + ) + + # Set by a caller whose message list is not the one that goes upstream -- today the # Responses API layer, whose `instructions` only becomes a system message further down. # Tells this hook to hand role-targeted points to the pass holding the final messages @@ -667,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return model, messages, non_default_params - # Create a deep copy of messages to avoid modifying the original list - processed_messages = copy.deepcopy(messages) - message_points: Final = tuple( cast(CacheControlMessageInjectionPoint, point) for point in injection_points @@ -677,18 +859,26 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message") - custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0]) + request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict + custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0]) stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + model_info: Final = routed_model_info(request_params.get("model_info")) openai_dialect: Final = ( stamped_dialect if isinstance(stamped_dialect, bool) else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( model, custom_llm_provider, - non_default_params.get("api_base") or non_default_params.get("base_url"), - non_default_params.get("prompt_cache_options"), + request_params.get("api_base") or request_params.get("base_url"), + request_params.get("prompt_cache_options"), + model_info, ) ) + processed_messages = copy.deepcopy( + [without_prompt_cache_breakpoints(message) for message in messages] + if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider) + else messages + ) # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint @@ -757,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object = None, prompt_cache_options: object = None, + model_info: Mapping[str, object] | None = None, ) -> bool: if model is None: return False - hosted_flag: Final = _hosted_openai_dialect_flag( - model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base) + declared: Final = declared_prompt_cache_breakpoint(model_info) + hosted_flag: Final = ( + _hosted_openai_dialect_flag( + model, + custom_llm_provider, + lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base), + _catalog_unless_unrouted(model_info), + ) + if declared is None + else _declared_hosted_flag( + declared, + model, + custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base), + routed_cost_map(), + None if model_info is None else model_info.get("litellm_provider"), + ) ) if hosted_flag is not None: - return hosted_flag - if not supports_openai_prompt_cache_breakpoint(model): + return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider) + if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) - @staticmethod - def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool: - return ( - _hosted_openai_dialect_flag( - model, - custom_llm_provider, - AnthropicCacheControlHook._resolve_provider, - GetModelCostMap.loaded_model_cost_map(), - ) - is True - ) - @staticmethod def _resolve_provider(model: str, api_base: object = None) -> str | None: return _served_provider_for_model(model, api_base) @@ -829,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return int(wire_cache_control is not None) + tool_blocks + envelope_blocks + @staticmethod + def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]: + return tuple( + tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool) + ) + + @staticmethod + def count_chat_external_cache_breakpoints(optional_params: object) -> int: + params: Final = _validated_object_mapping(optional_params) or {} + return AnthropicCacheControlHook.count_external_cache_breakpoints( + _as_object_list(params.get("tools")), params.get("cache_control") + ) + @staticmethod def count_external_cache_breakpoints_on_messages_route( tools: Iterable[object] | None, cache_control: object, request_kwargs: object @@ -984,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return message_count + content_count + tool_call_count + @staticmethod + def count_converse_cache_points(message: AllMessageValues) -> int: + content: Final = _as_object_list(_attribute_or_key(message, "content")) + tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or [] + marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block)) + marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call)) + if message.get("role") == "tool": + return int(_carries_cache_control(message) or marked_parts > 0) + return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls + @staticmethod def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool: """Return True if the message already carries any cache_control.""" @@ -1042,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement): return renamed return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) + @staticmethod + def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues: + """Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's + ``cache_control``. Marks with no OpenAI slot are left for the caller to strip.""" + if not _may_carry_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) + if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts): + return message + return cast( # cast-ok: the same message with its part marks renamed + AllMessageValues, + {**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]}, + ) + + @staticmethod + def prompt_cache_breakpoints_dropped_from_tool_results( + messages: Sequence[AllMessageValues], drop_params: bool, model: str + ) -> tuple[AllMessageValues, ...]: + if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages): + return tuple(messages) + if not drop_params: + raise UnsupportedParamsError( + message=( + f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool " + f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model) + return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages) + + @staticmethod + def prompt_cache_breakpoints_as_capped_cache_control( + messages: Sequence[AllMessageValues], + drop_params: bool, + sent_breakpoints: Callable[[AllMessageValues], int], + external_breakpoints: Callable[[], int], + ) -> tuple[AllMessageValues, ...]: + """``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's + breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends. + Under ``drop_params`` the first and last translated marks fill the slots the client's own + ``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest + turns stay cached, and without it a request over the cap is rejected.""" + translated: Final = tuple( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params) + for message in messages + ) + if all(after is before for before, after in zip(messages, translated)): + return translated + created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated)) + created_marks: Final = sum(sum(flags) for flags in created) + if created_marks == 0: + return translated + external: Final = external_breakpoints() + breakpoints: Final = sum(map(sent_breakpoints, translated)) + external + excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS + if excess <= 0: + return translated + if not drop_params: + raise UnsupportedParamsError( + message=( + f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this " + f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} " + f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them." + ), + status_code=400, + ) + verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params") + marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0)) + + def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]: + first_dropped: Final = (kept_marks + 1) // 2 + first_kept_after: Final = created_marks - kept_marks // 2 + return tuple( + _message_dropping_created_cache_controls_between( + message, flags, before, first_dropped, first_kept_after + ) + for message, flags, before in zip(translated, created, marks_before) + ) + + def over_cap(kept_marks: int) -> bool: + return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS + + first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap) + return capped_request(max(0, first_over_cap - 1)) + @staticmethod def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: if not _may_carry_prompt_cache_breakpoint(message): @@ -1181,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object, prompt_cache_options: object, + model_info: Mapping[str, object] | None = None, ) -> Sequence[Mapping[str, object]]: """Carry onto the points what the prompt-management hook never receives. @@ -1192,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): across requests. """ with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect( - points, model, custom_llm_provider, api_base, prompt_cache_options + points, model, custom_llm_provider, api_base, prompt_cache_options, model_info ) stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect) if external_breakpoints == 0 and not stamped_earlier: @@ -1206,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object, prompt_cache_options: object, + model_info: Mapping[str, object] | None = None, ) -> Sequence[Mapping[str, object]]: - if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider): + if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint( + model, custom_llm_provider + ): return points return AnthropicCacheControlHook._stamped( points, "_litellm_openai_dialect", AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, prompt_cache_options + model, custom_llm_provider, api_base, prompt_cache_options, model_info ), ) @@ -1272,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): cache_control: object = None, request_kwargs: object = None, on_messages_route: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. @@ -1289,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True: return [] - if not supports_anthropic_cache_control(model, custom_llm_provider): + if not supports_anthropic_cache_control(model, custom_llm_provider, model_info): return [] if AnthropicCacheControlHook._request_has_cache_control( @@ -1367,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools: list | None = None, enable_prompt_caching: bool | None = None, api_base: object = None, + model_info: Mapping[str, object] | None = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -1381,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points")) if configured: - tools_keeping_marks: Final = tuple( - tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool) - ) + tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools) stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook( configured, AnthropicCacheControlHook.count_external_cache_breakpoints( @@ -1393,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider, api_base, non_default_params.get("prompt_cache_options"), + model_info, ) non_default_params["cache_control_injection_points"] = ( AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider) @@ -1409,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): enable_prompt_caching=enable_prompt_caching, cache_control=non_default_params.get("cache_control"), request_kwargs=non_default_params, + model_info=model_info, ) if points: non_default_params["cache_control_injection_points"] = points @@ -1527,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): cache_control=cache_control, request_kwargs=kwargs, on_messages_route=True, + model_info=routed_model_info(kwargs.get("model_info")), ) if model is not None else () @@ -1535,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): return messages, system openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") + model, + custom_llm_provider, + api_base, + kwargs.get("prompt_cache_options"), + routed_model_info(kwargs.get("model_info")), ) breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( @@ -1669,9 +1982,17 @@ def messages_within_cache_control_cap( Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. """ - reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( - _validated_object_list(tools), cache_control, request_kwargs + return messages_within_reserved_cache_control_cap( + messages, + AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ), ) + + +def messages_within_reserved_cache_control_cap( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) if dropped > 0: diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index 69b528a29f8..b035732afa6 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -10,6 +10,7 @@ import json import time import uuid from collections.abc import Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable from pydantic import ValidationError @@ -73,7 +74,7 @@ class CodeInterpreterLogOutput(TypedDict): class CodeInterpreterCall(TypedDict): id: str type: Literal["code_interpreter_call"] - status: Literal["completed"] + status: Literal["completed", "failed"] code: str container_id: str | None outputs: list[CodeInterpreterLogOutput] @@ -141,6 +142,39 @@ class _SupportsOutput(Protocol): _CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None] +@dataclass(frozen=True, slots=True) +class _ReadySandbox: + container: ContainerHandle + params: SandboxToolParams | None + + +@dataclass(frozen=True, slots=True) +class _SandboxUnavailable: + output: str + + +_Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable + + +@dataclass(frozen=True, slots=True) +class _ToolRun: + output: str + status: Literal["completed", "failed"] + + +_SANDBOX_UNAVAILABLE_OUTPUT: Final = "The code did not run because the code execution sandbox is unavailable." +_SANDBOX_LOST_RESULT_OUTPUT: Final = ( + "The code execution sandbox failed while running the code, so the code may have run but its result is lost." +) + + +def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNAVAILABLE_OUTPUT) -> _SandboxUnavailable: + verbose_logger.warning( + "CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error + ) + return _SandboxUnavailable(output=output) + + def _output_item_type(item: object) -> object: if isinstance(item, dict): item_mapping: Final[dict[str, object]] = item @@ -189,6 +223,14 @@ def _extract_identity(kwargs: Mapping[str, object]) -> str: return identity if isinstance(identity, str) else "" +def _bridged_from_responses(kwargs: Mapping[str, object], call_type: CallTypes | None) -> bool: + route: Final = getattr(kwargs.get("litellm_logging_obj"), "call_type", None) + return call_type in (CallTypes.completion, CallTypes.acompletion) and route in ( + CallTypes.responses, + CallTypes.aresponses, + ) + + def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None: if sandbox_tool_name is None: return None @@ -261,7 +303,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def async_pre_call_deployment_hook( self, kwargs: dict[str, object], call_type: CallTypes | None ) -> dict | None: - if not kwargs.get("_agentic_loop_depth"): + if not kwargs.get("_agentic_loop_depth") and not _bridged_from_responses(kwargs, call_type): kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None) kwargs.pop(_SANDBOX_KEY, None) self._strip_interception_metadata(kwargs) @@ -451,16 +493,16 @@ class CodeInterpreterInterceptionLogger(CustomLogger): sandbox_key: Final = self._extract_sandbox_key(kwargs) is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY)) identity: Final = _extract_identity(kwargs) if is_session else None - container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) + sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity) try: - container_id: Final = self._container_id(container) + container_id: Final = self._sandbox_container_id(sandbox) input_list: Final = self._normalize_messages(messages) code_interpreter_calls: Final[list[CodeInterpreterCall]] = [] for tool_call in tool_calls: arguments = tool_call.get("arguments", "") code = self._parse_code(arguments) - stdout = await self._run_tool_call(container=container, params=params, arguments=arguments) + tool_run = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) input_list.append( { "type": "function_call", @@ -473,22 +515,23 @@ class CodeInterpreterInterceptionLogger(CustomLogger): { "type": "function_call_output", "call_id": tool_call.get("call_id"), - "output": stdout, + "output": tool_run.output, } ) code_interpreter_calls.append( { "id": f"ci_{uuid.uuid4().hex}", "type": "code_interpreter_call", - "status": "completed", + "status": tool_run.status, "code": code, "container_id": container_id, - "outputs": ([{"type": "logs", "logs": stdout}] if stdout else []), + "outputs": ([{"type": "logs", "logs": tool_run.output}] if tool_run.output else []), } ) except Exception: await self._delete_container_for_cache_key(sandbox_key) raise + await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls) optional_params: Final = anthropic_messages_optional_request_params request_patch: Final = AgenticLoopRequestPatch( @@ -526,14 +569,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): sandbox_key: Final = self._extract_sandbox_key(kwargs) is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY)) identity: Final = _extract_identity(kwargs) if is_session else None - container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) + sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity) try: - container_id: Final = self._container_id(container) + container_id: Final = self._sandbox_container_id(sandbox) tool_results: Final = [ await self._build_chat_completion_tool_result( - container=container, - params=params, + sandbox=sandbox, tool_call=tool_call, container_id=container_id, ) @@ -544,6 +586,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): raise tool_messages: Final = [result[0] for result in tool_results] code_interpreter_calls: Final = [result[1] for result in tool_results] + await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls) request_patch: Final = AgenticLoopRequestPatch( model=model, @@ -569,9 +612,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): ) @staticmethod - def _container_id(container: ContainerHandle) -> str | None: - container_id: Final[object] = getattr(container, "id", None) - return container_id if isinstance(container_id, str) else None + def _sandbox_container_id(sandbox: _Sandbox) -> str | None: + match sandbox: + case _ReadySandbox(container=container): + container_id: Final[object] = getattr(container, "id", None) + return container_id if isinstance(container_id, str) else None + case _SandboxUnavailable(): + return None @staticmethod def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]: @@ -588,28 +635,27 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def _build_chat_completion_tool_result( self, - container: ContainerHandle, - params: SandboxToolParams | None, + sandbox: _Sandbox, tool_call: CodeExecutionToolCall, container_id: str | None, ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]: arguments: Final = tool_call.get("arguments", "") code: Final = self._parse_code(arguments) - stdout: Final = await self._run_tool_call(container=container, params=params, arguments=arguments) + tool_run: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex return ( { "role": "tool", "tool_call_id": tool_call_id, - "content": stdout, + "content": tool_run.output, }, { "id": f"ci_{uuid.uuid4().hex}", "type": "code_interpreter_call", - "status": "completed", + "status": tool_run.status, "code": code, "container_id": container_id, - "outputs": [{"type": "logs", "logs": stdout}] if stdout else [], + "outputs": [{"type": "logs", "logs": tool_run.output}] if tool_run.output else [], }, ) @@ -694,19 +740,55 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except (json.JSONDecodeError, TypeError, AttributeError): return "" - async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str: + async def _evict_sandbox_if_a_run_failed( + self, sandbox_key: str | None, sandbox: _Sandbox, calls: Sequence[CodeInterpreterCall] + ) -> None: + if not sandbox_key or not isinstance(sandbox, _ReadySandbox): + return + if not any(call["status"] == "failed" for call in calls): + return + cached: Final = self._container_cache.get(sandbox_key) + if cached is None or cached[0] is not sandbox.container: + return + self._container_cache.pop(sandbox_key) + await self._delete_container(container=cached[0], params=cached[1]) + + async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun: + match sandbox: + case _SandboxUnavailable(output=output): + return _ToolRun(output=output, status="failed") + case _ReadySandbox(container=container, params=params): + match await self._run_tool_call(container=container, params=params, arguments=arguments): + case _SandboxUnavailable(output=output): + return _ToolRun(output=output, status="failed") + case str() as output: + return _ToolRun(output=output, status="completed") + + async def _run_tool_call( + self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str + ) -> str | _SandboxUnavailable: try: code: Final = json.loads(arguments).get("code", "") if arguments else "" except (json.JSONDecodeError, TypeError): return "[invalid tool arguments: could not parse code]" - result: Final = await self._run_code(container=container, params=params, code=code) + try: + result: Final = await self._run_code(container=container, params=params, code=code) + except Exception as e: + return _sandbox_unavailable("running the code", e, _SANDBOX_LOST_RESULT_OUTPUT) if getattr(result, "error", None): error: Final = result.error message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error) return f"[execution error] {message}" return getattr(result, "stdout", "") or "" + async def _acquire_sandbox(self, cache_key: str | None, identity: str | None) -> _Sandbox: + try: + container, params = await self._get_or_create_container(cache_key=cache_key, identity=identity) + except Exception as e: + return _sandbox_unavailable("creating the sandbox", e) + return _ReadySandbox(container=container, params=params) + async def _get_or_create_container( self, cache_key: str | None, diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index dfe508f3630..68876812c3e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger): enrich_http_exception_with_guardrail_context(result, self) raise result + def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool: + return ( + isinstance(request_data.get("guardrails"), list) + and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True + ) + async def async_post_call_success_deployment_hook( self, request_data: dict, @@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger): """ Allow modifying / reviewing the response just after it's received from the deployment. """ - # should run guardrail - litellm_guardrails: Final = request_data.get("guardrails") - if litellm_guardrails is None or not isinstance(litellm_guardrails, list): - return None - - if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True: + if not self.runs_post_call_deployment_hook(request_data): return None target: Final = self._deployment_hook_target() diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 320231430d2..5b81fd9a331 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -338,9 +338,9 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_ToolTable", "LiteLLM_TelemetryReport", "LiteLLM_AccessGroupTable", - "LiteLLM_AccountingDelivery", - "LiteLLM_AccountingEvent", + "LiteLLM_SpendLogDelivery", "LiteLLM_ClaudeCodePluginTable", + "LiteLLM_ClaudeCodePluginUpload", "LiteLLM_MemoryTable", "LiteLLM_AdaptiveRouterState", "LiteLLM_AdaptiveRouterSession", diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 1c9a443b546..fc1a601f9da 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -54,6 +54,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_cust from litellm.litellm_core_utils.service_tier_utils import ( get_service_tier_from_standard_logging_payload, ) +from litellm.llms.azure.common_utils import caused_by_entra_rejection from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.models.end_user import LiteLLM_EndUserTable @@ -448,7 +449,7 @@ def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, return not ( isinstance(exception, CallerCredentialAuthenticationError) or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)) - or logged_call.get("api_call_start_time") is None + or (logged_call.get("api_call_start_time") is None and not caused_by_entra_rejection(exception)) or _used_a_caller_credential(logged_call) ) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2bb89248be4..d41647fe5f1 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast from typing_extensions import Never, ReadOnly import litellm +from litellm._internal_context import billed_separately from litellm._logging import verbose_logger from litellm.anthropic_interface import messages as anthropic_messages from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME @@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger): search_kwargs: Final = MappingProxyType( {**configured_search_kwargs, **parent_correlation.as_search_kwargs()} ) - result: Final = ( - await litellm.asearch( - query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs + with billed_separately(): + result: Final = ( + await litellm.asearch( + query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs + ) + if search_metadata is None + else await litellm.asearch( + query=query_arg, + search_provider=search_provider, + litellm_metadata=search_metadata, + **_NO_ASEARCH_NAMED, + **search_kwargs, + ) ) - if search_metadata is None - else await litellm.asearch( - query=query_arg, - search_provider=search_provider, - litellm_metadata=search_metadata, - **_NO_ASEARCH_NAMED, - **search_kwargs, - ) - ) # Format using transformation function search_result_text: Final = WebSearchTransformation.format_search_response(result) diff --git a/litellm/litellm_core_utils/add_retry_fallback_headers.py b/litellm/litellm_core_utils/add_retry_fallback_headers.py index 12b26954e4c..4cf2d1d58d8 100644 --- a/litellm/litellm_core_utils/add_retry_fallback_headers.py +++ b/litellm/litellm_core_utils/add_retry_fallback_headers.py @@ -202,7 +202,7 @@ def response_total_token_count(response: object) -> int: def apply_remaining_usage_headers( additional_headers: dict[str, object], - remaining_usage: dict[str, int], + remaining_usage: Mapping[str, object], ) -> None: for header, value in remaining_usage.items(): if value is not None and header not in additional_headers: diff --git a/litellm/litellm_core_utils/agentic_followup_failure.py b/litellm/litellm_core_utils/agentic_followup_failure.py new file mode 100644 index 00000000000..318429d64b5 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_followup_failure.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass +from typing import Final, TypeAlias + +from litellm.types.utils import Usage + +ServedRound: TypeAlias = tuple[Usage, float] + +_MARK_ATTRIBUTE: Final = "_litellm_agentic_followup_mark" + + +@dataclass(frozen=True, slots=True) +class _FollowUpMark: + served_rounds: tuple[ServedRound, ...] + + +def _mark_of(error: BaseException) -> _FollowUpMark | None: + mark: Final[object] = getattr(error, _MARK_ATTRIBUTE, None) + return mark if isinstance(mark, _FollowUpMark) else None + + +def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None: + setattr(error, _MARK_ATTRIBUTE, _FollowUpMark(served_rounds)) + + +def is_agentic_followup_failure(error: BaseException) -> bool: + return _mark_of(error) is not None + + +def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]: + mark: Final = _mark_of(error) + return () if mark is None else mark.served_rounds + + +def carry_agentic_followup_mark(source: BaseException, mapped: BaseException) -> None: + if mapped is not source and is_agentic_followup_failure(source): + mark_agentic_followup_failure(mapped, rounds_served_before(source)) diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py new file mode 100644 index 00000000000..f3ee15dffdd --- /dev/null +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -0,0 +1,107 @@ +import asyncio +from collections.abc import Awaitable, Iterable, Mapping +from itertools import chain +from types import MappingProxyType +from typing import Final + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Choices, + ModelResponse, +) + + +def _strings(values: Iterable[object]) -> frozenset[str]: + return frozenset(value for value in values if isinstance(value, str)) + + +def _claimed_calls(tool_calls: object) -> tuple[object, ...]: + wrapped: Final = message_field(tool_calls, "tool_calls") + return parts_of(tool_calls if wrapped is None else wrapped) + + +async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[object, ...]: + try: + should_run, tool_calls = await gate + except Exception as e: + verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed with %s", type(e).__name__) + return () + return _claimed_calls(tool_calls) if should_run else () + + +async def every_claimed_tool_call( + refused_tool_calls: object, other_gates: Iterable[Awaitable[tuple[bool, object]]] +) -> Mapping[str, tuple[object, ...]]: + claims: Final = await asyncio.gather(*(_calls_a_gate_claims(gate) for gate in other_gates)) + return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))}) + + +def _call_ids(call: object) -> frozenset[str]: + return _strings((message_field(call, "id"), message_field(call, "call_id"))) + + +def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: + calls: Final = _claimed_calls(tool_calls) + ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls)) + names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call)) + return ids, names + + +def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: + return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names + + +def is_refused_item( + item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str] +) -> bool: + if message_field(item, "type") != item_type: + return False + return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names) + + +def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse: + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) + kept: Final = [ + item + for item in parts_of(message_field(response, "output")) + if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names) + ] + return response.model_copy(update={"output": kept}) + + +def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: + name: Final = message_field(message_field(call, "function"), "name") + return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names) + + +def _at_position( + call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, position: int +) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall: + return call if message_field(call, "index") is None else call.model_copy(update={"index": position}) + + +def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object: + if not isinstance(choice, Choices) or not choice.message.tool_calls: + return choice + surviving: Final = ( + call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names) + ) + kept: Final = [_at_position(call, position) for position, call in enumerate(surviving)] + finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason + return choice.model_copy( + update={ + "message": choice.message.model_copy(update={"tool_calls": kept or None}), + "finish_reason": finish_reason, + } + ) + + +def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse: + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) + return response.model_copy( + update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]} + ) diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py new file mode 100644 index 00000000000..b7fa693a22a --- /dev/null +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import datetime +import json +import traceback +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from typing import Final, Protocol + +import anyio +from openai import AsyncStream, Stream +from openai.types.audio import ( + TranscriptionStreamEvent, + TranscriptionTextDeltaEvent, + TranscriptionTextDoneEvent, +) +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, +) +from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage + +_STREAM_FLAG: Final = TypeAdapter(bool) + + +def transcription_stream_flag(value: object) -> bool | None: + try: + return _STREAM_FLAG.validate_python(value) + except ValidationError: + return None + + +def transcription_stream_unset(value: object) -> bool: + return value is None or value == "" + + +def transcription_stream_requested(value: object) -> bool: + return transcription_stream_flag(value) is True + + +TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" + +TranscriptionStreamExceptionMapper = Callable[[Exception], Exception] + + +def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper: + from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base + + model: Final = call_details.get("model") + custom_llm_provider: Final = call_details.get("custom_llm_provider") + extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params")) + + def map_exception(exception: Exception) -> Exception: + try: + mapped: Final[object] = exception_type( + model=model if isinstance(model, str) else None, + original_exception=exception, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + extra_kwargs=extra_kwargs, + ) + except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception + return mapped_exception + return mapped if isinstance(mapped, Exception) else exception + + return map_exception + + +def _guarded_transcription_event_payloads( + response: TranscriptionResponse, +) -> tuple[dict[str, object], dict[str, object]]: + text: Final = response.text or "" + usage: Final = response.usage + languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None)) + return ( + {"type": "transcript.text.delta", "delta": text}, + { + "type": "transcript.text.done", + "text": text, + **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}), + **({} if languages is None else {"languages": [language.model_dump() for language in languages]}), + }, + ) + + +def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]: + return ( + *( + f"data: {json.dumps(event, separators=(',', ':'))}\n\n" + for event in _guarded_transcription_event_payloads(response) + ), + TRANSCRIPTION_STREAM_DONE_FRAME, + ) + + +class GuardedTranscriptionReplay: + def __init__(self, response: TranscriptionResponse) -> None: + self.response: Final = response + self._hidden_params: dict[str, object] = response.hidden_params + + def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]: + return self._events() + + async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]: + delta, done = _guarded_transcription_event_payloads(self.response) + yield TranscriptionTextDeltaEvent.model_construct(**delta) + yield TranscriptionTextDoneEvent.model_construct(**done) + + +class TranscriptionStreamLogging(Protocol): + def success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def handle_sync_success_callbacks_for_async_calls( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + *, + for_async_request: bool = False, + ) -> None: ... + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ... + + def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ... + + +class _TranscriptionEventCollector: + def __init__(self, duration: float | None) -> None: + self.duration = duration + self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event + self.done_event: TranscriptionTextDoneEvent | None = None + + def add(self, event: TranscriptionStreamEvent) -> None: + if isinstance(event, TranscriptionTextDeltaEvent): + self.text_deltas.append(event.delta) + elif isinstance(event, TranscriptionTextDoneEvent): + self.done_event = event + + def response(self) -> TranscriptionResponse: + done_event: Final = self.done_event + response: Final = TranscriptionResponse( + text=done_event.text if done_event is not None else "".join(self.text_deltas), + usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, + languages=getattr(done_event, "languages", None) if done_event is not None else None, + ) + if self.duration is not None: + response.set_audio_transcription_duration(self.duration) + return response + + +def _billed_usage(response: TranscriptionResponse) -> Usage: + usage: Final = response.usage + token_usage: Final = ( + TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage) + if isinstance(usage, TranscriptionUsageTokensObject) + else None + ) + return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + +def _record_billable_partial_spend( + logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector +) -> None: + partial_response: Final = collector.response() + partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0 + if partial_cost <= 0: + return + logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost) + + +class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): + def __init__( + self, + stream: Stream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, + ) -> None: + self.__dict__.update(stream.__dict__) + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } + self._logging_obj = logging_obj + self._start_time = start_time + self._map_exception = map_exception + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + def _logging_iterator( + self, source_iterator: Iterator[TranscriptionStreamEvent] + ) -> Iterator[TranscriptionStreamEvent]: + try: + for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) + mapped_exception: Final = self._map_exception(exception) + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time) + raise mapped_exception from exception + finally: + self._finalize() + + def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + self._logging_obj.success_handler( + self._collector.response(), + self._start_time, + datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract + ) + + def close(self) -> None: + try: + super().close() + finally: + self._finalize() + + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + + +class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): + def __init__( + self, + stream: AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, + ) -> None: + self.__dict__.update(stream.__dict__) + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } + self._logging_obj = logging_obj + self._start_time = start_time + self._map_exception = map_exception + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + async def _logging_iterator( + self, source_iterator: AsyncIterator[TranscriptionStreamEvent] + ) -> AsyncIterator[TranscriptionStreamEvent]: + try: + async for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) + mapped_exception: Final = self._map_exception(exception) + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler( + mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True + ) + await self._logging_obj.async_failure_handler( + mapped_exception, + traceback.format_exc(), + self._start_time, + end_time, + ) + raise mapped_exception from exception + finally: + await self._finalize() + + async def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + response: Final = self._collector.response() + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.handle_sync_success_callbacks_for_async_calls( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + with anyio.CancelScope(shield=True): + await self._logging_obj.async_success_handler( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + + async def close(self) -> None: + try: + await super().close() + finally: + await self._finalize() + + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + + +def wrap_transcription_stream( + stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, +) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream: + if isinstance(stream, AsyncStream): + return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception) + return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception) diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 640559ac7e2..29b10ac3c83 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -21,6 +21,16 @@ from litellm.types.files import ( from litellm.types.utils import FileTypes +def normalized_audio_duration_seconds(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + seconds: Final = float(value) + except OverflowError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + @dataclass class ProcessedAudioFile: """ diff --git a/litellm/litellm_core_utils/batch_utils.py b/litellm/litellm_core_utils/batch_utils.py index 5b94d4b4e5b..7c7b7ea5c4b 100644 --- a/litellm/litellm_core_utils/batch_utils.py +++ b/litellm/litellm_core_utils/batch_utils.py @@ -3,21 +3,32 @@ import json from collections.abc import Callable, Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar +from dataclasses import dataclass from os import PathLike -from typing import Final, TypeVar +from typing import Final, Literal, TypeVar from openai.types import Batch from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import finite_cost +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose from litellm.types.utils import CallTypes, LlmProviders +from litellm.types.videos.main import VideoObject +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG, canonical_video_id, is_video_status_call_type _DECODED_JSONL_ROW: Final = TypeAdapter(object) _DEPLOYMENT_FIELD: Final = TypeAdapter(Mapping[str, object]) +_PAYLOAD_FIELDS: Final = TypeAdapter(Mapping[str, object]) _ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...]) +_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"}) +_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"}) BATCH_ENDPOINT: Final = "/v1/batch" +POLLED_BATCH_ID_KWARG: Final = "polled_batch_id" +POLLED_BATCH_COST_IS_FINAL_KWARG: Final = "polled_batch_cost_is_final" +BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost" _AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value}) _batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False) @@ -289,12 +300,151 @@ def is_batch_retrieve_call_type(call_type: object) -> bool: return isinstance(call_type, str) and call_type in BATCH_RETRIEVE_CALL_TYPES -def batch_charged_once_marker_key(spend_key: str, batch_id: str) -> str: - return f"{spend_key}:batch:{batch_id}" +@dataclass(frozen=True, slots=True) +class ChargeOnceJob: + kind: Literal["batch", "video"] + job_id: str + repeat_poll_cost: float = 0.0 -def batch_id_to_charge_once(call_type: object, response_obj: object, response_cost: float) -> str | None: - """A finished batch reports its whole cost on every poll, so its id is charged once per counter.""" - if response_cost <= 0 or not is_batch_retrieve_call_type(call_type): +def job_charged_once_marker_key(spend_key: str, job: ChargeOnceJob) -> str: + return f"{spend_key}:{job.kind}:{job.job_id}" + + +def _guardrail_cost_of_poll(standard_logging_payload: object) -> float: + guardrail_information: Final = ( + _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("guardrail_information") + if isinstance(standard_logging_payload, Mapping) + else None + ) + return guardrail_information_cost(guardrail_information) + + +def charged_poll_cost(kwargs: Mapping[str, object]) -> object: + """The poll's whole cost as the cost callback charges it, guardrails included, which the logging kwargs' own + response_cost leaves out.""" + standard_logging_payload: Final = kwargs.get("standard_logging_object") + logged_cost: Final = ( + _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("response_cost") + if isinstance(standard_logging_payload, Mapping) + else None + ) + return kwargs.get("response_cost") if logged_cost is None else logged_cost + + +def job_cost_of_poll(response_cost: object, standard_logging_payload: object) -> float: + """What the polled job itself cost, since a guardrail on the poll charges every poll, running or finished.""" + return (finite_cost(response_cost) or 0.0) - _guardrail_cost_of_poll(standard_logging_payload) + + +def repeat_poll_cost(call_type: object, response_cost: object, standard_logging_payload: object) -> float: + """What a poll still costs once an earlier poll charged its job: a video poll's guardrails, which every video + poll has always paid, and nothing for a batch poll, whose repeats have always been free.""" + if not is_video_status_call_type(call_type): + return 0.0 + return max(0.0, min(finite_cost(response_cost) or 0.0, _guardrail_cost_of_poll(standard_logging_payload))) + + +def batch_cost_is_final(batch: Batch) -> bool: + """Whether this retrieve of the batch is the one to account its cost from. + + A batch still in flight has nothing to price, and a "completed" batch can report + no output_file_id for a moment before the output populates; pricing either records + $0 under the batch's single spend row and pins it there. Final means a completed + batch whose output file has arrived or whose counts prove no line succeeded, or + any other terminal status (failed, cancelled, expired). + """ + if batch.status not in _TERMINAL_BATCH_STATUSES: + return False + if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None: + return True + request_counts: Final = batch.request_counts + return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0 + + +@dataclass(frozen=True, slots=True) +class PolledBatch: + id: str + cost_is_final: bool + + +def _polled_batch(kwargs: Mapping[str, object], response_obj: object) -> PolledBatch | None: + """The retrieved batch, read from the logging kwargs when message redaction replaced the logged response.""" + if not is_batch_retrieve_call_type(kwargs.get("call_type")): return None - return response_obj.id if isinstance(response_obj, Batch) else None + if isinstance(response_obj, Batch): + return PolledBatch(id=response_obj.id, cost_is_final=batch_cost_is_final(response_obj)) + stamped_id: Final = kwargs.get(POLLED_BATCH_ID_KWARG) + stamped_cost_is_final: Final = kwargs.get(POLLED_BATCH_COST_IS_FINAL_KWARG) + if not isinstance(stamped_id, str) or not stamped_id or not isinstance(stamped_cost_is_final, bool): + return None + return PolledBatch(id=stamped_id, cost_is_final=stamped_cost_is_final) + + +def defers_batch_cost(kwargs: Mapping[str, object], response_obj: object) -> bool: + """A batch retrieve before the batch's cost is final charges nothing, guardrails included, since the retrieve + that settles the batch charges its whole cost once.""" + polled: Final = _polled_batch(kwargs, response_obj) + return polled is not None and not polled.cost_is_final + + +def pays_only_its_guardrail(call_type: object, kwargs: Mapping[str, object], response_cost: object) -> bool: + """A video poll whose whole cost is its guardrails', charged like any other request.""" + if not is_video_status_call_type(call_type): + return False + return ( + job_cost_of_poll(response_cost, kwargs.get("standard_logging_object")) + <= 0 + < (finite_cost(response_cost) or 0.0) + ) + + +def _requested_video_id(kwargs: Mapping[str, object]) -> str | None: + litellm_params: Final = kwargs.get("litellm_params") + requested: Final = ( + _PAYLOAD_FIELDS.validate_python(litellm_params).get("video_id") if isinstance(litellm_params, Mapping) else None + ) + return requested if isinstance(requested, str) and requested else None + + +def charged_video_id( + call_type: object, kwargs: Mapping[str, object], fallback_video_id: str | None, response_cost: object +) -> str | None: + """The id a finished video's own cost is charged once under. + + That is the id the client polled, which names the deployment that made the video, since a + custom provider's own id can repeat across deployments. The status comes from the logging + kwargs because message redaction replaces the logged video object. + """ + if not is_video_status_call_type(call_type) or kwargs.get(POLLED_VIDEO_STATUS_KWARG) != "completed": + return None + if job_cost_of_poll(response_cost, kwargs.get("standard_logging_object")) <= 0: + return None + video_id: Final = _requested_video_id(kwargs) or fallback_video_id + return None if video_id is None else canonical_video_id(video_id) + + +def _charged_batch(kwargs: Mapping[str, object], response_obj: object, response_cost: float) -> ChargeOnceJob | None: + polled: Final = _polled_batch(kwargs, response_obj) + if polled is None or not polled.cost_is_final or response_cost <= 0: + return None + return ChargeOnceJob(kind="batch", job_id=polled.id) + + +def job_to_charge_once( + kwargs: Mapping[str, object], response_obj: object, response_cost: float +) -> ChargeOnceJob | None: + """A finished batch or video reports its whole cost on every poll, so it is charged once per counter.""" + call_type: Final = kwargs.get("call_type") + if not is_video_status_call_type(call_type): + return _charged_batch(kwargs, response_obj, response_cost) + video_id: Final = charged_video_id( + call_type, kwargs, response_obj.id if isinstance(response_obj, VideoObject) else None, response_cost + ) + if video_id is None: + return None + return ChargeOnceJob( + kind="video", + job_id=video_id, + repeat_poll_cost=repeat_poll_cost(call_type, response_cost, kwargs.get("standard_logging_object")), + ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 2ba47cbb400..ce98d2cbb01 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -1,28 +1,35 @@ # this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api import json -from collections.abc import Mapping +from collections.abc import Awaitable, Iterator, Mapping, Sequence from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from pydantic import TypeAdapter, ValidationError +from litellm._internal_context import internal_sub_call from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs +from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + AgenticLoopLimitError, AgenticLoopPlan, AgenticLoopRequestPatch, + AgenticLoopSafetyError, converted_stream_requested, is_interception_internal_key, stashed_stream_options, @@ -30,6 +37,8 @@ from litellm.types.integrations.custom_logger import ( from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper +_DispatchResult: Final = TypeVar("_DispatchResult") + _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset( ( "acompletion", @@ -43,22 +52,41 @@ _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset( ) +async def bill_first_round_on_failure( + dispatch: Awaitable[_DispatchResult], *, logging_obj: object, response: object, stream: bool +) -> _DispatchResult: + try: + return await dispatch + except Exception as error: + if stream or not isinstance(logging_obj, LiteLLMLoggingObject): + mark_agentic_followup_failure(error, rounds_served_before(error)) + raise + this_round: Final = logging_obj.served_round(response) + served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,))) + mark_agentic_followup_failure(error, served_rounds) + logging_obj.record_served_rounds_for_failure(served_rounds) + raise + + def _gate_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_should_run_agentic_loop - func: Final = type(callback).async_should_run_agentic_loop - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return type(callback).async_should_run_agentic_loop is not CustomLogger.async_should_run_agentic_loop def _build_plan_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_build_agentic_loop_plan - func: Final = type(callback).async_build_agentic_loop_plan - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return type(callback).async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan + + +def _chat_build_plan_overridden(callback: CustomLogger) -> bool: + return ( + type(callback).async_build_chat_completion_agentic_loop_plan + is not CustomLogger.async_build_chat_completion_agentic_loop_plan + ) def _post_hook_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_post_agentic_loop_response_hook - func: Final = type(callback).async_post_agentic_loop_response_hook - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return ( + type(callback).async_post_agentic_loop_response_hook is not CustomLogger.async_post_agentic_loop_response_hook + ) _STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -75,14 +103,14 @@ def _coerce_int(value: object, default: int) -> int: return int(value) if isinstance(value, (int, str)) else default -def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]: +def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, tuple[str, ...]]: depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0) configured: Final = validated_max_agentic_loops( kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" ) max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints") - fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else [] + fingerprints: Final = tuple(str(fp) for fp in raw_fingerprints) if isinstance(raw_fingerprints, list) else () return depth, max_loops, fingerprints @@ -95,16 +123,16 @@ def _fingerprint_tools(tool_calls: object) -> str: def _check_agentic_loop_safety( tool_calls: object, - fingerprints: list[str], + fingerprints: Sequence[str], depth: int, max_loops: int, model: str, ) -> str: fingerprint: Final = _fingerprint_tools(tool_calls) if fingerprint in fingerprints: - raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") + raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @@ -133,6 +161,83 @@ def _converted_stream_replay( ) +def _end_refused_turn( + finalized: ModelResponse, + *, + kwargs: Mapping[str, object], + depth: int, + model: str, + custom_llm_provider: str, + logging_obj: object, +) -> ModelResponse | CustomStreamWrapper: + replay: Final = _converted_stream_replay( + finalized, + kwargs=kwargs, + depth=depth, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return finalized if replay is None else replay + + +def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]: + return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)] + + +def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]: + import litellm + + dynamic: Final = getattr(logging_obj, "dynamic_success_callbacks", None) or [] + return tuple(callback for callback in (*litellm.callbacks, *dynamic) if isinstance(callback, CustomLogger)) + + +async def _refused_chat_turn( + callbacks: Sequence[CustomLogger], + refusing: CustomLogger, + refused_tool_calls: object, + *, + response: ModelResponse, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + custom_llm_provider: str, + kwargs: Mapping[str, object], +) -> ModelResponse: + surface_kwargs: Final = { + **kwargs, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + "custom_llm_provider": custom_llm_provider, + } + + def other_gates() -> Iterator[Awaitable[tuple[bool, object]]]: + for callback in callbacks: + if callback is refusing: + continue + yield callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=dict(kwargs), + ) + if _gate_overridden(callback): + yield callback.async_should_run_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=dict(surface_kwargs), + ) + + claimed: Final = await every_claimed_tool_call(refused_tool_calls, other_gates()) + return finalize_refused_chat_turn(response, claimed) + + def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: metadata: Final = kwargs_for_followup.get("litellm_metadata") return MappingProxyType( @@ -163,6 +268,12 @@ def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]: } +def _leave_pricing_to_the_request(response: object) -> None: + hidden_params: Final = get_hidden_params(response) + if hidden_params is not None and not served_from_cache(response): + hidden_params.pop("response_cost", None) + + async def _execute_chat_completion_agentic_plan( *, plan: AgenticLoopPlan, @@ -174,7 +285,7 @@ async def _execute_chat_completion_agentic_plan( custom_llm_provider: str, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, ) -> object: import litellm @@ -204,12 +315,13 @@ async def _execute_chat_completion_agentic_plan( ) try: - response_followup = await litellm.acompletion( - model=full_model_name, - messages=patch.messages, - **optional_params_for_followup, - **kwargs_for_followup, - ) + with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None): + response_followup = await litellm.acompletion( + model=full_model_name, + messages=patch.messages, + **optional_params_for_followup, + **kwargs_for_followup, + ) if _post_hook_overridden(callback): try: response_followup = await callback.async_post_agentic_loop_response_hook( @@ -224,6 +336,7 @@ async def _execute_chat_completion_agentic_plan( model, str(e), ) + _leave_pricing_to_the_request(response_followup) replay: Final = _converted_stream_replay( response_followup, kwargs=kwargs, @@ -250,23 +363,18 @@ async def maybe_run_chat_completion_agentic_loop( *, response: ModelResponse, model: str, - messages: list, - optional_params: dict, + messages: list[dict[str, object]], + optional_params: dict[str, object], kwargs: dict[str, object], logging_obj: object, custom_llm_provider: str, stream: bool, ) -> ModelResponse | CustomStreamWrapper | None: - import litellm - - callbacks: Final = litellm.callbacks + (getattr(logging_obj, "dynamic_success_callbacks", None) or []) + callbacks: Final = _custom_loggers(logging_obj) depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) - tools: Final = optional_params.get("tools", []) + tools: Final = _request_tools(optional_params) for callback in callbacks: - if not isinstance(callback, CustomLogger): - continue - if not _gate_overridden(callback): continue @@ -295,29 +403,39 @@ async def maybe_run_chat_completion_agentic_loop( if not should_run: continue - fingerprint = _check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) - try: - if not _build_plan_overridden(callback): - return await callback.async_run_agentic_loop( - tools=tool_calls, + fingerprint = _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if stream: + raise + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) + return _end_refused_turn( + await _refused_chat_turn( + callbacks, + callback, + tool_calls, + response=response, model=model, messages=messages, - response=response, - anthropic_messages_provider_config=None, - anthropic_messages_optional_request_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=hook_kwargs, - ) + tools=tools, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ), + kwargs=kwargs, + depth=depth, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) - plan = await callback.async_build_agentic_loop_plan( + if not _build_plan_overridden(callback): + return await callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -329,31 +447,38 @@ async def maybe_run_chat_completion_agentic_loop( kwargs=hook_kwargs, ) - if plan.response_override is not None: - return plan.response_override - if plan.terminate: - return response - if not plan.run_agentic_loop: - continue + plan = await callback.async_build_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) - return await _execute_chat_completion_agentic_plan( - plan=plan, - callback=callback, - model=model, - optional_params=optional_params, - kwargs=kwargs, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", - str(e), - ) + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + return response + if not plan.run_agentic_loop: + continue + + return await _execute_chat_completion_agentic_plan( + plan=plan, + callback=callback, + model=model, + optional_params=optional_params, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) return _converted_stream_replay( response, @@ -363,3 +488,113 @@ async def maybe_run_chat_completion_agentic_loop( custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) + + +async def run_chat_completion_agentic_hooks( + *, + response: ModelResponse, + model: str, + messages: list[dict[str, object]], + optional_params: dict[str, object], + kwargs: dict[str, object], + logging_obj: LiteLLMLoggingObject, + custom_llm_provider: str, + stream: bool, +) -> object | None: + callbacks: Final = _custom_loggers(logging_obj) + depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) + tools: Final = _request_tools(optional_params) + hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider} + + for callback in callbacks: + try: + should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + except Exception: + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop" + ) + continue + + if not should_run: + continue + + try: + fingerprint = _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if stream: + raise + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) + return await _refused_chat_turn( + callbacks, + callback, + tool_calls, + response=response, + model=model, + messages=messages, + tools=tools, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + + if not _chat_build_plan_overridden(callback): + return await callback.async_run_chat_completion_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) + + plan = await callback.async_build_chat_completion_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) + + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + verbose_logger.debug( + "Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason + ) + return response + if not plan.run_agentic_loop: + continue + + return await _execute_chat_completion_agentic_plan( + plan=plan, + callback=callback, + model=model, + optional_params=optional_params, + kwargs=hook_kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) + + return None diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 6fd6e91a627..b28d5a4ea21 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -97,7 +97,7 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non return float(seconds / denominator) -_DROP_PARAMS_BOOL: Final = TypeAdapter(bool) +_LAX_BOOL: Final = TypeAdapter(bool) def as_str_mapping(value: object) -> Mapping[str, object] | None: @@ -108,6 +108,17 @@ def as_object_list(value: object) -> Sequence[object] | None: return value if isinstance(value, list) else None +_Key = TypeVar("_Key") # rebind-ok: a TypeVar cannot be Final +_Value = TypeVar("_Value") # rebind-ok: a TypeVar cannot be Final + + +def without( + mapping: Mapping[_Key, _Value], *keys: object +) -> dict[_Key, _Value]: # mutable-ok: a fresh copy the caller owns + """``mapping`` without ``keys``, as a new dict in the same order; ``mapping`` is unchanged.""" + return {key: value for key, value in mapping.items() if key not in keys} + + def decoded_json_object(value: object) -> Mapping[str, object]: mapping: Final = as_str_mapping(value) if mapping is not None: @@ -160,15 +171,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]: return iter_items(value) if value else () -def normalize_drop_params(value: object) -> bool | None: +def lax_bool(value: object) -> bool | None: if value is None or isinstance(value, bool): return value try: - return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value) + return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value) except ValidationError: return None +def normalize_drop_params(value: object) -> bool | None: + return lax_bool(value) + + def drops_unsupported_params(drop_params: object) -> bool: return litellm.drop_params is True or normalize_drop_params(drop_params) is True diff --git a/litellm/litellm_core_utils/decisions_pricing.py b/litellm/litellm_core_utils/decisions_pricing.py index 9199a8097cc..accd315da23 100644 --- a/litellm/litellm_core_utils/decisions_pricing.py +++ b/litellm/litellm_core_utils/decisions_pricing.py @@ -12,6 +12,7 @@ from typing import Final DECISIONS_PRICING_SUFFIX: Final = "_decisions" DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"}) +QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"}) def is_decisions_call_type(call_type: str) -> bool: @@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]: } +def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None: + if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS: + return None + rates: Final = {**model_info, **overrides} + request_price: Final = rates.get("input_cost_per_query") + if not isinstance(request_price, (int, float)): + return None + bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token")) + return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price) + + def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]: """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields.""" plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)} diff --git a/litellm/litellm_core_utils/deployment_pricing.py b/litellm/litellm_core_utils/deployment_pricing.py index 622cc91ab5e..7aa85921634 100644 --- a/litellm/litellm_core_utils/deployment_pricing.py +++ b/litellm/litellm_core_utils/deployment_pricing.py @@ -1,4 +1,4 @@ -from collections.abc import Iterator, Mapping +from collections.abc import Iterable, Iterator, Mapping from types import MappingProxyType from typing import Final @@ -25,6 +25,14 @@ def deployment_pricing_models(deployment: object) -> tuple[str, ...]: return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model")) +def deployment_backend_model(deployment: object) -> str: + return next(iter(deployment_pricing_models(deployment)), "") + + +def deployment_backend_models(deployments: Iterable[object]) -> tuple[str, ...]: + return tuple(map(deployment_backend_model, deployments)) + + def _named_models(*candidates: object) -> Iterator[str]: return (candidate for candidate in candidates if isinstance(candidate, str) and candidate) diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py index 00317e502ec..717db34ede5 100644 --- a/litellm/litellm_core_utils/error_normalization.py +++ b/litellm/litellm_core_utils/error_normalization.py @@ -133,6 +133,8 @@ _STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType( } ) +_OVERLOADED_STATUS_CODE: Final = "529" + _INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( TypeError, KeyError, @@ -201,6 +203,8 @@ def normalize_error(exc: Exception | None, status_code: str, message: str) -> st ) if by_message is not None: return by_message + if status_code == _OVERLOADED_STATUS_CODE: + return PROVIDER_OVERLOADED by_class: Final = _classify_by_class(exc) if by_class is not None: return by_class diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index f1195802fc5..6d39f272f3e 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import _ENABLE_SECRET_REDACTION, redact_string, verbose_logger +from litellm.litellm_core_utils.agentic_followup_failure import carry_agentic_followup_mark from litellm.litellm_core_utils.bug_report import ( bug_report_notice, build_bug_report, @@ -605,6 +606,11 @@ def _map_openai_exception( ) +def _anthropic_overload_status(original_exception: _ProviderHTTPException) -> int | None: + upstream_status: Final = getattr(original_exception, "status_code", None) + return 529 if isinstance(upstream_status, int) and upstream_status >= 500 else None + + def _map_anthropic_exception( *, model: str, @@ -630,6 +636,7 @@ def _map_anthropic_exception( message=f"AnthropicError - {error_str}", model=model, llm_provider="anthropic", + provider_status_code=_anthropic_overload_status(original_exception), ) if "Invalid API Key" in error_str: raise AuthenticationError( @@ -694,6 +701,7 @@ def _map_anthropic_exception( llm_provider="anthropic", model=model, response=getattr(original_exception, "response", None), + provider_status_code=original_exception.status_code, ) elif original_exception.status_code == 502: raise BadGatewayError( @@ -2895,6 +2903,7 @@ def exception_type( request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request ) except Exception as e: + carry_agentic_followup_mark(source=original_exception, mapped=e) # LOGGING exception_logging( logger_fn=None, @@ -2920,6 +2929,7 @@ def exception_type( model="", ) setattr(raised_exc, "litellm_response_headers", litellm_response_headers) + carry_agentic_followup_mark(source=original_exception, mapped=raised_exc) raise raised_exc diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d6d9be4e132..d0ac5c00513 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -33,6 +33,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( ) PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header" +AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset( + {"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"} +) OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset( { "token_exchange_endpoint", @@ -94,6 +97,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( | ANTHROPIC_WIF_KWARGS_KEYS | OPENAI_WIF_KWARGS_KEYS | OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS + | AGENTIC_LOOP_CONTROL_KWARGS_KEYS | frozenset(CustomPricingLiteLLMParams.model_fields) ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 94ba02bfbd7..1ffb5e2f8e5 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_routing_generalization, ) from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.secret_managers.main import get_secret, get_secret_str from ..types.router import GenericLiteLLMParams, LiteLLM_Params @@ -1008,6 +1009,4 @@ def _get_openai_compatible_provider_info( def get_provider_affinity_header(api_base: str | None) -> str | None: - from litellm.llms.opencode.transformation import OpenCodeProviderAdapter - - return OpenCodeProviderAdapter().get_provider_affinity_header(api_base) + return "x-opencode-session" if is_opencode_api_base(api_base) else None diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 7fc6eab6c60..e452befe295 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -320,9 +320,7 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") - model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition - dict[str, object], parsed - ) + model_cost_map: Final = _BUNDLED_CATALOG_ADAPTER.validate_python(parsed) return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag) diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index 62dfcb073e5..b5916369917 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -12,10 +12,18 @@ _PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderS class ProviderSpecificHeaderUtils: @staticmethod - def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None: - headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + def validated_provider_specific_headers( + provider_specific_header: object, custom_llm_provider: str | None + ) -> Mapping[str, str]: + return ProviderSpecificHeaderUtils.get_provider_specific_headers( _PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider ) + + @staticmethod + def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None: + headers: Final = ProviderSpecificHeaderUtils.validated_provider_specific_headers( + provider_specific_header, custom_llm_provider + ) return next((value for key, value in headers.items() if key.lower() == "authorization"), None) @staticmethod diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 37657a94f16..ce57a9bd4a5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -19,9 +19,11 @@ from datetime import datetime as dt_object from enum import Enum from functools import lru_cache from types import MappingProxyType, TracebackType -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast +from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast from httpx import Response, Timeout +from openai import AsyncStream as OpenAIAsyncStream +from openai import Stream as OpenAIStream from pydantic import BaseModel, JsonValue, TypeAdapter import litellm @@ -36,7 +38,7 @@ from litellm._logging import ( verbose_logger, ) from litellm._uuid import uuid -from litellm.batches.batch_utils import batch_cost_is_final, handle_completed_batch +from litellm.batches.batch_utils import handle_completed_batch from litellm.caching.caching import DualCache from litellm.caching.caching_handler import LLMCachingHandler from litellm.caching.redis_batch import flush_post_call_redis_batches @@ -52,8 +54,11 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( + BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, + declared_cost_map_pricing, + get_response_cost_from_hidden_params, get_usage_object, pricing_entry_for_cost_calc, select_model_name_for_cost_calc, @@ -72,6 +77,13 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.litellm_core_utils.agentic_followup_failure import ServedRound +from litellm.litellm_core_utils.batch_utils import ( + POLLED_BATCH_COST_IS_FINAL_KWARG, + POLLED_BATCH_ID_KWARG, + batch_cost_is_final, +) from litellm.litellm_core_utils.classifier_logging import ( classifier_audit_fields, classifier_input_snapshot, @@ -97,6 +109,7 @@ from litellm.litellm_core_utils.hidden_params import ( get_hidden_params, served_from_cache, set_hidden_param, + set_hidden_params, ) from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -122,6 +135,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( diagnose_zero_cost, zero_cost_warning, ) +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, @@ -142,7 +156,7 @@ from litellm.litellm_core_utils.served_output_texts import ( from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse @@ -211,7 +225,11 @@ from litellm.types.utils import ( batch_chat_usage, ) from litellm.types.videos.main import VideoObject -from litellm.utils import get_base_model_from_metadata, print_verbose +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG +from litellm.utils import ( + get_base_model_from_metadata, + print_verbose, +) from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -271,8 +289,6 @@ if TYPE_CHECKING: from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation - from litellm.proxy.spend_tracking.usage_capture import CapturedUsage - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, @@ -300,6 +316,11 @@ class _EnterpriseAlertingLoggers(NamedTuple): smtp_email: type[CustomLogger] +class BridgedChatRequest(NamedTuple): + optional_params: Mapping[str, object] + litellm_params: Mapping[str, object] + + @functools.cache def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers: try: @@ -580,6 +601,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__) _REDACTED_RESULT: Final = TypeAdapter(object) +_LoggedResponse: Final = TypeVar("_LoggedResponse") _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -644,54 +666,7 @@ class UsageObservation(Enum): ESTIMATED = "estimated" -@dataclass(frozen=True, slots=True) -class UsageCaptureObservation: - usage_json: str | None - provenance: UsageObservation - response_cost: float | None = None - - @classmethod - def from_usage( - cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None - ) -> "UsageCaptureObservation | UsageCaptureError": - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError - - try: - serialized: Final = ( - None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json() - ) - except Exception as error: # noqa: BLE001 # observation failures must not interrupt legacy retry handling - return UsageCaptureError(str(error)) - else: - return cls(serialized, provenance, response_cost) - - class Logging(LiteLLMLoggingBaseClass): - usage_capture_callback: ClassVar[ - Callable[ - ["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None" - ] - | None - ] = None - - def capture_usage_before_logging( - self, - response: object, - start_time: datetime.datetime | None = None, - end_time: datetime.datetime | None = None, - *, - freeze_only: bool = False, - ) -> None: - capture: Final = type(self).usage_capture_callback - if capture is not None and (not self.stream or self._is_assembled_stream_success(response)): - capture( - self, - response, - start_time or self.start_time, - end_time or datetime.datetime.now(datetime.timezone.utc), - freeze_only, - ) - global \ supabaseClient, \ promptLayerLogger, \ @@ -706,6 +681,7 @@ class Logging(LiteLLMLoggingBaseClass): custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None custom_cost_per_second: float | None = None + custom_model_info: ModelInfo | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -873,6 +849,7 @@ class Logging(LiteLLMLoggingBaseClass): # once that response is priced, the same way a non-streamed response is logged self.client_facing_stream_model: str | None = None self.client_facing_response_id: str | None = None + self.bridged_chat_request: BridgedChatRequest | None = None self.zero_cost_warned: bool = False self.llm_caching_handler: LLMCachingHandler | None = None @@ -926,6 +903,10 @@ class Logging(LiteLLMLoggingBaseClass): self._sync_success_scheduled: bool = False self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None self._stream_failure_claimed: bool = False + self._stash_owed_on_success: bool = False + self._stash_billed_by_continuation: bool = False + self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = () + self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = () self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging": @@ -2130,6 +2111,9 @@ class Logging(LiteLLMLoggingBaseClass): if resolved_cache_hit is True: return 0.0 + if isinstance(result, (OpenAIStream, OpenAIAsyncStream)): + return None + if is_unbilled_non_inference_call( self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result ): @@ -2207,6 +2191,7 @@ class Logging(LiteLLMLoggingBaseClass): "custom_pricing": custom_pricing, "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, + "requested_model_info_override": self.custom_model_info if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, @@ -2298,12 +2283,19 @@ class Logging(LiteLLMLoggingBaseClass): return 0.0 def uncached_price(self, result: object) -> UncachedPrice: + return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True) + + def _price_without_recording( + self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool + ) -> UncachedPrice: previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates debug_key: Final = "response_cost_failure_debug_information" previous_failure_debug_info: Final = self.model_call_details.get(debug_key) try: - cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) + cost: Final = self.response_cost_calculator( + result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost + ) return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown) finally: self.cost_breakdown = previous_cost_breakdown @@ -2403,6 +2395,10 @@ class Logging(LiteLLMLoggingBaseClass): additional_headers=additional_headers, ) + def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None: + """Price this call from its own full price entry instead of ``litellm.model_cost``.""" + self.custom_model_info = custom_model_info + def custom_pricing_for(self, result: object) -> bool: if self.spilled_over_ptu_for(result): return False @@ -2542,11 +2538,6 @@ class Logging(LiteLLMLoggingBaseClass): ) return - self.capture_usage_before_logging( - TypeAdapter(object).validate_python(result), - TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time), - TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time), - ) await self.async_success_handler( result, start_time=start_time, @@ -2568,6 +2559,33 @@ class Logging(LiteLLMLoggingBaseClass): **kwargs, ) + def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool: + if ( + cost != 0 + or self._provider_reported_cost == 0 + or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0 + ): + return True + if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result): + return True + params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {}) + if is_unbilled_non_inference_call( + payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result + ): + return True + if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None: + return True + passthrough: Final = _LITELLM_PARAMS.validate_python( + self._model_call_details_view.get("passthrough_logging_payload") or {} + ) + if passthrough.get("cost_per_request") is not None: + return True + model_info: Final = payload["model_map_information"]["model_map_value"] + if model_info is None: + return False + prices: Final = declared_cost_map_pricing(model_info["key"], model_info["key"], payload["custom_llm_provider"]) + return any(value is not None and ("cost" in key or "pricing" in key) for key, value in prices.items()) + def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None: if self.prepared_cost is not None: return self.prepared_cost @@ -2620,6 +2638,7 @@ class Logging(LiteLLMLoggingBaseClass): "cost_breakdown": copy.deepcopy(payload["cost_breakdown"]), }, usage, + price_known=self._response_price_is_known(prepared[2], raw_cost, payload), ) self.record_partial_usage_for_failure( usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"] @@ -2630,20 +2649,12 @@ class Logging(LiteLLMLoggingBaseClass): self.finalized_cost = finalized self.model_call_details["response_cost"] = finalized.amount - def record_usage_capture(self, captured: "CapturedUsage") -> None: - self.model_call_details["_accounting_capture"] = captured - - def record_usage_pricing(self, pricing: object) -> None: - self.model_call_details["_accounting_pricing_snapshot"] = pricing - def record_partial_usage_for_failure( self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN ) -> None: """Stash what an interrupted stream already consumed so the failure log bills it instead of zero.""" self.model_call_details["combined_usage_object"] = usage self.model_call_details["response_cost"] = response_cost - if type(self).usage_capture_callback is not None: - self.model_call_details["_accounting_usage_observed"] = observation def record_failure_after_earlier_rounds( self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float @@ -2723,66 +2734,28 @@ class Logging(LiteLLMLoggingBaseClass): own usage while the request's failure log still bills every attempt.""" usage: Final = self._model_call_details_view.get("combined_usage_object") response_cost: Final = self._model_call_details_view.get("response_cost") - if isinstance(usage, Usage) and type(self).usage_capture_callback is not None: - self.capture_failed_attempt_usage(usage) - self.model_call_details.pop("_accounting_capture", None) + billed_by_continuation: Final = self._stash_billed_by_continuation + owed_on_success: Final = self._stash_owed_on_success + self._stash_billed_by_continuation = False + self._stash_owed_on_success = False self.model_call_details.pop("_accounting_pricing_snapshot", None) self.model_call_details.pop("_accounting_usage_observed", None) self.model_call_details.pop("combined_usage_object", None) self.model_call_details.pop("response_cost", None) - if isinstance(usage, Usage): - self.model_call_details["current_attempt_api_call_start_time"] = None - self.model_call_details["failed_attempts_usage"] = ( - *self.model_call_details.get("failed_attempts_usage", ()), - (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0), - ) - - def capture_failed_attempt_usage(self, usage: Usage) -> None: - from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY - from litellm.proxy.spend_tracking.usage_capture import CapturedUsage - - capture: Final = type(self).usage_capture_callback - if capture is None: + if not isinstance(usage, Usage): return - attempt: Final = self.attempt_scoped_copy() - attempt.prepared_cost = None - attempt.finalized_cost = None - identity: Final = ( - f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}" - ) - params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {}) - metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params) - observed: Final[object] = self.model_call_details.get("_accounting_usage_observed") - cost: Final[object] = self.model_call_details.get("response_cost") - attempt.model_call_details = { - **attempt.model_call_details, - "litellm_call_id": identity, - "parent_request_id": self.litellm_call_id, - "_accounting_capture": None, - "_accounting_prior_captures": (), - "_accounting_usage_snapshot": UsageCaptureObservation.from_usage( - usage, - observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN, - float(cost) if isinstance(cost, (int, float)) else None, - ), - "litellm_params": { - **params, - "metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"}, - "litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"}, - }, - } - captured: Final = capture( + attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0) + self.model_call_details["current_attempt_api_call_start_time"] = None + self.model_call_details["failed_attempts_usage"] = ( + *self.model_call_details.get("failed_attempts_usage", ()), attempt, - {"id": identity, "usage": usage.model_dump(), "status": "failed"}, - self.start_time, - datetime.datetime.now(datetime.timezone.utc), - False, ) - if isinstance(captured, CapturedUsage): - self.model_call_details["_accounting_prior_captures"] = ( - *self.model_call_details.get("_accounting_prior_captures", ()), - captured, - ) + if not owed_on_success or billed_by_continuation: + return + self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt) + + def mark_stash_billed_by_continuation(self) -> None: + self._stash_billed_by_continuation = True def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None: """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw.""" @@ -2790,20 +2763,17 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["continued_partial_cost"] = response_cost self.model_call_details["continued_response_id"] = response_id - def _add_continued_partial_spend(self, response: object) -> None: + def _add_continued_partial_spend(self, response: object) -> Usage | None: partial_usage: Final = self.model_call_details.get("continued_partial_usage") partial_cost: Final = self.model_call_details.get("continued_partial_cost") own_usage: Final = getattr(response, "usage", None) own_cost: Final = self.model_call_details.get("response_cost") if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)): - return - from litellm.cost_calculator import BaseTokenUsageProcessor - - self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( - usage_objects=[own_usage, partial_usage] - ) + return own_usage if isinstance(own_usage, Usage) else None + combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage]) + self.model_call_details["combined_usage_object"] = combined_usage if not isinstance(own_cost, (int, float)): - return + return combined_usage self.model_call_details["response_cost"] = own_cost + partial_cost if self.cost_breakdown is not None: self.cost_breakdown = self.cost_breakdown | CostBreakdown( @@ -2813,6 +2783,25 @@ class Logging(LiteLLMLoggingBaseClass): "mid_stream_partial_cost": partial_cost, }, ) + return combined_usage + + def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None: + own_cost: Final = self.model_call_details.get("response_cost") + if own_usage is None or not isinstance(own_cost, (int, float)): + return + failed_attempts: Final = self._failed_attempts_owed_on_success + if not failed_attempts: + return + failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts) + self.model_call_details.pop("failed_attempts_usage", None) + self.failed_attempts_billed_on_success = failed_attempts + self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)] + ) + self.model_call_details["response_cost"] = own_cost + failed_attempts_cost + if self.cost_breakdown is not None: + self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost) def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: @@ -2820,6 +2809,24 @@ class Logging(LiteLLMLoggingBaseClass): self._stream_failure_claimed = True return True + def served_round(self, response: object) -> ServedRound | None: + usage: Final = get_usage_object(completion_response=response) + if usage is None: + return None + price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False) + return usage, price.cost or 0.0 + + def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None: + if not served_rounds: + return + self._stash_owed_on_success = True + self.record_partial_usage_for_failure( + served_rounds[0][0] + if len(served_rounds) == 1 + else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]), + sum(cost for _, cost in served_rounds), + ) + def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" usage: Final = getattr(assembled, "usage", None) @@ -2951,45 +2958,6 @@ class Logging(LiteLLMLoggingBaseClass): return self._handle_a2a_response_logging(result=result) return result - def usage_capture_response(self, result: object) -> object: - if self.stream and not self._is_assembled_stream_success(result): - return None - copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result - normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied)) - if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list): - frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result) - measured: Final = any( - isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping) - for frame in frames - if frame.get("type") == "response.done" - ) - if not measured: - metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY) - return { - "id": self.litellm_call_id, - "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")} - if metered is not None - else None, - } - assembled: Final = self._get_assembled_streaming_response( - normalized, self.start_time, self.start_time, True, [] - ) - return normalized if assembled is None else assembled - - def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]: - from litellm.utils import count_characters - - return TypeAdapter(dict[str, JsonValue]).validate_python( - { - "custom_pricing": self.custom_pricing_for(result), - "custom_cost_per_token": self.custom_cost_per_token, - "custom_cost_per_second": self.custom_cost_per_second, - "prompt_characters": count_characters(self._prompt_for_cost_calculation()) - if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value) - else None, - } - ) - def normalize_logging_result(self, result: object) -> object: """ Some endpoints return a different type of result than what is expected by the logging system. @@ -3124,9 +3092,11 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.finalized_cost.amount elif self.model_call_details.get("cache_hit") is True: self.model_call_details["response_cost"] = 0.0 + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -3136,6 +3106,7 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) if not build_logging_payload: return None @@ -3230,21 +3201,46 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"model": model}) - def _with_continued_response_id( - self, - response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: - response_id: Final = self.model_call_details.get("continued_response_id") - if not isinstance(response_id, str) or getattr(response, "id", None) == response_id: + def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): return response - return response.model_copy(update={"id": response_id}) + return self._bridged_chat_response(response, request) - def _with_client_facing_response_id( + def _as_served_response( + self, response: _LoggedResponse + ) -> _LoggedResponse | ModelResponse | TextCompletionResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): + return response + served: Final = self._bridged_chat_response(response, request) + if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped + return served + return self._as_text_completion_priced_as_chat(served) + + def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse: + text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion( + served, TextCompletionResponse() + ) + set_hidden_params(text_completion, dict(get_hidden_params(served) or {})) + return text_completion + + def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse: + translated: Final = self._translate_responses_api_response_to_model_response(response, request) + served: Final = translated.model_copy( + update={ + "id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id) + } + ) + set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})}) + return served + + def _with_response_id( self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + response_id: str | None, ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: - response_id: Final = self.client_facing_response_id - if response_id is None or getattr(response, "id", None) == response_id: + if response_id is None or response.id == response_id: return response return response.model_copy(update={"id": response_id}) @@ -3252,8 +3248,13 @@ class Logging(LiteLLMLoggingBaseClass): self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: - return self._with_client_facing_response_id( - self._with_continued_response_id(self._with_client_facing_stream_model(response)) + continued_response_id: Final = self.model_call_details.get("continued_response_id") + return self._with_response_id( + self._with_response_id( + self._with_client_facing_stream_model(response), + continued_response_id if isinstance(continued_response_id, str) else None, + ), + self.client_facing_response_id, ) def _success_handler_helper_fn( @@ -3311,6 +3312,13 @@ class Logging(LiteLLMLoggingBaseClass): ) result = self.normalize_endpoint_logging_result(result) + if self.stream is not True: + result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response + if isinstance(result, VideoObject): + self.model_call_details[POLLED_VIDEO_STATUS_KWARG] = result.status + if isinstance(result, LiteLLMBatch): + self.model_call_details[POLLED_BATCH_ID_KWARG] = result.id + self.model_call_details[POLLED_BATCH_COST_IS_FINAL_KWARG] = batch_cost_is_final(result) logging_result: Final = self.normalize_logging_result(result=result) if cache_hit is None and _served_from_cache_at_no_charge(logging_result): @@ -3625,6 +3633,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result, request_logging_payload = self.prepare_success_logging( start_time=start_time, end_time=end_time, @@ -3639,7 +3648,12 @@ class Logging(LiteLLMLoggingBaseClass): try: ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: ( - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None ) = None if "complete_streaming_response" in self.model_call_details and not for_async_request: return # break out of this. @@ -3655,17 +3669,20 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_streaming_response"] = complete_streaming_response self._surface_response_headers_from_result(complete_streaming_response) self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=complete_streaming_response if provider_transcription is None else provider_transcription ) - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["complete_streaming_response"] = logged_streaming_response - self.capture_usage_before_logging(logged_streaming_response, start_time, end_time) ## STANDARDIZED LOGGING PAYLOAD if not for_async_request: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get( "standard_logging_object" @@ -4185,13 +4202,12 @@ class Logging(LiteLLMLoggingBaseClass): cost_for_built_in_tools_cost_usd_dollar=0.0, ) - self.capture_usage_before_logging(result, start_time, end_time) - self.truncated_messages_for_logging = await truncate_base64_in_messages_async( StandardLoggingPayloadSetup.append_system_prompt_messages( kwargs=self.model_call_details, messages=self.model_call_details.get("messages") ) ) + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result = self._success_handler_helper_fn( start_time=start_time, end_time=end_time, @@ -4211,9 +4227,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Final[ - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None - ] = self._get_assembled_streaming_response( + complete_streaming_response: Final = self._get_assembled_streaming_response( result=result, start_time=start_time, end_time=end_time, @@ -4235,7 +4249,7 @@ class Logging(LiteLLMLoggingBaseClass): get_base_model_from_metadata(model_call_details=self.model_call_details) # base_model defaults to None if not set on model_info self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=complete_streaming_response if provider_transcription is None else provider_transcription ) verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) @@ -4250,19 +4264,23 @@ class Logging(LiteLLMLoggingBaseClass): self.model, ) self.model_call_details["response_cost"] = None - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD try: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release) verbose_logger.exception( @@ -5132,10 +5150,17 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: list[object], - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None: + ) -> ( + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None + ): if self.stream is not True: return None - if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse): + if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)): return result elif isinstance( result, @@ -5206,6 +5231,13 @@ class Logging(LiteLLMLoggingBaseClass): row is built from this response rather than re-parsed from the served body.""" self.bridged_model_response = response + def record_bridged_chat_request( + self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> None: + self.bridged_chat_request = BridgedChatRequest( + MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params)) + ) + def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None: self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response)) @@ -5277,6 +5309,9 @@ class Logging(LiteLLMLoggingBaseClass): ): result = result.response if isinstance(result, ResponsesAPIResponse): + bridged_request: Final = self.bridged_chat_request + if bridged_request is not None: + return self._bridged_chat_response(result, bridged_request) return self._translate_responses_api_response_to_model_response(result) raw_body: Final[object] = result @@ -5328,7 +5363,9 @@ class Logging(LiteLLMLoggingBaseClass): set_response_cost_in_hidden_params(result, reported_cost) return result - def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse: + def _translate_responses_api_response_to_model_response( + self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None + ) -> ModelResponse: """ Convert a Responses API response into a ModelResponse for spend_logs. @@ -5351,8 +5388,8 @@ class Logging(LiteLLMLoggingBaseClass): logging_obj=self, request_data={}, messages=[], - optional_params={}, - litellm_params={}, + optional_params={} if request is None else dict(request.optional_params), + litellm_params={} if request is None else dict(request.litellm_params), encoding=litellm.encoding, ) translated.id = served_id or translated.id @@ -6306,7 +6343,7 @@ def _init_custom_logger_compatible_class( logging_integration: str, internal_usage_cache: DualCache | None, llm_router: object, - custom_logger_init_args: dict[str, object] | None = {}, + custom_logger_init_args: dict[str, object] | None = None, raise_import_errors: bool = False, ) -> CustomLogger | None: """ @@ -7106,6 +7143,7 @@ class StandardLoggingPayloadSetup: init_response_obj: object, api_base: str | None = None, router_model_id: str | None = None, + custom_model_info: ModelInfo | None = None, ) -> StandardLoggingModelInformation: model_cost_name: Final = select_model_name_for_cost_calc( model=base_model if custom_pricing else None, @@ -7124,6 +7162,8 @@ class StandardLoggingPayloadSetup: if router_model_id is not None else model_cost_name ) + if model_cost_name is not None and custom_model_info is not None: + return StandardLoggingModelInformation(model_map_key=model_cost_name, model_map_value=custom_model_info) if model_cost_name is None: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: @@ -7720,6 +7760,20 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown: ) +def _with_failed_attempts_cost(breakdown: CostBreakdown, own_cost: float, failed_attempts_cost: float) -> CostBreakdown: + return breakdown | CostBreakdown( + total_cost=own_cost + failed_attempts_cost, + additional_costs={**breakdown.get("additional_costs", {}), "failed_attempts_cost": failed_attempts_cost}, + ) + + +def _cached_cost_breakdown(served: CostBreakdown | None, failed_attempts: tuple[ServedRound, ...]) -> CostBreakdown: + cached: Final = _cache_hit_cost_breakdown(served) + if not failed_attempts: + return cached + return _with_failed_attempts_cost(cached, 0.0, sum(cost for _, cost in failed_attempts)) + + def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None: """Duck-typed logging objects (and test doubles) carry no finalized cost.""" finalized: Final = getattr(logging_obj, "finalized_cost", None) @@ -7860,6 +7914,7 @@ def get_standard_logging_object_payload( init_response_obj=init_response_obj, api_base=litellm_params.get("api_base"), router_model_id=logging_obj.get_router_model_id(), + custom_model_info=logging_obj.custom_model_info if custom_pricing else None, ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( @@ -7913,7 +7968,7 @@ def get_standard_logging_object_payload( if finalized_cost is not None else cost_breakdown_with_guardrail( ( - _cache_hit_cost_breakdown(uncached_price.cost_breakdown) + _cached_cost_breakdown(uncached_price.cost_breakdown, logging_obj.failed_attempts_billed_on_success) if uncached_price is not None else logging_obj.cost_breakdown ), diff --git a/litellm/litellm_core_utils/llm_cost_calc/finalization.py b/litellm/litellm_core_utils/llm_cost_calc/finalization.py index a13134ada4a..756c53ea882 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/finalization.py +++ b/litellm/litellm_core_utils/llm_cost_calc/finalization.py @@ -17,6 +17,7 @@ class PreparedCost: amount: float payload: StandardLoggingPayload usage: Usage + price_known: bool = True @dataclass(frozen=True, slots=True) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 54213f40ac0..e3aecda1deb 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -224,7 +224,20 @@ class StandardBuiltInToolCostTracking: web_search_options=standard_built_in_tools_params.get("web_search_options", None), model_info=model_info, ) - return per_call_cost * StandardBuiltInToolCostTracking._count_web_search_calls(response_object) + return per_call_cost * StandardBuiltInToolCostTracking._billable_web_search_calls( + resolved_usage, response_object + ) + + @staticmethod + def _billable_web_search_calls(usage: Usage | None, response_object: object) -> int: + """ + Providers without their own calculator but whose usage carries server_tool_use.web_search_requests + (OpenAI-compatible providers that report a per-tool count) are billed that many calls on the per-call path. + """ + reported: Final = None if usage is None else get_web_search_requests_from_usage(usage) + if isinstance(reported, int) and not isinstance(reported, bool) and reported > 0: + return reported + return StandardBuiltInToolCostTracking._count_web_search_calls(response_object) @staticmethod def _count_web_search_calls(response_object: object) -> int: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 50c26f4d4ca..9ac4f9d5ee8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] + + cached_text_tokens: ReadOnly[int] + cached_audio_tokens: ReadOnly[int] + cached_image_tokens: ReadOnly[int] + has_cached_tokens_details: ReadOnly[bool] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, cache_hit_audio_tokens=cached_audio_tokens, + cached_text_tokens=cached_text_tokens, + cached_audio_tokens=cached_audio_tokens, + cached_image_tokens=cached_image_tokens, + has_cached_tokens_details=cached_tokens_details is not None, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -1194,21 +1203,55 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu ) +def get_modality_token_cost_key( + model_info: ModelInfo, + cost_key: str, + prompt_tokens: int, + *, + threshold_is_inclusive: bool = False, +) -> str: + suffix: Final = next((suffix for suffix in _NON_STANDARD_THRESHOLD_SUFFIXES if cost_key.endswith(suffix)), "") + base_key: Final = cost_key.removesuffix(suffix) + threshold_key: Final = max( + ( + key + for key in model_info + if key.startswith(f"{base_key}_above_") + and key.endswith(("_tokens", f"_tokens{suffix}")) + and model_info.get(key) is not None + and _prompt_exceeds_threshold(prompt_tokens, _parse_above_token_threshold(key), threshold_is_inclusive) + ), + key=_parse_above_token_threshold, + default=None, + ) + if threshold_key is None: + return cost_key + return f"{threshold_key.removesuffix(suffix)}{suffix}" + + def _modality_token_cost( model_info: ModelInfo, modality_cost_key: str, tokens: int, prompt_base_cost: float, service_tier: str | None, + *, + prompt_tokens: int = 0, + threshold_is_inclusive: bool = False, ) -> float: """ Audio, image and video prompt tokens bill at their modality rate when the model has one. Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which `_get_token_base_cost` has already resolved for the service tier and long-context tier. """ - tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier) - cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key - if model_info.get(cost_key) is None: + cost_key: Final = get_modality_token_cost_key( + model_info, + get_service_tier_cost_key(modality_cost_key, service_tier), + prompt_tokens, + threshold_is_inclusive=threshold_is_inclusive, + ) + standard_key: Final = cost_key.removesuffix(get_service_tier_cost_key("", service_tier)) + if model_info.get(cost_key) is None and model_info.get(standard_key) is None: return float(tokens) * prompt_base_cost return calculate_cost_component(model_info, cost_key, tokens) @@ -1221,6 +1264,9 @@ def _calculate_input_cost( cache_creation_cost: float, cache_creation_cost_by_ttl: Mapping[str, float], service_tier: str | None = None, + *, + prompt_tokens: int = 0, + threshold_is_inclusive: bool = False, ) -> float: """ Calculates the input cost for a given model, prompt tokens, and completion tokens. @@ -1234,15 +1280,11 @@ def _calculate_input_cost( prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + prompt_cost += _calculate_cache_read_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + cache_read_cost=cache_read_cost, + service_tier=service_tier, ) ### AUDIO COST @@ -1255,6 +1297,8 @@ def _calculate_input_cost( prompt_tokens_details["audio_tokens"], prompt_base_cost, service_tier, + prompt_tokens=prompt_tokens, + threshold_is_inclusive=threshold_is_inclusive, ) ### IMAGE TOKEN COST @@ -1331,6 +1375,38 @@ def _calculate_input_cost( return prompt_cost +def _calculate_cache_read_cost( + prompt_tokens_details: PromptTokensDetailsResult, + model_info: ModelInfo, + cache_read_cost: float, + service_tier: str | None, +) -> float: + cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"] + cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"] + cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"] + classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens + unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0) + total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost + float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost + ) + + if cached_audio_tokens: + cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier) + cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached audio contributes to cache-read cost + float(cached_audio_tokens) * float(cached_audio_cost or 0.0) + ) + + if cached_image_tokens: + cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier) + cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached images contribute to cache-read cost + float(cached_image_tokens) * float(cached_image_cost or 0.0) + ) + + return total_cost + + def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1365,6 +1441,29 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | _get_regional_uplift_multiplier = get_regional_uplift_multiplier +def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: + """ + Resolve the per-model uplift multiplier for the OpenAI FedRAMP endpoint. + + OpenAI applies a flat percentage uplift (e.g. +10%) on all token costs for + requests served from gov.api.openai.com. The multiplier is stored on the + model entry as ``fedramp_uplift_multiplier`` (e.g. 1.10). + + Returns 1.0 (no uplift) unless ``data_residency`` is ``"fedramp"`` and the + model has a multiplier configured. + """ + if data_residency is None or data_residency.lower() != DataResidency.FEDRAMP.value: + return 1.0 + multiplier: Final = model_info.get("fedramp_uplift_multiplier") + if multiplier is None: + return 1.0 + try: + return float(multiplier) + except (TypeError, ValueError): + verbose_logger.exception("Invalid fedramp_uplift_multiplier for model; defaulting to 1.0") + return 1.0 + + def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and @@ -1474,8 +1573,8 @@ def generic_cost_per_token( Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information - - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), - used to apply the per-model regional-processing uplift multiplier. + - data_residency: optional OpenAI data-residency region (e.g. "eu", "us") or + "fedramp", used to apply the per-model regional-processing or FedRAMP uplift multiplier. - vertex_location: optional Vertex AI location the request was served from (e.g. "us-east5", "global"), used to apply the per-model regional-endpoint uplift multiplier when non-global. @@ -1502,6 +1601,10 @@ def generic_cost_per_token( prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, cache_hit_audio_tokens=0, + cached_text_tokens=0, + cached_audio_tokens=0, + cached_image_tokens=0, + has_cached_tokens_details=False, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1577,6 +1680,8 @@ def generic_cost_per_token( cache_creation_cost=base_costs.cache_creation_rate, cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, service_tier=service_tier, + prompt_tokens=usage.prompt_tokens, + threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) ## CALCULATE OUTPUT COST @@ -1634,11 +1739,9 @@ def generic_cost_per_token( ## IMAGE COST if not is_text_tokens_total and image_tokens and image_tokens > 0: - _output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None) - _output_cost_per_image_token = ( - _output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost - ) - completion_cost += float(image_tokens) * _output_cost_per_image_token + image_cost_key: Final = get_service_tier_cost_key("output_cost_per_image_token", service_tier) + image_rate: Final = get_cost_per_unit(resolved_model_info, image_cost_key, None) + completion_cost += float(image_tokens) * (completion_base_cost if image_rate is None else image_rate) ## VIDEO COST if not is_text_tokens_total and video_tokens and video_tokens > 0: @@ -1648,10 +1751,9 @@ def generic_cost_per_token( ) completion_cost += float(video_tokens) * _output_cost_per_video_token - ## REGIONAL DATA-RESIDENCY UPLIFT - # Applied as a flat multiplier across all token costs for the request - # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com). - uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency) + uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency) * get_fedramp_uplift_multiplier( + resolved_model_info, data_residency + ) if uplift != 1.0: prompt_cost *= uplift completion_cost *= uplift @@ -1680,6 +1782,7 @@ class BilledTokenRates: cache_creation_input_token_cost: float cache_creation_input_token_cost_by_ttl: Mapping[str, float] output_cost_per_reasoning_token: float + cache_read_input_image_token_cost: float | None = None def scaled(self, multiplier: float) -> "BilledTokenRates": if multiplier == 1.0: @@ -1694,6 +1797,11 @@ class BilledTokenRates: {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()} ), output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + cache_read_input_image_token_cost=( + self.cache_read_input_image_token_cost * multiplier + if self.cache_read_input_image_token_cost is not None + else None + ), ) @@ -1788,6 +1896,7 @@ def _cost_map_billed_rates( ) multiplier: Final = ( get_regional_uplift_multiplier(model_info, data_residency) + * get_fedramp_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(model_info, vertex_location) * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) @@ -1802,6 +1911,11 @@ def _cost_map_billed_rates( cache_creation_input_token_cost=base_costs.cache_creation_rate, cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, output_cost_per_reasoning_token=reasoning_rate, + cache_read_input_image_token_cost=_get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier), + None, + ), ).scaled(multiplier) @@ -1882,6 +1996,12 @@ def get_token_type_cost_breakdown( cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( usage ) + cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"] + image_cache_read_rate: Final = ( + rates.cache_read_input_image_token_cost + if rates.cache_read_input_image_token_cost is not None + else rates.cache_read_input_token_cost + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1895,8 +2015,9 @@ def get_token_type_cost_breakdown( return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + + float(cached_image_tokens) * image_cache_read_rate ), cache_creation_cost=cache_creation_cost, rates=rates, @@ -1961,8 +2082,10 @@ def calculate_prompt_caching_savings( {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} ), ) - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( - model_info, vertex_location + uplift: Final = ( + get_regional_uplift_multiplier(model_info, data_residency) + * get_fedramp_uplift_multiplier(model_info, data_residency) + * get_vertex_regional_endpoint_uplift(model_info, vertex_location) ) return (read_discount - write_premium) * uplift diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 6bb036e9080..08ab2d1a2e5 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -970,9 +970,11 @@ def convert_to_model_response_object( model_response_object.data = response_object["data"] if "usage" in response_object and response_object["usage"] is not None: - model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) - model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) - model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) + embedding_usage: Final = model_response_object.usage or Usage() + embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) + embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) + embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0) + model_response_object.usage = embedding_usage if start_time is not None and end_time is not None: model_response_object._response_ms = ( @@ -1015,6 +1017,10 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) + detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages")) + if detected_languages is not None: + model_response_object.languages = detected_languages + if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None @@ -1075,4 +1081,8 @@ def convert_to_model_response_object( end_time=end_time, convert_tool_call_to_json_mode=convert_tool_call_to_json_mode, ) - raise Exception(f"Invalid response object {traceback.format_exc()}\n\nreceived_args={received_args}") + # The conversion's own traceback, without the errors being handled when it ran: a router + # retry runs inside the handler of the attempt before it, and that attempt's error text + # (a rate limit) would otherwise decide how this one is classified. + own_traceback: Final = "".join(traceback.format_exception(type(e), e, e.__traceback__, chain=False)) + raise Exception(f"Invalid response object {own_traceback}\n\nreceived_args={received_args}") diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index b08596e5244..3e82077f9d6 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from functools import lru_cache -from typing import ClassVar, Final +from typing import ClassVar, Final, get_args from openai.types.chat.completion_create_params import ( CompletionCreateParamsNonStreaming, @@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams from litellm.types.rerank import RerankRequest @@ -178,6 +179,7 @@ class ModelParamHelper: This follows the OpenAI API Spec """ + litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams)) try: from openai.types.audio.transcription_create_params import ( TranscriptionCreateParamsNonStreaming, @@ -187,11 +189,11 @@ class ModelParamHelper: non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys()) streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys()) - all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs) + all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs) return all_transcription_kwargs except Exception as e: verbose_logger.debug("Error getting transcription kwargs %s", str(e)) - return set() + return litellm_transcription_kwargs @staticmethod def _get_litellm_supported_responses_api_kwargs() -> set[str]: diff --git a/litellm/litellm_core_utils/pattern_match_deployments.py b/litellm/litellm_core_utils/pattern_match_deployments.py index 2b8d1e9c11b..a9091348c50 100644 --- a/litellm/litellm_core_utils/pattern_match_deployments.py +++ b/litellm/litellm_core_utils/pattern_match_deployments.py @@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching import copy import itertools import re +from collections.abc import Mapping from re import Match from typing import Final @@ -22,6 +23,16 @@ class _RoutedDeployment(BaseModel): _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...]) +_PATTERN_WIDE_CACHE_FLAGS: Final = frozenset(("supports_prompt_cache_breakpoint", "supports_prompt_caching")) +_AZURE_AI_PATTERN_CACHE_FLAGS: Final = frozenset(("supports_prompt_caching",)) + + +def _cache_flags_left_to_the_catalog(litellm_params: Mapping[str, object]) -> frozenset[str]: + model: Final = litellm_params.get("model") + provider: Final = litellm_params.get("custom_llm_provider") or inferred_provider( + model if isinstance(model, str) else None + ) + return _AZURE_AI_PATTERN_CACHE_FLAGS if provider == "azure_ai" else _PATTERN_WIDE_CACHE_FLAGS class PatternUtils: @@ -133,6 +144,11 @@ class PatternMatchRouter: matched_pattern=matched_pattern, litellm_deployment_litellm_model=deployment["litellm_params"]["model"], ) + if "*" in deployment["litellm_params"]["model"] and isinstance(new_deployment.get("model_info"), dict): + left_to_the_catalog = _cache_flags_left_to_the_catalog(deployment["litellm_params"]) + new_deployment["model_info"] = { + key: value for key, value in new_deployment["model_info"].items() if key not in left_to_the_catalog + } new_deployments.append(new_deployment) return new_deployments diff --git a/litellm/litellm_core_utils/per_call_pricing.py b/litellm/litellm_core_utils/per_call_pricing.py new file mode 100644 index 00000000000..686639dc10f --- /dev/null +++ b/litellm/litellm_core_utils/per_call_pricing.py @@ -0,0 +1,104 @@ +"""The price entry for a single call that sets its own custom pricing, built without writing ``litellm.model_cost``.""" + +from collections.abc import Mapping +from typing import Final, Literal + +from pydantic import TypeAdapter + +import litellm +from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing +from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfo + +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_MODEL_INFO: Final = TypeAdapter(ModelInfo) + + +def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + """A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that + deployment's full custom pricing under that id when the deployment is added.""" + for metadata_key in ("litellm_metadata", "metadata"): + metadata = kwargs.get(metadata_key) + if not isinstance(metadata, Mapping): + continue + model_info = _STR_MAPPING.validate_python(metadata).get("model_info") + if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str): + return _STR_MAPPING.validate_python(model_info) + return None + + +def per_call_model_info( + model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"] +) -> ModelInfo | None: + """The full price entry for one call, or None when the cost map (or the Router's registered deployment entry) + already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with + the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices. + The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the + same model name is priced from the cost map again.""" + deployment: Final = _router_deployment_model_info(kwargs) + registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None + custom: Final = { + field: kwargs[field] + for field in CustomPricingLiteLLMParams.model_fields + if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field]) + } + if not custom: + return None + unmapped: Final[Mapping[str, object]] = { + "key": model, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": custom_llm_provider, + "mode": mode, + "supported_openai_params": None, + } + row: Final = _catalog_row(model, custom_llm_provider) or unmapped + catalog: Final = ( + {**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row + ) + zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None + return _MODEL_INFO.validate_python( + { + **catalog, + **_token_rates_cleared_for_per_second_overrides(custom), + **custom, + **_tiers_with_flat_overrides(catalog, custom), + **(zeroed or {}), + "mode": catalog.get("mode") or mode, + } + ) + + +_FLAT_TOKEN_RATE_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token") +_PER_SECOND_RATE_FIELDS: Final = ("cost_per_second", "input_cost_per_second", "output_cost_per_second") +_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _token_rates_cleared_for_per_second_overrides(custom: Mapping[str, object]) -> Mapping[str, object]: + """Token rates in the entry make the cost calculator skip per-second pricing, so a caller who sets only a + per-second rate gets the catalog's token rates and tier table cleared for this call.""" + if not any(key in custom for key in _PER_SECOND_RATE_FIELDS) or any( + key in custom for key in _FLAT_TOKEN_RATE_FIELDS + ): + return {} + return {"input_cost_per_token": None, "output_cost_per_token": None, "tiered_pricing": None} + + +def _tiers_with_flat_overrides(catalog: Mapping[str, object], custom: Mapping[str, object]) -> Mapping[str, object]: + """A ``tiered_pricing`` table wins over flat rates in the cost calculator, so a flat rate the caller set is written + into every tier of the table in force (the caller's own, else the inherited one) for that direction, and the other + direction keeps each tier's own rate.""" + overrides: Final = {key: custom[key] for key in _FLAT_TOKEN_RATE_FIELDS if key in custom} + tiers: Final = custom.get("tiered_pricing", catalog.get("tiered_pricing")) + if not overrides or not isinstance(tiers, (list, tuple)) or not tiers: + return {} + return {"tiered_pricing": [{**tier, **overrides} for tier in _TIERS.validate_python(tiers)]} + + +def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None: + try: + return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6287dc17e8f..f48a42963cf 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.batch_utils import InMemoryFile from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAudioObject, + ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, ChatCompletionPromptCacheBreakpoint, @@ -397,6 +399,74 @@ def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, objec return projection.content, projection.media +_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/") + + +def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower() + if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES): + return () + return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},) + + +_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}") + + +def _is_base64_data_uri(value: str) -> bool: + header_end: Final = value.find(",", 0, 256) + if header_end < 0 or not value[:header_end].endswith(";base64"): + return False + payload_start: Final = header_end + 1 + if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None: + return False + last_quantum: Final = value[max(payload_start, len(value) - 4) :] + return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum + + +def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection: + projection: Final = _tpm_file_media(part) + fields: Final = as_str_mapping(part.get("file")) + data: Final = fields.get("file_data") if fields is not None else None + if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data): + return projection + media: Final = _tpm_embedding_file(data) + return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media) + + +def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is not None and part.get("type") == "file": + return _tpm_embedding_file_block(part) + if not isinstance(value, str) or not value.startswith("data:"): + return _tpm_media_leaf(value) + if not _is_base64_data_uri(value): + return _TPMMediaProjection(value) + media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value) + return _TPMMediaProjection("" if media else value, media) + + +def _is_json_array(value: object) -> bool: + return isinstance(value, (list, tuple)) + + +def _tpm_embedding_item(value: object) -> _TPMMediaProjection: + return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value) + + +def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = ( + _tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value) + ) + return projection.content, projection.media + + +def embedding_input_is_all_media(value: object) -> bool: + items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array + return all(_tpm_embedding_leaf(item).media for item in items) + + def _content_parts_contain_image(parts: Sequence[object]) -> bool: """Depth-bounded frontier walk over nested content lists, iterative because the repo bans recursion; an Anthropic tool_result nests its image parts exactly one level down.""" @@ -2183,6 +2253,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: _MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) +PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( + "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" +) def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: @@ -2262,6 +2335,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. + A ``cache_control`` the client already wrote on the block wins, TTL included.""" + marker: Final = block.get("prompt_cache_breakpoint") + if marker is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + control: Final = ( + None + if block.get("cache_control") is not None + else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) + ) + marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: + try: + breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + if not drop_params: + raise UnsupportedParamsError( + message=( + f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " + f"{marker!r}. Fix the marker, or " + f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + ttl: Final = breakpoint_mark.get("ttl") + if ttl is not None and not drop_params: + raise UnsupportedParamsError( + message=( + f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " + f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " + "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " + "where the model offers a 1h cache." + ), + status_code=400, + ) + return ChatCompletionCachedContent(type="ephemeral") + + _BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d6d753635f6..4c3907dc52f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -77,7 +77,7 @@ from .common_utils import ( parse_tool_call_arguments, salvage_concatenated_tool_arguments, ) -from .image_handling import convert_url_to_base64 +from .image_handling import convert_url_to_base64, is_pdf_media_type def default_pt(messages): @@ -911,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> ) +def _anthropic_https_url_block( + image_url: str, image_format: str | None +) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: + source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url) + if image_format and is_pdf_media_type(image_format): + return AnthropicMessagesDocumentParam(type="document", source=source) + return AnthropicMessagesImageParam(type="image", source=source) + + def create_anthropic_image_param( - image_url_input: str | dict, + image_url_input: str | Mapping[str, object], format: str | None = None, is_bedrock_invoke: bool = False, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: @@ -921,13 +930,11 @@ def create_anthropic_image_param( Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. """ - # Extract URL and format from input - if isinstance(image_url_input, str): - image_url = image_url_input - else: - image_url = image_url_input.get("url", "") - if format is None: - format = image_url_input.get("format") + url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url") + format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format") + image_url: Final = url_value if isinstance(url_value, str) else "" + explicit_format: Final = format if format is not None else format_value + image_format: Final = explicit_format if isinstance(explicit_format, str) else None # Check if the image URL is an HTTP/HTTPS URL if image_url.startswith("http://") or image_url.startswith("https://"): @@ -935,21 +942,14 @@ def create_anthropic_image_param( # as these providers don't support URL sources for images if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) - image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) - return _anthropic_content_element_factory(image_chunk) + image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format) + return anthropic_content_element_factory(image_chunk) else: - # HTTPS URL - pass directly for regular Anthropic - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSourceUrl( - type="url", - url=image_url, - ), - ) + return _anthropic_https_url_block(image_url, image_format) else: # Convert to base64 for data URIs or other formats - image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) - return _anthropic_content_element_factory(image_chunk) + image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format) + return anthropic_content_element_factory(image_chunk) # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex @@ -2255,16 +2255,16 @@ def add_cache_control_to_content( return anthropic_content_element -def _anthropic_content_element_factory( +def anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: - if image_chunk["media_type"] == "application/pdf": + if is_pdf_media_type(image_chunk["media_type"]): _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = ( AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type="application/pdf", data=image_chunk["data"], ), ) @@ -4254,6 +4254,7 @@ def _convert_to_bedrock_tool_call_invoke( tool_calls: list, model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockContentBlock]: """ OpenAI tool invokes: @@ -4338,6 +4339,7 @@ def _convert_to_bedrock_tool_call_invoke( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts_list.append(_cache_point_block) @@ -4358,6 +4360,7 @@ def _convert_to_bedrock_tool_call_invoke( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: _parts_list.append(cache_point_block) @@ -5029,6 +5032,7 @@ class BedrockConverseMessagesProcessor: deployment_model: str | None = None, base_model: str | None = None, drop_params: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockMessageBlock]: from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking @@ -5091,6 +5095,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -5098,7 +5103,11 @@ class BedrockConverseMessagesProcessor: elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model, deployment_model=deployment_model + message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) user_content.append(_part) if _cache_point_block is not None: @@ -5149,6 +5158,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -5214,6 +5224,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None and not ( drop_params and _as_thinking_block(element) is not None @@ -5231,6 +5242,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) @@ -5239,7 +5251,7 @@ class BedrockConverseMessagesProcessor: if _tool_calls: assistant_content.extend( _convert_to_bedrock_tool_call_invoke( - _tool_calls, model=model, deployment_model=deployment_model + _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info ) ) @@ -5434,6 +5446,7 @@ def _bedrock_converse_messages_pt( deployment_model: str | None = None, base_model: str | None = None, drop_params: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockMessageBlock]: """ Converts given messages from OpenAI format to Bedrock format @@ -5502,6 +5515,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -5509,7 +5523,11 @@ def _bedrock_converse_messages_pt( elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model, deployment_model=deployment_model + message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) user_content.append(_part) if _cache_point_block is not None: @@ -5562,6 +5580,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -5628,6 +5647,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None and not ( drop_params and _as_thinking_block(element) is not None @@ -5640,14 +5660,20 @@ def _bedrock_converse_messages_pt( assistant_content.append(BedrockContentBlock(text=_assistant_content)) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model + assistant_message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: assistant_content.extend( - _convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model) + _convert_to_bedrock_tool_call_invoke( + _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info + ) ) msg_i += 1 @@ -5703,7 +5729,10 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str: def add_cache_point_tool_block( - tool: dict, model: str | None = None, deployment_model: str | None = None + tool: dict, + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> BedrockToolBlock | None: from litellm.llms.bedrock.common_utils import ( bedrock_cache_point_ttl_supported, @@ -5711,7 +5740,7 @@ def add_cache_point_tool_block( ) cache_control: Final = tool.get("cache_control", None) - if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model): + if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model, model_info): cache_point: Final = cache_control.get("type", "ephemeral") if cache_point == "ephemeral": cache_point_block: Final[CachePointBlock] = {"type": "default"} @@ -5748,7 +5777,10 @@ def _is_bedrock_tool_block(tool: dict) -> bool: def _bedrock_tools_pt( - tools: list, model: str | None = None, deployment_model: str | None = None + tools: list, + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockToolBlock]: """ OpenAI tools looks like: @@ -5869,7 +5901,9 @@ def _bedrock_tools_pt( tool_block_list.append(tool_block) ## ADD CACHE POINT TOOL BLOCK ## - cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model) + cache_point_tool_block = add_cache_point_tool_block( + tool, model=model, deployment_model=deployment_model, model_info=model_info + ) if cache_point_tool_block is not None: tool_block_list.append(cache_point_tool_block) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 3287515c81d..ffc0e82e410 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,28 +5,33 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from itertools import chain +from functools import partial +from itertools import accumulate, chain, takewhile, tee from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar +from urllib.parse import urlparse import httpx -from httpx import Response +from httpx import USE_CLIENT_DEFAULT, Response import litellm from litellm import verbose_logger from litellm.caching.caching import InMemoryCache -from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB +from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get from litellm.types.llms.openai import AllMessageValues MAX_IMGS_IN_MEMORY: Final = 10 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20 +_Fetched = TypeVar("_Fetched") _FETCH_ATTEMPTS: Final = 3 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY) +pdf_probe_verdicts: Final = InMemoryCache() def _header_value(response: Response, name: str) -> str | None: @@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool: return media.part_type == "image_url" +_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"}) +_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"}) +_PDF_SIGNATURE: Final = b"%PDF-" +_PDF_SIGNATURE_WINDOW: Final = 1024 +_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"}) +_PDF_PROBE_STATUSES: Final = frozenset({200, 206}) +_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError) +_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError) +_PDF_PROBE_VERDICT_TTL: Final = 600 +_PDF_PROBE_FAILURE_TTL: Final = 60 +_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"}) + + +def _bare_media_type(media_type: str) -> str: + return media_type.split(";", 1)[0].strip().lower() + + +def is_pdf_media_type(media_type: str) -> bool: + return _bare_media_type(media_type) in _PDF_MEDIA_TYPES + + +def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool: + lowered: Final = url.lower() + return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf")) + + +def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool: + image_format: Final = media.fields.get("format") + return inline_remote_image_urls(media) and is_unformatted_pdf_link( + media.url, image_format if isinstance(image_format, str) else None + ) + + +def _serves_pdf(response: Response) -> bool: + if response.status_code not in _PDF_PROBE_STATUSES: + return False + served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "") + if served_type and served_type not in _GENERIC_MEDIA_TYPES: + return served_type in _PDF_MEDIA_TYPES + return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW] + + +def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]: + reading, counting = tee(chunks) + read_so_far: Final = chain((0,), accumulate(map(len, counting))) + still_reading: Final = takewhile( + lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far + ) + return (chunk for _, chunk in zip(still_reading, reading)) + + +def _with_body_prefix(response: Response, prefix: bytes) -> Response: + return Response( + response.status_code, + headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS], + content=prefix, + request=response.request, + ) + + +@dataclass(frozen=True, slots=True) +class _BodyPrefixReader: + client: httpx.Client + deadline: float + + def get( + self, + url: str, + *, + headers: Mapping[str, str] | None = None, + follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + ) -> Response: + with self.client.stream( + "GET", + url, + headers=headers, + follow_redirects=follow_redirects, + timeout=USE_CLIENT_DEFAULT if timeout is None else timeout, + ) as response: + prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline)) + return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW]) + + +@dataclass(frozen=True, slots=True) +class _AsyncBodyPrefixReader: + client: httpx.AsyncClient + + async def get( + self, + url: str, + *, + headers: Mapping[str, str] | None = None, + follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, + ) -> Response: + async with self.client.stream( + "GET", + url, + headers=headers, + follow_redirects=follow_redirects, + timeout=USE_CLIENT_DEFAULT if timeout is None else timeout, + ) as response: + return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b"")) + + +def _cached_pdf_verdict(url: str) -> bool | None: + cached: Final = pdf_probe_verdicts.get_cache(url) + return cached if isinstance(cached, bool) else None + + +def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool: + pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl) + return verdict + + +def _remember_probe_response(url: str, response: Response) -> bool: + ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL + return _remember_pdf_verdict(url, _serves_pdf(response), ttl) + + +def _failed_pdf_probe(url: str, error: Exception) -> bool: + verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error) + return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL) + + +def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool: + if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0: + return False + cached: Final = _cached_pdf_verdict(url) + if cached is not None: + return cached + try: + response: Final = safe_get( + _BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout), + url, + headers=dict(_PDF_PROBE_HEADERS), + timeout=timeout, + ) + except _PDF_PROBE_FAILURES as e: + return _failed_pdf_probe(url, e) + return _remember_probe_response(url, response) + + +async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool: + if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0: + return False + cached: Final = _cached_pdf_verdict(url) + if cached is not None: + return cached + try: + response: Final = await asyncio.wait_for( + async_safe_get( + _AsyncBodyPrefixReader(litellm.module_level_aclient.client), + url, + headers=dict(_PDF_PROBE_HEADERS), + timeout=timeout, + max_response_bytes=_PDF_SIGNATURE_WINDOW, + ), + timeout, + ) + except _ASYNC_PDF_PROBE_FAILURES as e: + return _failed_pdf_probe(url, e) + return _remember_probe_response(url, response) + + def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None: if fields.get("type") != "image_url": return None @@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call return _inline(remote, data_url) if data_url is not None else part -def _inline_message( - message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool] -) -> AllMessageValues: +def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues: parts: Final = _content_parts(message) if not parts: return message - inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts] - inlined_message: Final = {**message, "content": inlined_parts} - return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined + rewritten: Final = {**message, "content": [rewrite(part) for part in parts]} + return rewritten # pyright: ignore[reportReturnType] # the same message with some of its parts rewritten + + +def _inline_message( + message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool] +) -> AllMessageValues: + return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline)) + + +def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object: + remote: Final = _parse_remote_part(part) + match remote: + case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)): + image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS + return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}} + case _: + return part + + +def _with_pdf_formats( + messages: Iterable[AllMessageValues], + probed: Iterable[tuple[str, bool]], + should_probe: Callable[[RemoteMedia], bool], +) -> tuple[AllMessageValues, ...]: + pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf) + if not pdf_links: + return tuple(messages) + rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe) + return tuple(_with_parts(message, rewrite) for message in messages) async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str: @@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float return await async_convert_url_to_base64(url, timeout=timeout) -async def async_convert_urls_to_base64( - remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None -) -> tuple[str, ...]: - in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls) +async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool: + async with in_flight: + return await async_serves_pdf(url) + + +async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]: try: return tuple(await asyncio.gather(*fetches)) except BaseException: @@ -322,6 +520,15 @@ async def async_convert_urls_to_base64( raise +async def async_convert_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> tuple[str, ...]: + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + return await _gather_or_cancel_all( + tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls) + ) + + def _remote_urls_to_inline( messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] ) -> tuple[str, ...]: @@ -351,3 +558,27 @@ async def async_inline_remote_media( data_urls: Final = await async_convert_urls_to_base64(remote_urls) inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))) return [_inline_message(message, inlined, should_inline) for message in messages] + + +def format_pdf_links( + messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool] +) -> tuple[AllMessageValues, ...]: + links: Final = _remote_urls_to_inline(messages, should_probe) + if not links: + return tuple(messages) + with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes: + verdicts: Final = tuple(probes.map(serves_pdf, links)) + return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe) + + +async def async_format_pdf_links( + messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool] +) -> tuple[AllMessageValues, ...]: + links: Final = _remote_urls_to_inline(messages, should_probe) + if not links: + return tuple(messages) + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + verdicts: Final = await _gather_or_cancel_all( + tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links) + ) + return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 46f5ca87a93..b6af181980e 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,4 +1,6 @@ import asyncio +import base64 +import functools import json import time import traceback @@ -8,7 +10,7 @@ from enum import Enum, auto from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly import litellm @@ -19,9 +21,12 @@ from litellm.constants import ( REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import ( @@ -35,6 +40,8 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDelta, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeTranslationClosedEvent, + OpenAIRealtimeTranslationDurationUsage, ) from litellm.types.realtime import ALL_DELTA_TYPES @@ -42,9 +49,11 @@ from .litellm_logging import Logging as LiteLLMLogging from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason if TYPE_CHECKING: + from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats from websockets.asyncio.client import ClientConnection from websockets.exceptions import ConnectionClosed + from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks CLIENT_CONNECTION_CLASS = ClientConnection @@ -57,6 +66,10 @@ class BackendClose: code: int reason: str + @property + def ended_cleanly(self) -> bool: + return self.code == 1000 + @property def message(self) -> str: if not self.reason: @@ -67,6 +80,7 @@ class BackendClose: class ClientLoopExit(Enum): CLIENT_DISCONNECTED = auto() BACKEND_CLOSED = auto() + RELAY_FAILED = auto() def backend_close_from(error: "ConnectionClosed") -> BackendClose: @@ -96,6 +110,7 @@ class _ClientEventFrame(TypedDict, total=False): type: ReadOnly[str] item: ReadOnly[_ClientEventItem] session: ReadOnly[Mapping[str, object]] + response: ReadOnly[Mapping[str, object]] class _ResponseDoneBody(TypedDict, total=False): @@ -212,6 +227,87 @@ def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, ob return _json_object(container.get(key)) +_BETA_AUDIO_FORMAT_BY_GA_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"} +) +_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"output_modalities": "modalities", "max_output_tokens": "max_response_output_tokens"} +) +_GA_TO_BETA_AUDIO_INPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + { + "format": "input_audio_format", + "transcription": "input_audio_transcription", + "noise_reduction": "input_audio_noise_reduction", + "turn_detection": "turn_detection", + } +) +_GA_TO_BETA_AUDIO_OUTPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"format": "output_audio_format", "voice": "voice", "speed": "speed"} +) +_GA_ONLY_SESSION_FIELDS: Final = frozenset(("type", "audio", *_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS)) + + +def _beta_session_value(ga_field: str, value: object) -> object: + if ga_field == "output_modalities": + return ["audio", "text"] if value == ["audio"] else value + if ga_field != "format": + return value + ga_format: Final = _json_object(value) + format_type: Final = None if ga_format is None else ga_format.get("type") + return _BETA_AUDIO_FORMAT_BY_GA_TYPE.get(format_type, value) if isinstance(format_type, str) else value + + +def _beta_session_fields(ga_fields: Mapping[str, object], beta_names: Mapping[str, str]) -> Mapping[str, object]: + return { + beta_names[name]: _beta_session_value(name, value) for name, value in ga_fields.items() if name in beta_names + } + + +def ga_session_to_beta(session: Mapping[str, object]) -> Mapping[str, object]: + audio: Final = _mapping_field(session, "audio") or _EMPTY_SESSION + return { + **{key: value for key, value in session.items() if key not in _GA_ONLY_SESSION_FIELDS}, + **_beta_session_fields(session, _GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS), + **_beta_session_fields(_mapping_field(audio, "input") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_INPUT_FIELDS), + **_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS), + } + + +_GA_ONLY_RESPONSE_FIELDS: Final = frozenset(("audio", "output_modalities")) +_GA_TO_BETA_RESPONSE_FIELDS: Final[Mapping[str, str]] = MappingProxyType({"output_modalities": "modalities"}) + + +def ga_response_to_beta(response: Mapping[str, object]) -> Mapping[str, object]: + audio: Final = _mapping_field(response, "audio") or _EMPTY_SESSION + return { + **{key: value for key, value in response.items() if key not in _GA_ONLY_RESPONSE_FIELDS}, + **_beta_session_fields(response, _GA_TO_BETA_RESPONSE_FIELDS), + **_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS), + } + + +_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...]) +_GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"text": "output_text", "audio": "output_audio"} +) + + +def _ga_assistant_content_block(block: object) -> object: + content_block: Final = _json_object(block) + block_type: Final = None if content_block is None else content_block.get("type") + if content_block is None or not isinstance(block_type, str): + return block + ga_type: Final = _GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE.get(block_type) + return block if ga_type is None else {**content_block, "type": ga_type} + + +def beta_item_to_ga(item: Mapping[str, object]) -> Mapping[str, object]: + content: Final = item.get("content") + if item.get("role") != "assistant" or not isinstance(content, list): + return item + return {**item, "content": [_ga_assistant_content_block(block) for block in _JSON_ARRAY.validate_python(content)]} + + def _guardrail_event_id() -> str: return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}" @@ -270,6 +366,26 @@ def _gate_session_update( return {**client_session, "turn_detection": gate(None)} +@functools.cache +def _translation_audio_format_adapter() -> "TypeAdapter[str | RealtimeAudioFormats | None]": + from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats + + return TypeAdapter(str | RealtimeAudioFormats | None) + + +def _translation_audio_bytes_per_second(audio_format: "str | RealtimeAudioFormats | None") -> float | None: + from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU + + match audio_format: + case "pcm16" | AudioPCM(): + return 48000.0 + case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA(): + return 8000.0 + case _: + pass + return None + + class RealtimeEventNormalizer(Protocol): def should_drop(self, event: object) -> bool: ... def normalize(self, event: dict) -> dict: ... @@ -294,6 +410,79 @@ DefaultLoggedRealTimeEventTypes: Final = [ ] +TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE: Final = "Translation sessions cannot enforce configured realtime guardrails" + + +def _guardrails_run_for_event_hooks( + request_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], + callbacks: Sequence[object], +) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + + return any( + isinstance(cb, CustomGuardrail) + and any(cb.should_run_guardrail(data={**request_data, "stream": True}, event_type=et) for et in event_hooks) + for cb in callbacks + ) + + +def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[str, object]) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + + return ( + isinstance(callback, CustomGuardrail) + and callback.event_hook is None + and callback.get_disable_global_guardrail(dict(request_data)) is True + ) + + +def _realtime_runner_guardrail( + callback: object, + check_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], +) -> "CustomGuardrail | None": + from litellm.integrations.custom_guardrail import CustomGuardrail + + if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks: + return None + runs: Final = any( + callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks + ) + return callback if runs else None + + +def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + callbacks: Final = tuple( + callback for callback in litellm.callbacks if not _opted_out_of_always_on_guardrail(callback, request_data) + ) + if _guardrails_run_for_event_hooks(request_data, (GuardrailEventHooks.post_call,), callbacks): + return True + if any( + isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket + for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata")) + ): + from litellm.proxy.utils import pipeline_managed_guardrail_names + + if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names( + request_data, "post_call" + ): + return True + return realtime_guardrails_apply(request_data, callbacks) + + +def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Sequence[object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + check_data: Final = {**request_data, "stream": True} + runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call) + return any( + _realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks + ) + + class RealTimeStreaming: def __init__( self, @@ -303,12 +492,15 @@ class RealTimeStreaming: provider_config: BaseRealtimeConfig | None = None, model: str = "", user_api_key_dict: object | None = None, - request_data: dict | None = None, + request_data: Mapping[str, object] | None = None, backend_uses_beta_protocol: bool | None = None, force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + translation_session: bool = False, + bridges_beta_client_to_ga: bool = False, + drop_params: bool = False, lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS, lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, clock: Callable[[], float] = time.monotonic, @@ -326,14 +518,24 @@ class RealTimeStreaming: self._logging_worker = logging_worker self.messages: list[OpenAIRealtimeEvents] = [] self._backend_sent_frames: bool = False + self._backend_sent_non_error_frames: bool = False self.input_message: dict = {} self.input_messages: list[dict[str, str]] = [] self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] + self._is_translation_session = translation_session + self._bridges_beta_client_to_ga: Final = bridges_beta_client_to_ga + self._remaps_all_beta_fields_to_ga: Final = bridges_beta_client_to_ga or translation_session + self._drop_params: Final = drop_params + self._translation_input_seconds = 0.0 + self._translation_input_bytes_per_second = 48000.0 + self._translation_output_seconds = 0.0 + self._translation_output_bytes_per_second = 48000.0 + self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self.detect_beta_header(websocket) - self._backend_uses_beta_protocol = ( + self._backend_uses_beta_protocol = not translation_session and ( self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol ) @@ -352,7 +554,7 @@ class RealTimeStreaming: self.current_delta_type: ALL_DELTA_TYPES | None = None self.session_configuration_request: str | None = None self.user_api_key_dict = user_api_key_dict - self.request_data: dict = request_data or {} + self.request_data: Mapping[str, object] = request_data or {} # Violation counter for end_session_after_n_fails support self._violation_count: int = 0 # When a text message is blocked, hold the guardrail reason so the next @@ -384,6 +586,7 @@ class RealTimeStreaming: # their input_audio_transcription.completed usage drives duration-based cost. self._force_transcription_model = force_transcription_model self._is_transcription_session: bool = force_transcription_model is not None + self._bound_nested_transcription_model: str | None = None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer self._gate_turn_detection: Final[_TurnDetectionGate] = ( @@ -407,6 +610,7 @@ class RealTimeStreaming: _MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) + _RESPONSE_EVENT_TYPES: Final = frozenset(("response.created", "response.done")) _CLIENT_AUDIO_BUFFER_TYPES = frozenset( [ "input_audio_buffer.append", @@ -429,8 +633,8 @@ class RealTimeStreaming: _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"]) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, - "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, - "g711_alaw": {"type": "audio/G711-alaw", "rate": 8000}, + "g711_ulaw": {"type": "audio/pcmu"}, + "g711_alaw": {"type": "audio/pcma"}, } # GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1) _GA_TO_BETA_EVENT_TYPES: dict[str, str] = { @@ -449,12 +653,12 @@ class RealTimeStreaming: def _should_store_message( self, - message_obj: dict | OpenAIRealtimeEvents, + message_obj: Mapping[str, object], ) -> bool: _msg_type: Final = message_obj["type"] if "type" in message_obj else None if self.logged_real_time_event_types == "*": return True - if _msg_type and _msg_type in self.logged_real_time_event_types: + if isinstance(_msg_type, str) and _msg_type in self.logged_real_time_event_types: return True return False @@ -555,7 +759,7 @@ class RealTimeStreaming: input_messages, so it is not duplicated into the response log here. """ try: - usage: Final = event_obj.get("usage") + usage: Final = as_str_mapping(event_obj.get("usage")) if usage is None: return # If this event type is already captured by store_message (e.g. the user @@ -623,6 +827,7 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" + self._finalize_translation_usage() if self.logging_obj: if self._usage_meter is not None: self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot() @@ -631,8 +836,6 @@ class RealTimeStreaming: if self.session_tools or self.tool_calls: self.logging_obj.model_call_details["realtime_tools"] = self.session_tools self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls - if LiteLLMLogging.usage_capture_callback is not None: - self.logging_obj.capture_usage_before_logging(self.messages) # Route through the bounded logging worker (per-coroutine timeout + # concurrency cap) instead of a bare create_task, so a slow callback # can't leave suspended tasks pinning each call's response in memory. @@ -641,6 +844,142 @@ class RealTimeStreaming: ) self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True + def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: + if not self._is_translation_session: + return + if event_obj.get("type") == "session.closed": + usage: Final = as_str_mapping(event_obj.get("usage")) + output_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("output_seconds")) if usage is not None else None + ) + reported_input_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("input_seconds")) if usage is not None else None + ) + input_seconds: Final = ( + reported_input_seconds + if reported_input_seconds is not None + else self._translation_input_seconds or None + ) + synthetic_output_seconds: Final = self._translation_output_seconds or None + resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds + if input_seconds is not None or resolved_output_seconds is not None: + if self._should_store_message(event_obj): + supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0, + output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0, + ) + if ( + supplemental_usage.get("input_seconds", 0.0) > 0 + or supplemental_usage.get("output_seconds", 0.0) > 0 + ): + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=supplemental_usage, + ) + ) + else: + normalized_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=input_seconds, + output_seconds=float(resolved_output_seconds or 0.0), + ) + if input_seconds is not None + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=float(resolved_output_seconds or 0.0) + ) + ) + self.messages.append( + OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage) + ) + self._translation_usage_finalized = True + return + self._capture_translation_audio_formats(event_obj) + if event_obj.get("type") not in ( + "session.output_audio.delta", + "response.output_audio.delta", + "response.audio.delta", + ): + return + delta: Final = event_obj.get("delta") + if not isinstance(delta, str): + return + try: + decoded: Final = base64.b64decode(delta, validate=True) + except (ValueError, TypeError): + return + self._translation_output_seconds += len(decoded) / self._translation_output_bytes_per_second + + def _capture_translation_input_audio(self, message: str) -> None: + if not self._is_translation_session: + return + try: + event: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return + if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): + return + try: + decoded: Final = base64.b64decode(audio, validate=True) + except (ValueError, TypeError): + return + self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + + def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None: + if event_obj.get("type") not in ("session.created", "session.updated"): + return + session: Final = as_str_mapping(event_obj.get("session")) + if session is None: + return + audio: Final = as_str_mapping(session.get("audio")) + if audio is None: + return + for direction, raw_configuration in (("input", audio.get("input")), ("output", audio.get("output"))): + if (configuration := as_str_mapping(raw_configuration)) is None: + continue + try: + if ( + bytes_per_second := _translation_audio_bytes_per_second( + _translation_audio_format_adapter().validate_python(configuration.get("format")) + ) + ) is None: + continue + except ValidationError: + continue + if direction == "input": + self._translation_input_bytes_per_second = bytes_per_second + else: + self._translation_output_bytes_per_second = bytes_per_second + + def _finalize_translation_usage(self) -> None: + if self._translation_usage_finalized: + return + for event in self.messages: + if event.get("type") != "session.closed": + continue + if (event_usage := as_str_mapping(event.get("usage"))) is not None and normalized_audio_duration_seconds( + event_usage.get("output_seconds") + ) is not None: + self._translation_usage_finalized = True + return + if self._translation_output_seconds == 0 and self._translation_input_seconds == 0: + return + synthetic_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=self._translation_input_seconds, + output_seconds=self._translation_output_seconds, + ) + if self._translation_input_seconds > 0 + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=self._translation_output_seconds + ) + ) + self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) + self._translation_usage_finalized = True + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. @@ -653,10 +992,25 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = self._enforce_transcription_session_model(message) + from litellm.proxy._types import ProxyException + + try: + authorized_message: Final = await self._apply_nested_transcription_model_policy(message) + except ProxyException as denied: + denial_message: Final = redact_internal_details_from_client_message(denied.message) + try: + await self.websocket.send_text(realtime_error_event(denial_message, error_type="permission_error")) + finally: + await self.websocket.close( + code=1008, + reason=websocket_close_reason( + denial_message, fallback="Nested transcription model is not authorized" + ), + ) + return False if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( - message, self.model, self.session_configuration_request + authorized_message, self.model, self.session_configuration_request ) sent = False for msg in transformed: @@ -691,14 +1045,94 @@ class RealTimeStreaming: if is_content_message: self._content_sent_after_setup = True sent = True + if sent: + self._capture_translation_input_audio(authorized_message) return sent - await self.backend_ws.send(message) + await self.backend_ws.send(authorized_message) if self._usage_meter is not None: - sent_event: Final = self._parse_backend_event(message) + sent_event: Final = self._parse_backend_event(authorized_message) if sent_event is not None: self._usage_meter.observe_client_event(sent_event) + self._capture_translation_input_audio(authorized_message) return True + async def _apply_nested_transcription_model_policy(self, message: str) -> str: + if self._force_transcription_model is not None: + return self._enforce_transcription_session_model(message) + if self._is_translation_session: + return await self._enforce_translation_nested_transcription_model(message) + return message + + def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None: + try: + message_obj: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return None + if message_obj.get("type") not in ( + "session.update", + "transcription_session.update", + ): + return None + return message_obj + + def _nested_transcription_models_from_session( + self, + session: Mapping[str, object], + ) -> tuple[str, ...]: + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = audio_input.get("transcription") if audio_input is not None else None + nested_model: Final = self._transcription_model_value(nested_transcription) + flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) + return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) + + def _transcription_model_value(self, transcription_config: object) -> str | None: + if (fields := as_str_mapping(transcription_config)) is None: + return None + model: Final = fields.get("model") + if isinstance(model, str) and model: + return model + return None + + def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: + return message + session: Final = as_str_mapping(message_obj.get("session")) + if session is None: + return message + transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + rewrite_flat: Final = transcription is not None and transcription.get("model") != authorized_model + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = ( + as_str_mapping(audio_input.get("transcription")) if audio_input is not None else None + ) + rewrite_nested: Final = ( + nested_transcription is not None and nested_transcription.get("model") != authorized_model + ) + if not rewrite_flat and not rewrite_nested: + return message + updated_session: Final = { + **session, + **( + {"input_audio_transcription": {**transcription, "model": authorized_model}} + if transcription is not None and rewrite_flat + else {} + ), + **( + { + "audio": { + **audio, + "input": {**audio_input, "transcription": {**nested_transcription, "model": authorized_model}}, + } + } + if audio is not None and audio_input is not None and nested_transcription is not None and rewrite_nested + else {} + ), + } + return json.dumps({**message_obj, "session": updated_session}) + def _builds_setup_lazily(self) -> bool: if self.provider_config is None: return False @@ -877,56 +1311,52 @@ class RealTimeStreaming: if self._force_transcription_model is None: return message - try: - message_obj: Final = _decode_json_object(message) - except (json.JSONDecodeError, TypeError): + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message - - if message_obj.get("type") not in ( - "session.update", - "transcription_session.update", - ): - return message - - session: Final = message_obj.get("session") - if not isinstance(session, dict): - return message - - if session.get("type") == "transcription": + session: Final = as_str_mapping(message_obj.get("session")) + if session is not None and session.get("type") == "transcription": self._is_transcription_session = True + return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) - authorized_model: Final = self._force_transcription_model - changed = False + async def _enforce_translation_nested_transcription_model(self, message: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + session: Final = as_str_mapping(message_obj.get("session")) if message_obj is not None else None + bound_model: Final = self._bound_nested_transcription_model + requested_models: Final = ( + tuple(model for model in self._nested_transcription_models_from_session(session) if model != bound_model) + if session is not None + else () + ) + if not requested_models: + return ( + message + if bound_model is None + else self._rewrite_session_update_transcription_model(message, bound_model) + ) - transcription: Final = session.get("input_audio_transcription") - if isinstance(transcription, dict) and transcription.get("model") != authorized_model: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - changed = True + valid_token: Final = self.user_api_key_dict - audio: Final = session.get("audio") - if isinstance(audio, dict): - audio_input: Final = audio.get("input") - if isinstance(audio_input, dict): - nested_transcription: Final = audio_input.get("transcription") - if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model: - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - changed = True + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.proxy_server import llm_model_list, llm_router + from litellm.proxy.realtime_endpoints.endpoints import nested_transcription_upstream_model - if not changed: + if not isinstance(valid_token, UserAPIKeyAuth): return message - return json.dumps(message_obj) + + for requested_model in requested_models: + await can_key_call_resolved_model( + model=requested_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + upstream_model: Final = nested_transcription_upstream_model( + requested_models[0], valid_token.team_id, llm_router + ) + self._bound_nested_transcription_model = upstream_model + return self._rewrite_session_update_transcription_model(message, upstream_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -1047,7 +1477,19 @@ class RealTimeStreaming: return event def _event_to_client_json(self, event: dict) -> str: - return json.dumps(self._normalize_event_for_ga_client(event)) + return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event))) + + def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]: + if not self._bridges_beta_client_to_ga: + return event + event_type: Final = event.get("type") + if event_type in self._SESSION_EVENT_TYPES: + session: Final = _mapping_field(event, "session") + return event if session is None else {**event, "session": ga_session_to_beta(session)} + if event_type in self._RESPONSE_EVENT_TYPES: + response: Final = _mapping_field(event, "response") + return event if response is None else {**event, "response": ga_response_to_beta(response)} + return event async def _send_event_to_client(self, event: object, event_str: str) -> bool: if self._should_drop_event_from_client(event): @@ -1060,7 +1502,7 @@ class RealTimeStreaming: translated: Final = self._translate_event_to_beta(event) if translated is None: return False - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) return True except Exception as e: verbose_logger.warning( @@ -1183,32 +1625,7 @@ class RealTimeStreaming: self, event_hooks: Sequence["GuardrailEventHooks"], ) -> bool: - """Return True if any callback would run for one of ``event_hooks``.""" - from litellm.integrations.custom_guardrail import CustomGuardrail - - return any( - isinstance(cb, CustomGuardrail) - and any( - cb.should_run_guardrail( - data={**self.request_data, "stream": True}, - event_type=et, - ) - for et in event_hooks - ) - for cb in litellm.callbacks - ) - - def _has_realtime_guardrails(self) -> bool: - """Return True if any callback is registered for realtime guardrail event types.""" - from litellm.types.guardrails import GuardrailEventHooks - - return self._has_realtime_guardrails_for_event_hooks( - [ - GuardrailEventHooks.realtime_input_transcription, - GuardrailEventHooks.pre_call, - GuardrailEventHooks.post_call, - ] - ) + return _guardrails_run_for_event_hooks(self.request_data, event_hooks, litellm.callbacks) def _has_audio_transcription_guardrails(self) -> bool: """Return True when a guardrail is configured for the audio/VAD transcript path. @@ -1222,8 +1639,12 @@ class RealTimeStreaming: return self._has_realtime_guardrails_for_event_hooks([GuardrailEventHooks.realtime_input_transcription]) def _should_disable_vad_auto_response(self) -> bool: - """Transcription-only sessions have no assistant turn to gate and reject realtime session updates.""" - return not self._is_transcription_session and self._has_audio_transcription_guardrails() + """Transcription and translation sessions have no assistant turn to gate and reject realtime session updates.""" + return ( + not self._is_transcription_session + and not self._is_translation_session + and self._has_audio_transcription_guardrails() + ) def _refuses_unscreened_audio(self, msg_type: str | None) -> bool: return ( @@ -1264,7 +1685,6 @@ class RealTimeStreaming: transcript completion uses ``realtime_input_transcription`` only; typed user messages and tool outputs use ``pre_call``. """ - from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks if event_hooks is None: @@ -1273,14 +1693,11 @@ class RealTimeStreaming: _check_data: Final = {**self.request_data, "transcript": transcript, "stream": True} _already_run: Final[set] = set() - for callback in litellm.callbacks: - if not isinstance(callback, CustomGuardrail): - continue - if callback.use_native_lifecycle_hooks: - continue - if id(callback) in _already_run: - continue - if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types): + runner_guardrails: Final = ( + _realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks + ) + for callback in runner_guardrails: + if callback is None or id(callback) in _already_run: continue _already_run.add(id(callback)) try: @@ -1369,7 +1786,10 @@ class RealTimeStreaming: async def _handle_provider_config_message(self, raw_response: str) -> None: """Process a backend message when a provider_config is set (transformed path).""" - returned_object: Final = self.provider_config.transform_realtime_response( + provider_config: Final = self.provider_config + if provider_config is None: + raise RuntimeError("Provider response handling requires a provider configuration") + returned_object: Final = provider_config.transform_realtime_response( raw_response, self.model, self.logging_obj, @@ -1398,6 +1818,7 @@ class RealTimeStreaming: continue if isinstance(event, dict): self._detect_transcription_session_from_backend(event) + self._capture_translation_output_audio(event) is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created" if is_session_created_event: if self._uses_deferred_backend_setup() and not self._backend_setup_complete: @@ -1514,6 +1935,7 @@ class RealTimeStreaming: continue if self.provider_config: + self._backend_sent_non_error_frames = True try: await self._handle_provider_config_message(raw_response) except Exception as e: @@ -1521,6 +1943,8 @@ class RealTimeStreaming: continue else: event = self._parse_backend_event(raw_response) + if event is None or event.get("type") != "error": + self._backend_sent_non_error_frames = True if event is None: await self.websocket.send_text(raw_response) continue @@ -1535,6 +1959,7 @@ class RealTimeStreaming: verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event) continue + self._capture_translation_output_audio(event) if await self._handle_raw_backend_message(event, raw_response): continue @@ -1548,7 +1973,7 @@ class RealTimeStreaming: translated = self._translate_event_to_beta(event) if translated is None: continue - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) async def backend_to_client_send_messages(self) -> BackendClose: import websockets @@ -1573,8 +1998,8 @@ class RealTimeStreaming: def _backend_refused_session(self, close: BackendClose) -> bool: return ( - close.code != 1000 - and not self._backend_sent_frames + not self._backend_sent_non_error_frames + and (close.code != 1000 or self._backend_sent_frames) and not self._holds_session_updates_until_first_content() ) @@ -1617,7 +2042,7 @@ class RealTimeStreaming: _detect_beta_header = detect_beta_header @staticmethod - def _remap_beta_session_to_ga(session: dict) -> dict: + def _remap_beta_session_to_ga(session: dict, all_beta_fields: bool = False, drop_temperature: bool = False) -> dict: """ Convert a beta-style session.update payload to the GA nested schema. @@ -1630,6 +2055,11 @@ class RealTimeStreaming: session.output_audio_format → session.audio.output.format (with type/rate) session.turn_detection → session.audio.input.turn_detection session.input_audio_transcription → session.audio.input.transcription + Only with all_beta_fields: + session.input_audio_noise_reduction → session.audio.input.noise_reduction + session.speed → session.audio.output.speed + session.max_response_output_tokens → session.max_output_tokens + session.temperature (dropped when drop_temperature) ───────────────────────────────────────────────────────────────────── Fields not in the mapping (instructions, tools, etc.) are passed through. GA clients that already use the nested shape are unaffected. @@ -1681,6 +2111,19 @@ class RealTimeStreaming: if "input_audio_transcription" in session: inp["transcription"] = session.pop("input_audio_transcription") + if all_beta_fields and "input_audio_noise_reduction" in session: + inp["noise_reduction"] = session.pop("input_audio_noise_reduction") + + if all_beta_fields and "speed" in session: + out["speed"] = session.pop("speed") + + if all_beta_fields and "max_response_output_tokens" in session: + max_tokens: Final = session.pop("max_response_output_tokens") + session.setdefault("max_output_tokens", max_tokens) + + if all_beta_fields and drop_temperature: + session.pop("temperature", None) + if inp: audio["input"] = inp if out: @@ -1699,6 +2142,53 @@ class RealTimeStreaming: return session + @staticmethod + def _remap_beta_response_to_ga(response: Mapping[str, object], drop_temperature: bool) -> dict[str, object]: + moved: Final = frozenset( + ("modalities", "voice", "output_audio_format", "max_response_output_tokens") + + (("temperature",) if drop_temperature else ()) + ) + raw_format: Final = response.get("output_audio_format") + output_audio: Final = { + **({} if "voice" not in response else {"voice": response["voice"]}), + **( + {} + if "output_audio_format" not in response + else { + "format": RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw_format, raw_format) + if isinstance(raw_format, str) + else raw_format + } + ), + } + modalities: Final = response.get("modalities") + modality_names: Final = ( + frozenset(m.lower() for m in modalities if isinstance(m, str)) + if isinstance(modalities, list) + else frozenset() + ) + output_modalities: Final = ( + ["audio"] if "audio" in modality_names else ["text"] if "text" in modality_names else None + ) + existing_audio: Final = as_str_mapping(response.get("audio")) or {} + existing_output: Final = as_str_mapping(existing_audio.get("output")) or {} + return { + **{key: value for key, value in response.items() if key not in moved}, + **( + {} + if output_modalities is None or "output_modalities" in response + else {"output_modalities": output_modalities} + ), + **( + {} + if "max_response_output_tokens" not in response or "max_output_tokens" in response + else {"max_output_tokens": response["max_response_output_tokens"]} + ), + **( + {} if not output_audio else {"audio": {**existing_audio, "output": {**existing_output, **output_audio}}} + ), + } + @staticmethod def _translate_event_to_beta(event: dict) -> dict | None: """Translate a single GA event dict to its beta equivalent. @@ -1932,10 +2422,34 @@ class RealTimeStreaming: if msg_type == "session.update" and not self._backend_uses_beta_protocol: session = client_event.get("session", {}) if isinstance(session, dict): - session = self._remap_beta_session_to_ga(session) + session = self._remap_beta_session_to_ga( + session, + all_beta_fields=self._remaps_all_beta_fields_to_ga, + drop_temperature=self._drop_params, + ) + if self._is_translation_session: + session.pop("type", None) msg_obj["session"] = session message = json.dumps(msg_obj) + if ( + msg_type == "response.create" + and self._remaps_all_beta_fields_to_ga + and (beta_response := as_str_mapping(client_event.get("response"))) is not None + ): + msg_obj["response"] = self._remap_beta_response_to_ga( + beta_response, drop_temperature=self._drop_params + ) + message = json.dumps(msg_obj) + + if ( + msg_type == "conversation.item.create" + and self._bridges_beta_client_to_ga + and (beta_item := as_str_mapping(client_event.get("item"))) is not None + ): + msg_obj["item"] = beta_item_to_ga(beta_item) + message = json.dumps(msg_obj) + if msg_type == "session.update" and self._event_normalizer: session = client_event.get("session") if isinstance(session, dict): @@ -2016,16 +2530,28 @@ class RealTimeStreaming: return ClientLoopExit.BACKEND_CLOSED except Exception as e: verbose_logger.debug("Error in client ack messages: %s", e) - return ClientLoopExit.CLIENT_DISCONNECTED + return ClientLoopExit.RELAY_FAILED async def bidirectional_forward(self) -> None: + if self._is_translation_session and translation_guardrails_apply(self.request_data): + await self.websocket.send_text( + realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error") + ) + await self.websocket.close(code=1008, reason=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) + return forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages()) client_task: Final = asyncio.create_task(self.client_ack_messages()) try: await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED) - if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED: + client_exit: Final = client_task.result() if client_task.done() else None + if client_exit is ClientLoopExit.RELAY_FAILED and self.logging_obj: + self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True + if client_exit in (ClientLoopExit.CLIENT_DISCONNECTED, ClientLoopExit.RELAY_FAILED): return - await self._close_client(await forward_task) + close: Final = await forward_task + if not close.ended_cleanly and self.logging_obj: + self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True + await self._close_client(close) finally: forward_task.cancel() client_task.cancel() @@ -2035,7 +2561,7 @@ class RealTimeStreaming: redacted_message: Final = redact_internal_details_from_client_message(close.message) redacted_reason: Final = redact_internal_details_from_client_message(close.reason) try: - if close.code != 1000: + if not close.ended_cleanly: await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error")) await self.websocket.close( code=client_close_code(close.code), diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 2f8ecdcc9b9..ac5c0720eff 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -10,7 +10,7 @@ import asyncio import copy import inspect -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final @@ -107,7 +107,13 @@ def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger: def _is_redactable_response(response_obj: object) -> bool: return isinstance( - response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse) + response_obj, + ( + litellm.ModelResponse, + litellm.TextCompletionResponse, + litellm.ResponsesAPIResponse, + litellm.EmbeddingResponse, + ), ) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj)) @@ -171,6 +177,8 @@ def _redact_streaming_response(streaming_response): _redact_bedrock_guardrail_trace(streaming_response) elif isinstance(streaming_response, litellm.ResponsesAPIResponse): return perform_redaction({}, streaming_response) + elif isinstance(streaming_response, litellm.TranscriptionResponse): + streaming_response.text = REDACTED_BY_LITELLM elif hasattr(streaming_response, "output"): _redact_responses_api_output(streaming_response.output) if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None: @@ -206,6 +214,7 @@ def _redact_choice_content(choice): choice.message.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.message, "thinking_blocks"): choice.message.thinking_blocks = None + _drop_field_when_present(choice.message, "reasoning_items") _redact_provider_specific_fields(choice.message) _redact_tool_calls(getattr(choice.message, "tool_calls", None)) _redact_function_call(getattr(choice.message, "function_call", None)) @@ -216,12 +225,14 @@ def _redact_choice_content(choice): choice.delta.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.delta, "thinking_blocks"): choice.delta.thinking_blocks = None + _drop_field_when_present(choice.delta, "reasoning_items") _redact_provider_specific_fields(choice.delta) _redact_tool_calls(getattr(choice.delta, "tool_calls", None)) _redact_function_call(getattr(choice.delta, "function_call", None)) elif isinstance(choice, litellm.utils.TextChoices): _redact_field_when_present(choice, "text") _redact_field_when_present(choice, "reasoning_content") + _drop_field_when_present(choice, "logprobs") def _as_mapping(value: object) -> Mapping[str, object] | None: @@ -261,15 +272,31 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ return { **fields, **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}), + **({"reasoning": REDACTED_BY_LITELLM} if fields.get("reasoning") is not None else {}), + **({"reasoning_text": REDACTED_BY_LITELLM} if fields.get("reasoning_text") is not None else {}), **{ key: None for key in ( "thinking_blocks", "reasoningContentBlocks", + "reasoningContent", + "annotations", "citations", + "citation", + "citationsContent", "compaction_blocks", "compaction_start", "compaction_delta", + "reasoning_details", + "web_search_calls", + "web_search_results", + "tool_results", + "code_interpreter_results", + "search_results", + "rerank_results", + "executed_tools", + "custom_outputs", + "tool_search_items", "token_ids", ) if key in fields @@ -282,6 +309,11 @@ def _redact_field_when_present(item: object, field: str) -> None: setattr(item, field, REDACTED_BY_LITELLM) +def _drop_field_when_present(item: object, field: str) -> None: + if getattr(item, field, None) is not None: + setattr(item, field, None) + + def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: @@ -438,32 +470,22 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: function_call["arguments"] = REDACTED_BY_LITELLM -def _redact_model_response_dict_choices(choices, redacted_str: str): +def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: str) -> None: for choice in choices: if isinstance(choice, dict): _redact_provider_specific_fields_dict(choice) if choice.get("token_ids") is not None: choice["token_ids"] = None - if "message" in choice and isinstance(choice["message"], dict): - if choice["message"].get("content") is not None: - choice["message"]["content"] = redacted_str - if choice["message"].get("reasoning_content") is not None: - choice["message"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - _redact_tool_calls_dict(choice["message"]) - elif "delta" in choice and isinstance(choice["delta"], dict): - if choice["delta"].get("content") is not None: - choice["delta"]["content"] = redacted_str - if choice["delta"].get("reasoning_content") is not None: - choice["delta"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - _redact_tool_calls_dict(choice["delta"]) + if ( + body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None) + ) is not None: + for field in ("content", "reasoning_content"): + if body.get(field) is not None: + body[field] = redacted_str + for field in ("thinking_blocks", "reasoning_items", "audio"): + if field in body: + body[field] = None + _redact_tool_calls_dict(body) elif choice.get("text") is not None: choice["text"] = redacted_str else: @@ -586,7 +608,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons return {"text": REDACTED_BY_LITELLM} _result: Final = copy.deepcopy(result) - if isinstance(_result, litellm.ModelResponse): + if isinstance(_result, (litellm.ModelResponse, litellm.TextCompletionResponse)): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index a06860e7786..62bdc1c10a7 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -378,7 +378,8 @@ def filter_team_based_models( ids_to_remove: Final = set() if isinstance(healthy_deployments, dict): - return healthy_deployments + # A deployment asked for by id is filtered like any other: another team's is not served. + return [] if _owner_team(healthy_deployments) not in (None, request_team_id) else healthy_deployments # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the legacy untyped deployment dict for deployment in healthy_deployments: _model_info = deployment.get("model_info") or {} model_team_id = _model_info.get("team_id") @@ -431,6 +432,16 @@ def _deployment_supports_web_search(deployment: dict) -> bool: return True +def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool: + tools: Final[object] = request_kwargs.get("tools") + return isinstance(tools, (list, tuple)) and any( + isinstance(tool, dict) + and isinstance(tool_type := tool.get("type"), str) + and tool_type in _WEB_SEARCH_TOOL_TYPES + for tool in tools + ) + + def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, @@ -445,14 +456,7 @@ def filter_web_search_deployments( if isinstance(healthy_deployments, dict): return healthy_deployments - tools: Final[object] = request_kwargs.get("tools") - is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( - isinstance(tool, dict) - and isinstance(tool_type := tool.get("type"), str) - and tool_type in _WEB_SEARCH_TOOL_TYPES - for tool in tools - ) - if not is_web_search_request: + if not is_web_search_request(request_kwargs): return healthy_deployments # Filter out deployments that don't support web search @@ -607,3 +611,9 @@ def warn_on_provider_credential_mismatch(model_name: str, litellm_params: Mappin ) verbose_router_logger.warning(warning) return warning + + +def _owner_team(deployment: Mapping[str, object]) -> object: + """The team a deployment belongs to (``model_info.team_id``), or None.""" + model_info: Final = deployment.get("model_info") + return model_info.get("team_id") if isinstance(model_info, Mapping) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a deployment dict's model_info diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index a9ef06c3b4f..aa6ce6ea99e 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2826,8 +2826,6 @@ class CustomStreamWrapper: return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk def _accumulate_streamed_delta(self, delta: object) -> None: - # Shared by the sync, async, and non-aiohttp iteration sites so answer - # text and the disqualifying-content latch stay in step across all three. get: Final = getattr(delta, "get", None) content: Final = get("content") if callable(get) else None self.response_uptil_now += content if isinstance(content, str) else "" diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index df8038e756c..52a5026c44f 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -610,6 +610,127 @@ def estimate_image_tokens( return max(_image_policy_tokens(family, dimensions, detail) for family in families) +_MEDIA_FILE_CHARS_PER_TOKEN: Final = 4 +_EmbeddingInputT: Final = TypeVar("_EmbeddingInputT") + + +def _max_input_tokens(model: str) -> int | None: + try: + limit: Final = litellm.get_model_info(model).get("max_input_tokens") + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + return None + return limit if isinstance(limit, int) and limit > 0 else None + + +def _media_file_tokens(data: str, models: Sequence[str]) -> int: + text_estimate: Final = len(data) // _MEDIA_FILE_CHARS_PER_TOKEN + ceilings: Final = tuple(map(_max_input_tokens, models)) + known_ceilings: Final = tuple(ceiling for ceiling in ceilings if ceiling is not None) + if not known_ceilings or len(known_ceilings) != len(ceilings): + return text_estimate + return min(text_estimate, max(known_ceilings)) + + +def _block_str_field(block: Mapping[str, object], container: str, field: str) -> str | None: + inner: Final = as_str_mapping(block.get(container)) + value: Final = inner.get(field) if inner is not None else None + return value if isinstance(value, str) else None + + +def estimate_media_block_tokens(block: Mapping[str, object], models: Sequence[str], fallback_model: str | None) -> int: + from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens + + match block.get("type"): + case "input_audio": + return estimate_audio_tokens( + _block_str_field(block, "input_audio", "data"), models, fallback_model=fallback_model + ) + case "file": + return _media_file_tokens(_block_str_field(block, "file", "file_data") or "", models) + case _: + detail: Final = _block_str_field(block, "image_url", "detail") + return estimate_image_tokens( + models, + _block_str_field(block, "image_url", "url") or "", + "auto" if detail is None else detail, + fallback_model=fallback_model, + ) + + +_PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS: Final = frozenset({"vertex_ai", "gemini", "azure_ai"}) + +_DataUriMediaScope: TypeAlias = Literal["per_item", "whole_batch"] + + +def _is_pdf_block(block: Mapping[str, object]) -> bool: + return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf") + + +def _bedrock_data_uri_media_scope( + resolved_model: str, media: Sequence[Mapping[str, object]] +) -> _DataUriMediaScope | None: + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.utils import supports_embedding_image_input + + base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/")) + if not supports_embedding_image_input(f"bedrock/{base_model}"): + return None + match BaseAWSLLM.get_bedrock_embedding_provider(base_model): + case "cohere": + return "whole_batch" + case "twelvelabs": + return None + case "nova" if any(map(_is_pdf_block, media)): + return None + case _: + return "per_item" + + +def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> _DataUriMediaScope | None: + from litellm.utils import supports_embedding_image_input + + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model) + except litellm.exceptions.BadRequestError: + return None + if provider == "bedrock": + return _bedrock_data_uri_media_scope(resolved_model, media) + if not supports_embedding_image_input(model): + return None + if provider == "cohere": + return "whole_batch" + return "per_item" if provider in _PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS else None + + +def split_embedding_input_tokens( + value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None +) -> tuple[_EmbeddingInputT, int]: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + embedding_input_is_all_media, + split_embedding_media_for_tpm, + ) + + text, media = split_embedding_media_for_tpm(value) + if not media: + return value, 0 + models: Final = tuple(media_models()) + scopes: Final = frozenset(_data_uri_media_scope(model, media) for model in models) + if not models or None in scopes or ("whole_batch" in scopes and not embedding_input_is_all_media(value)): + return value, 0 + media_tokens: Final = sum(estimate_media_block_tokens(block, models, fallback_model) for block in media) + return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape + + +def embedding_aware_token_count( + messages: Sequence[Mapping[str, str]] | None, + text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str] + media_models: Callable[[], Sequence[str]], +) -> int: + embedding_text, media_tokens = split_embedding_input_tokens(text, media_models) + return litellm.token_counter(messages=messages, text=embedding_text) + media_tokens + + TokenCounterFunction = Callable[[str], int] """ Type for a function that counts tokens in a string. diff --git a/litellm/litellm_core_utils/tool_fields.py b/litellm/litellm_core_utils/tool_fields.py index 5ee96ea9133..10b6f07bc4d 100644 --- a/litellm/litellm_core_utils/tool_fields.py +++ b/litellm/litellm_core_utils/tool_fields.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params ROUTES_WITH_TOOLS: Final = frozenset( { @@ -58,6 +58,19 @@ _TOOL_ENTRY: Final = TypeAdapter(_ToolEntry) _WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True)) +def declared_function_name(tool: object) -> str | None: + fields: Final = as_str_mapping(tool) + if fields is None: + return None + function: Final = as_str_mapping(fields.get("function")) + name: Final = (fields if function is None else function).get("name") + return name if isinstance(name, str) and name else None + + +def qualified_member_name(namespace: str, name: str) -> str: + return f"{namespace}__{name}" + + @dataclass(frozen=True, slots=True) class DropPolicy: drop_params: bool diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 0db6a41b503..eae8ac04e8e 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -102,6 +102,14 @@ class SSRFError(ValueError): """Raised when a URL targets a blocked network.""" +def is_https_url_with_host(url: str) -> bool: + try: + parsed_url: Final = urlsplit(url) + return parsed_url.scheme == "https" and bool(parsed_url.hostname) + except ValueError: + return False + + def encode_url_path_segment(value: object, *, field_name: str = "path parameter") -> str: """Percent-encode one user-controlled URL path segment. diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7f688ae733b..6436cd832c7 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -34,6 +34,7 @@ from litellm.llms.anthropic.messages.adapters.transformation import ( ) from litellm.llms.anthropic.messages.convertible_blocks import ( keep_non_string_tool_result_texts, + keep_url_documents, scannable_tool_result_request, ) from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details @@ -780,12 +781,17 @@ class AnthropicMessagesHandler(BaseTranslation): for event in AnthropicMessagesHandler._parse_sse_data_line(line) ) - def _translate_to_openai(self, data: dict) -> ChatCompletionRequest: + def _translate_to_openai( + self, + data: dict, # mutable-ok: the request body the adapter reads + *, + keep_cache_control: bool = False, + ) -> ChatCompletionRequest: """Translate Anthropic request to OpenAI chat completion format.""" ( chat_completion_compatible_request, _tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai( anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) @@ -812,9 +818,13 @@ class AnthropicMessagesHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + *, + request_data: dict[str, object] | None = None, # mutable-ok: the request dict guardrails receive + keep_cache_control: bool = False, ) -> Mapping[str, object]: """ - Process input messages by applying guardrails to text content. + Process input messages by applying guardrails to text content. ``request_data`` is what the guardrail sees as + the request when ``data`` is only its body, e.g. a provider pass-through route that nests the body """ messages: Final = data.get("messages") if messages is None: @@ -826,8 +836,17 @@ class AnthropicMessagesHandler(BaseTranslation): # The top-level prompt is translated on its own below so it can be hoisted in front of # any mid-turn system entries and scanned first, aligned with that structured position. - translation_source: Final = {key: value for key, value in data.items() if key != "system"} - chat_completion_compatible_request: Final = self._translate_to_openai(translation_source) + body_model: Final = data.get("model") + request_model: Final = (request_data or {}).get("model") + model: Final[str] = ( + body_model + if isinstance(body_model, str) and body_model + else (request_model if isinstance(request_model, str) else "") + ) + translation_source: Final = {**{key: value for key, value in data.items() if key != "system"}, "model": model} + chat_completion_compatible_request: Final = self._translate_to_openai( + translation_source, keep_cache_control=keep_cache_control + ) full_structured_messages: Final = cast( list[AllMessageValues], @@ -836,7 +855,9 @@ class AnthropicMessagesHandler(BaseTranslation): has_midturn_system_message: Final = any( str(message.get("role") or "").lower() == "system" for message in full_structured_messages ) - hoisted_system_message: Final = None if skip_system else self._hoisted_top_level_system_message(data) + hoisted_system_message: Final = ( + None if skip_system else self._hoisted_top_level_system_message(data, model, keep_cache_control) + ) if hoisted_system_message is not None: full_structured_messages.insert(0, hoisted_system_message) # skip_system already excluded the trusted top-level prompt (it is simply not hoisted); @@ -898,7 +919,6 @@ class AnthropicMessagesHandler(BaseTranslation): if structured_messages: inputs["structured_messages"] = structured_messages # Include model information if available - model: Final = data.get("model") if model: inputs["model"] = model original_tool_names: Final = _original_tool_names( @@ -906,7 +926,7 @@ class AnthropicMessagesHandler(BaseTranslation): ) guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, - request_data=data, + request_data=data if request_data is None else request_data, input_type="request", logging_obj=litellm_logging_obj, ) @@ -954,6 +974,7 @@ class AnthropicMessagesHandler(BaseTranslation): hoisted_system_message=hoisted_system_message, preserve_system_messages=has_midturn_system_message, original_tool_names=original_tool_names.used, + model=model, ) else: if guardrailed_texts and len(guardrailed_texts) != len(scanned): @@ -976,17 +997,20 @@ class AnthropicMessagesHandler(BaseTranslation): return data - def _hoisted_top_level_system_message(self, data: Mapping[str, object]) -> AllMessageValues | None: + def _hoisted_top_level_system_message( + self, data: Mapping[str, object], model: str, keep_cache_control: bool + ) -> AllMessageValues | None: """Return the system message produced by translating the top-level prompt.""" system: Final = data.get("system") if not system: return None probe: Final = self._translate_to_openai( { - "model": data.get("model") or "", + "model": model, "messages": [], "system": system, - } + }, + keep_cache_control=keep_cache_control, ) hoisted: Final = probe.get("messages") or [] return hoisted[0] if hoisted else None @@ -1120,6 +1144,7 @@ class AnthropicMessagesHandler(BaseTranslation): preserve_system_messages: bool = False, *, original_tool_names: Mapping[str, str], + model: str, ) -> None: """Write a guardrail's structured-message rewrite back without losing corrections.""" from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -1128,7 +1153,6 @@ class AnthropicMessagesHandler(BaseTranslation): ) _is_system: Final = AnthropicMessagesHandler._is_system - model: Final = str(data.get("model") or "") converted: Final[list] = [] # mutable-ok: API message payload def _convert_run(run: list) -> None: # mutable-ok: API message payload @@ -1187,7 +1211,10 @@ class AnthropicMessagesHandler(BaseTranslation): cast(Sequence[object], converted), # cast-ok: Anthropic message payload original_tool_names, ) - data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages")) + original_messages: Final = data.get("messages") + data["messages"] = keep_url_documents( + keep_non_string_tool_result_texts(converted, original_messages), original_messages + ) @staticmethod def _extract_midturn_system_text( diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 4eae06dbcdf..c273c715ba6 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -734,6 +734,7 @@ class ModelResponseIterator: self._split_answer_blocks: tuple[str, ...] = () self._withheld_answer_close: str = "" self._streamed_answer_keys: frozenset[str] | None = None + self._held_prose: str = "" # For handling partial JSON chunks from fragmentation # See: https://github.com/BerriAI/litellm/issues/17473 @@ -826,8 +827,13 @@ class ModelResponseIterator: thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = [] if "text" in content_block["delta"]: - text = content_block["delta"]["text"] - return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content + return ( + self._hold_prose(content_block["delta"]["text"]), + tool_use, + thinking_blocks, + provider_specific_fields, + reasoning_content, + ) self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: # Only emit tool calls if we're in a tool_use or server_tool_use block @@ -1025,7 +1031,7 @@ class ModelResponseIterator: if self.current_content_block_type in _THINKING_BLOCK_TYPES: self.thinking_block_index += 1 if content_block_start["content_block"]["type"] == "text": - text = self._release_answer_close() + content_block_start["content_block"]["text"] + text = self._release_answer_close() + self._hold_prose(content_block_start["content_block"]["text"]) elif ( content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" @@ -1161,7 +1167,7 @@ class ModelResponseIterator: elif type_chunk == "message_delta": finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk) provider_specific_fields.update(stop_reason_fields) - text = self._release_answer_close() + self._drain_split_answer_blocks() + text = self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() if container: provider_specific_fields["container"] = container # Store container_id and re-emit code_interpreter_results @@ -1211,7 +1217,9 @@ class ModelResponseIterator: status_code=500, # it looks like Anthropic API does not return a status code in the chunk error - default to 500 ) - text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use) + text, tool_use = self._handle_json_mode_chunk( + text=text + self._settle_held_prose(tool_use), tool_use=tool_use + ) refusal_fields: Final = refusal_params(provider_specific_fields) returned_chunk: Final = ModelResponseStream( @@ -1309,6 +1317,24 @@ class ModelResponseIterator: self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME) return self._withhold_answer_close(buffered) if self._response_format_verdict == "pass_through" else "" + def _hold_prose(self, text: str) -> str: + if not self.json_mode or self.response_format_schema is None: + return text + self._held_prose += text + return "" + + def _settle_held_prose(self, tool_use: ChatCompletionToolCallChunk | None) -> str: + if tool_use is None or tool_use.get("id") is None or self.current_content_block_type != "tool_use": + return "" + held: Final = self._held_prose + self._held_prose = "" + return "" if tool_use.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME else held + + def _release_held_prose(self) -> str: + held: Final = self._held_prose + self._held_prose = "" + return "" if self.converted_response_format_tool else held + def _withhold_answer_close(self, text: str) -> str: pending: Final = self._withheld_answer_close + text body: Final = pending.rstrip() @@ -1362,7 +1388,9 @@ class ModelResponseIterator: def cut_stream_chunk(self) -> ModelResponseStream | None: unsent: Final = self._flush_response_format_buffer() if self._response_format_buffer.getvalue() else "" - text: Final = unsent + self._release_answer_close() + self._drain_split_answer_blocks() + text: Final = ( + unsent + self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose() + ) if not text: return None return ModelResponseStream( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2e908e66626..4f890ed922f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -26,8 +26,14 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, +) +from litellm.litellm_core_utils.asyncify import is_event_loop_running +from litellm.litellm_core_utils.core_helpers import ( + coerce_token_limit, + drops_unsupported_params, + map_finish_reason, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, invalid_type_message, @@ -47,8 +53,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, + async_format_pdf_links, async_inline_remote_media, + format_pdf_links, inline_remote_image_urls, + is_unformatted_pdf_link_image, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( message_field, @@ -577,12 +586,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) @classmethod - def get_config(cls, *, model: str | None = None): + def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None): config: Final = super().get_config() # anthropic requires a default value for max_tokens if config.get("max_tokens") is None: - config["max_tokens"] = cls.get_max_tokens_for_model(model) + config["max_tokens"] = ( + configured_max_output_tokens + if configured_max_output_tokens is not None + else cls.get_max_tokens_for_model(model) + ) return config @@ -1774,7 +1787,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - open_object_takes_forced_json_tool: Final = ( + open_object_takes_json_tool: Final = ( json_schema is not None and schema_has_open_object(json_schema) and not self.is_thinking_requested(request_params) @@ -1782,7 +1795,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and not request_params.get("tools") and not AnthropicModelInfo.forced_tool_use_unsupported(model) ) - return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + return self.uses_native_structured_output(model) and not open_object_takes_json_tool def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) @@ -1797,7 +1810,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) def map_response_format_to_anthropic_tool( - self, value: dict | None, optional_params: dict, is_thinking_enabled: bool + self, value: dict | None, optional_params: dict, tool_is_forced: bool ) -> AnthropicMessagesTool | None: ignore_response_format_types: Final = ["text"] if value is None or value["type"] in ignore_response_format_types: # value is a no-op @@ -1815,6 +1828,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, + description=None if tool_is_forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) return _tool @@ -1917,7 +1931,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=drop_params, runs_hosted_mcp=self.runs_hosted_mcp, ) - is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params) + response_format_tool_is_forced: Final = ( + "response_format" in request_params + and AnthropicModelInfo.response_format_tool_can_be_forced( + model, self._resolved_provider, REQUEST_PARAMS.validate_python(request_params) + ) + ) # NB: ``map_openai_params`` deliberately does NOT sanitize tool names # here. Names are the *original* OpenAI names at this stage, and must @@ -1977,10 +1996,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if _output_format is not None: optional_params["output_format"] = _output_format else: - _tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + _tool = self.map_response_format_to_anthropic_tool( + value, optional_params, response_format_tool_is_forced + ) if _tool is None: continue - if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model): + if response_format_tool_is_forced: _tool_choice = { "name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool", @@ -2109,12 +2130,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _create_json_tool_call_for_response_format( self, json_schema: dict | None = None, + description: str | None = None, ) -> AnthropicMessagesTool: """ Handles creating a tool call for getting responses in JSON format. Args: json_schema (Optional[dict]): The JSON schema the response should be in + description (Optional[str]): The tool description, left off the tool when None Returns: AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format @@ -2131,11 +2154,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: _input_schema.update(cast(AnthropicInputSchema, json_schema)) - _tool: Final = AnthropicMessagesTool( - name=RESPONSE_FORMAT_TOOL_NAME, - input_schema=_input_schema, + if description is None: + return AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema) + return AnthropicMessagesTool( + name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema, description=description ) - return _tool def should_strip_billing_metadata(self) -> bool: """ @@ -2333,6 +2356,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def inlines_remote_media(self, media: RemoteMedia) -> bool: return inline_remote_image_urls(media) and media.url.startswith("http://") + def probes_pdf_link(self, media: RemoteMedia) -> bool: + return is_unformatted_pdf_link_image(media) and not self.inlines_remote_media(media) + + def _with_pdf_links_formatted(self, messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + return messages if is_event_loop_running() else format_pdf_links(messages, should_probe=self.probes_pdf_link) + async def async_transform_request( self, model: str, @@ -2341,9 +2370,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): litellm_params: dict[str, object], # mutable-ok: BaseConfig signature headers: dict[str, object], # mutable-ok: BaseConfig signature ) -> dict[str, object]: # mutable-ok: BaseConfig signature + inlined: Final = await async_inline_remote_media(messages, should_inline=self.inlines_remote_media) return self.transform_request( model=model, - messages=await async_inline_remote_media(messages, should_inline=self.inlines_remote_media), + messages=list(await async_format_pdf_links(inlined, should_probe=self.probes_pdf_link)), optional_params=optional_params, litellm_params=litellm_params, headers=headers, @@ -2359,6 +2389,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params ) + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return True + def transform_request( self, model: str, @@ -2372,19 +2405,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ ## VALIDATE REQUEST """Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed).""" + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + translated_messages: Final = ( + list( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + messages, + drop_params, + cache_control_count, + lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params), + ) + ) + if self.translates_prompt_cache_breakpoint(model) + else messages + ) capped_messages: Final = ( list( self._messages_within_cache_control_cap( - messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) ) ) if drop_params - else messages + else translated_messages ) if "tools" not in optional_params and has_tool_call_blocks(capped_messages): @@ -2477,7 +2523,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): try: anthropic_messages = anthropic_messages_pt( model=model, - messages=list(conversation), + messages=list(self._with_pdf_links_formatted(conversation)), llm_provider=self._resolved_provider, drop_params=drop_params, ) @@ -2512,7 +2558,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["tools"] = tools ## Load Config - config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model)) + model_info: Final[object] = cast( # cast-ok: request parameters are a Mapping + Mapping[str, object], litellm_params + ).get("model_info") + configured_max_output_tokens: Final = ( + coerce_token_limit( + cast(Mapping[str, object], model_info).get("max_output_tokens") # cast-ok: runtime Mapping check + ) + if isinstance(model_info, Mapping) + else None + ) + config: Final[Mapping[str, object]] = MappingProxyType( + litellm.AnthropicConfig.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens) + ) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c9df903cbd6..bc9bd769d83 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -28,7 +28,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, is_encrypted_reasoning_block, @@ -307,9 +307,10 @@ def count_tokens_feature_betas( ) -def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: +def supports_anthropic_cache_control( + model: str, custom_llm_provider: str | None, model_info: Mapping[str, object] | None = None +) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - from litellm.utils import supports_prompt_caching try: provider: Final = custom_llm_provider if custom_llm_provider is not None else get_llm_provider(model=model)[1] @@ -318,10 +319,27 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None return ( provider in ("anthropic", "bedrock", "vertex_ai", "azure_ai") and "claude" in model.lower() - and supports_prompt_caching(model=model, custom_llm_provider=provider) + and _deployment_supports_prompt_caching(model, provider, model_info) ) +def _deployment_supports_prompt_caching(model: str, provider: str, model_info: Mapping[str, object] | None) -> bool: + from litellm.utils import catalog_with_runtime_registrations, get_model_info, supports_prompt_caching + + declared: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) + if declared is not None: + return declared + live: Final = supports_prompt_caching(model=model, custom_llm_provider=provider) + if model_info is None: + return live + try: + key: Final = get_model_info(model=model, custom_llm_provider=provider).get("key") + except Exception: # noqa: BLE001 # an unmapped model keeps the live answer + return live + catalog_entry: Final = catalog_with_runtime_registrations().get(key) + return live if catalog_entry is None else catalog_entry.get("supports_prompt_caching") is True + + def _tool_config_point_control(points: object) -> Mapping[object, object] | None: point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ()) tool_config_point: Final = next( @@ -1123,6 +1141,34 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider) + @staticmethod + def thinks_by_default(model: str, custom_llm_provider: str) -> bool: + return AnthropicModelInfo._is_always_on_thinking_model( + model, custom_llm_provider + ) or AnthropicModelInfo._supports_model_capability(model, "thinking_on_by_default", custom_llm_provider) + + @staticmethod + def will_think(model: str, custom_llm_provider: str, request_params: Mapping[str, object]) -> bool: + if AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider): + return True + match request_params: + case {"thinking": True | {"type": "enabled" | "adaptive"}}: + return True + case {"reasoning_effort": effort} if effort is not None and effort != "none": + return True + case {"thinking": {"type": "disabled"}} | {"reasoning_effort": "none"}: + return False + case _: + return AnthropicModelInfo.thinks_by_default(model, custom_llm_provider) + + @staticmethod + def response_format_tool_can_be_forced( + model: str, custom_llm_provider: str, request_params: Mapping[str, object] + ) -> bool: + return not AnthropicModelInfo.forced_tool_use_unsupported(model) and not AnthropicModelInfo.will_think( + model, custom_llm_provider, request_params + ) + @staticmethod def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool: """Whether ``model`` is an adaptive-thinking model that still accepts legacy diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 1c8f0c206cd..3034927c612 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -49,7 +49,10 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) def _count_image(block: JsonValue) -> JsonValue: if not isinstance(block, dict) or block.get("type") != "image_url": return block - from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_content_element_factory, + convert_to_anthropic_image_obj, + ) image_block: Final = _IMAGE_BLOCK.validate_python(block) image_url: Final = image_block["image_url"] @@ -57,7 +60,7 @@ def _count_image(block: JsonValue) -> JsonValue: openai_image_url=image_url if isinstance(image_url, str) else image_url["url"], format=image_url.get("format") if isinstance(image_url, dict) else None, ) - image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source}) + image: Final = _COUNT_REQUEST.validate_python(anthropic_content_element_factory(source)) return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image} diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index c41deb00f1d..87be996ebbf 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -670,6 +670,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage + # The provider's error behind this stream's error frame, for a router that accounts for it. + self.stream_failure: Exception | None = None self._refusal_text: str = "" self._paused_turn: bool = False self._claude_refusal_stop_details: AnthropicStopDetails | None = None @@ -1396,6 +1398,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): verbose_logger.exception("Anthropic Adapter - mid-stream error: %s", e) logging_obj: Final = self.litellm_logging_obj provider_error: Final = _provider_error(e) + self.stream_failure = provider_error if logging_obj is not None and logging_obj.on_detached_stream_failure is not None: if provider_error is e: raise @@ -1619,6 +1622,10 @@ class AnthropicSSEStream(AsyncIterator[bytes]): self._byte_stream: Final[AsyncIterator[bytes]] = anthropic_wrapper.async_anthropic_sse_wrapper() self._hidden_params: dict[str, object] = {} + @property + def stream_failure(self) -> Exception | None: + return self._anthropic_wrapper.stream_failure + @property def chunks(self) -> "list[ModelResponseStream] | None": return self._anthropic_wrapper.chunks diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b1b5a8a11a..afcd94c5b2c 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -380,8 +380,8 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va class LiteLLMAnthropicMessagesAdapter: - def __init__(self): - pass + def __init__(self, *, keep_cache_control: bool = False) -> None: + self.keep_cache_control: Final = keep_cache_control ### FOR [BETA] `/v1/messages` endpoint support @@ -437,7 +437,7 @@ class LiteLLMAnthropicMessagesAdapter: cache_control: Final = ( source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None) ) - if cache_control and model and self.target_consumes_cache_control(model): + if cache_control and (self.keep_cache_control or (model and self.target_consumes_cache_control(model))): # TypedDict objects support dict operations at runtime # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432) if isinstance(target, dict): diff --git a/litellm/llms/anthropic/messages/caller_model_checks.py b/litellm/llms/anthropic/messages/caller_model_checks.py index 3822f075ccf..c89d8a1ea23 100644 --- a/litellm/llms/anthropic/messages/caller_model_checks.py +++ b/litellm/llms/anthropic/messages/caller_model_checks.py @@ -243,8 +243,7 @@ async def caller_within_model_budget( return "allowed" model_max_budget: Final = getattr(user_api_key_auth, "model_max_budget", None) - token: Final = getattr(user_api_key_auth, "token", None) - if isinstance(model_max_budget, dict) and model_max_budget and token is not None: + if isinstance(model_max_budget, dict) and model_max_budget and user_api_key_auth.token is not None: try: await model_max_budget_limiter.is_key_within_model_budget( user_api_key_dict=user_api_key_auth, @@ -300,10 +299,8 @@ async def caller_within_model_budget( ) return "denied_for_now" - end_user_model_max_budget: Final[dict[str, object] | None] = getattr( - user_api_key_auth, "end_user_model_max_budget", None - ) - end_user_id: Final[str | None] = getattr(user_api_key_auth, "end_user_id", None) + end_user_model_max_budget: Final[dict[str, object] | None] = user_api_key_auth.end_user_model_max_budget + end_user_id: Final[str | None] = user_api_key_auth.end_user_id if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None: try: await model_max_budget_limiter.is_end_user_within_model_budget( diff --git a/litellm/llms/anthropic/messages/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py index 09d662da543..b98b4cf3f75 100644 --- a/litellm/llms/anthropic/messages/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -314,6 +314,76 @@ def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: obj return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] +def _url_source_url(block: object) -> str | None: + match as_str_mapping(block): + case {"type": "image" | "document", "source": {"type": "url", "url": str(url)}}: + return url + case _: + return None + + +def _is_document(block: object) -> bool: + match as_str_mapping(block): + case {"type": "document"}: + return True + case _: + return False + + +@dataclass(frozen=True, slots=True, order=True) +class _UrlBlockPlace: + in_tool_result: bool + url: str + + +def _url_sourced_blocks(block: object, in_tool_result: bool = False) -> tuple[tuple[_UrlBlockPlace, object], ...]: + url: Final = _url_source_url(block) + if url is not None: + return ((_UrlBlockPlace(in_tool_result, url), block),) + match as_str_mapping(block): + case {"type": "tool_result", "content": list() as content}: + in_this_tool_result: Final = partial(_url_sourced_blocks, in_tool_result=True) + return tuple(chain.from_iterable(map(in_this_tool_result, _list_items(content)))) + case _: + return () + + +def _entry_place(entry: tuple[_UrlBlockPlace, object]) -> _UrlBlockPlace: + return entry[0] + + +def _entry_block(entry: tuple[_UrlBlockPlace, object]) -> object: + return entry[1] + + +def _restored_url_document( + block: object, originals: Mapping[_UrlBlockPlace, Iterator[object]], in_tool_result: bool = False +) -> object: + url: Final = _url_source_url(block) + if url is not None: + remaining: Final = originals.get(_UrlBlockPlace(in_tool_result, url)) + original: Final = None if remaining is None else next(remaining, None) + return original if _is_document(original) else block + match as_str_mapping(block): + case {"type": "tool_result", "content": list() as content} as tool_result: + restore: Final = partial(_restored_url_document, originals=originals, in_tool_result=True) + return {**tool_result, "content": list(map(restore, _list_items(content)))} + case _: + return block + + +def keep_url_documents(rewritten: Sequence[object], original: object) -> Sequence[object]: + blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) + by_place: Final = tuple(sorted(chain.from_iterable(map(_url_sourced_blocks, blocks)), key=_entry_place)) + if not any(_is_document(block) for _, block in by_place): + return rewritten + originals_in_order: Final = MappingProxyType( + {place: iter(tuple(map(_entry_block, group))) for place, group in groupby(by_place, key=_entry_place)} + ) + restore: Final = partial(_restored_url_document, originals=originals_in_order) + return [_with_user_blocks(message, restore) for message in rewritten] + + def _without_non_string_text(block: object) -> object: match as_str_mapping(block): case {"type": "tool_result", "content": list()} as tool_result: diff --git a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index 25d4d7b3ce1..0aa30998557 100644 --- a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES +from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -35,10 +36,16 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = ( b'data: {"type": "error", "error": {"type": "api_error", "message": ' b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n' ) +AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = ( + b"event: error\n" + b'data: {"type": "error", "error": {"type": "api_error", "message": ' + b'"The agentic loop reached its limit before this turn finished. ' + b'Raise max_agentic_loops to allow more loops."}}\n\n' +) def is_server_fulfilled_tool_leak_error(chunk: object) -> bool: - return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES + return chunk in (SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES) async def _anext_or_none(iterator: AsyncIterator) -> bytes | None: @@ -216,6 +223,7 @@ class AgenticAnthropicStreamingIterator: self._follow_up_chunk_task: asyncio.Task | None = None self._replay_index = 0 self._error_emitted = False + self._loop_limit_reached = False @property def has_buffered_provider_output(self) -> bool: @@ -288,6 +296,8 @@ class AgenticAnthropicStreamingIterator: "tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client", self._model, ) + if self._loop_limit_reached: + return AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES if self._replay_index < len(self._collected_bytes): @@ -393,14 +403,9 @@ class AgenticAnthropicStreamingIterator: "AgenticStreamingIterator: Unexpected result type from hooks: %s", type(result).__name__, ) - except Exception as e: - _call_id: Final = getattr(self._logging_obj, "litellm_call_id", "unknown") - verbose_logger.exception( - "AgenticStreamingIterator: Error in agentic hook processing [call_id=%s model=%s]: %s", - _call_id, - self._model, - str(e), - ) + except AgenticLoopSafetyError as e: + self._loop_limit_reached = isinstance(e, AgenticLoopLimitError) + verbose_logger.warning("AgenticStreamingIterator: agentic loop refused [rail=%s]", type(e).__name__) @staticmethod def _rebuild_anthropic_response_from_sse( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 8b638f2c03a..ec09987837c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.asyncify import ( @@ -187,6 +188,7 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob provider if isinstance(provider, str) else None, forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), forwarded_kwargs.get("prompt_cache_options"), + routed_model_info(forwarded_kwargs.get("model_info")), ) @@ -194,10 +196,11 @@ def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[s provider: Final = forwarded_kwargs.get("custom_llm_provider") provider_name: Final = provider if isinstance(provider, str) else None base_model: Final = forwarded_kwargs.get("base_model") - return supports_openai_prompt_cache_breakpoint(model, provider_name) or ( + model_info: Final = routed_model_info(forwarded_kwargs.get("model_info")) + return supports_openai_prompt_cache_breakpoint(model, provider_name, model_info) or ( isinstance(base_model, str) and bool(base_model) - and supports_openai_prompt_cache_breakpoint(base_model, provider_name) + and supports_openai_prompt_cache_breakpoint(base_model, provider_name, model_info) ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index dae96ba4ce1..f0c3316c509 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -141,6 +141,8 @@ class AnthropicResponsesStreamWrapper: if litellm_logging_obj is not None: litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) self._unreported_failure: Exception | None = None + # The failure behind this stream's error frame, for a router that accounts for it. + self.stream_failure: Exception | None = None self._current_block_index: int = -1 # Map item_id -> content_block_index so we can stop the right block later self._item_id_to_block_index: dict[str, int] = {} @@ -467,6 +469,7 @@ class AnthropicResponsesStreamWrapper: llm_provider=getattr(self.responses_stream, "custom_llm_provider", None), model=self.model, ) + self.stream_failure = self._unreported_failure self._chunk_queue.append(_anthropic_error_chunk(status_code, message)) async def _report_unreported_failure(self) -> None: @@ -578,5 +581,9 @@ class AnthropicResponsesSSEByteStream: async def aclose(self) -> None: await self._frames.aclose() + @property + def stream_failure(self) -> Exception | None: + return self._wrapper.stream_failure + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": return self._wrapper.partial_response_on_disconnect() diff --git a/litellm/llms/aws_polly/text_to_speech/transformation.py b/litellm/llms/aws_polly/text_to_speech/transformation.py index 325c9fd364d..3b52a8df3f2 100644 --- a/litellm/llms/aws_polly/text_to_speech/transformation.py +++ b/litellm/llms/aws_polly/text_to_speech/transformation.py @@ -178,7 +178,6 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM): else: mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT - # model arrives without the "aws_polly/" prefix, e.g. "standard" engine: Final = self._extract_engine_from_model(model) mapped_params["engine"] = engine diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index df8a0614665..386815412b5 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,11 +1,17 @@ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final -from openai import AsyncAzureOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.llms.base_llm.audio_transcription.transformation import ( + acreate_sdk_transcription, + create_sdk_transcription, + sdk_compatible_transcription_request_data, +) from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -38,13 +44,18 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): data: Final = {"model": model, "file": audio_file, **optional_params} + sdk_data: Final = sdk_compatible_transcription_request_data(data) if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, - data=data, + data=sdk_data, model_response=model_response, timeout=timeout, api_key=api_key, @@ -58,7 +69,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=api_version, + api_version=None if self.is_azure_v1_api_version(api_version) else api_version, api_base=api_base, api_key=api_key, model=model, @@ -84,10 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - response: Final = azure_client.audio.transcriptions.create( - **data, - timeout=timeout, - ) + _, response = create_sdk_transcription(azure_client, sdk_data, timeout) + if isinstance(response, Stream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() @@ -114,7 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion): self, audio_file: FileTypes, model: str, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: "LiteLLMLoggingObj", @@ -125,7 +135,7 @@ class AzureAudioTranscription(AzureChatCompletion): max_retries=None, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse: + ) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: response = None try: async_azure_client: Final = self.get_azure_openai_client( @@ -155,12 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **data, timeout=timeout - ) - - headers: Final = dict(raw_response.headers) - response = raw_response.parse() + headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout) + if isinstance(response, AsyncStream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 6bb6d584385..16c8bf6f9f1 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -1,15 +1,16 @@ import asyncio import hashlib +import itertools import json import os import re -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from contextlib import closing from types import MappingProxyType -from typing import Final, Literal, NamedTuple, cast +from typing import Final, Literal, NamedTuple, NoReturn, cast import httpx -from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from openai import APIError, AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import ConfigDict, TypeAdapter, with_config from typing_extensions import ReadOnly, TypedDict @@ -20,7 +21,7 @@ from litellm.constants import DEFAULT_MAX_RETRIES from litellm.litellm_core_utils.single_flight import SingleFlight, single_flight_lru_cache from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import HTTPHandler, MaskedHTTPStatusError from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -52,6 +53,13 @@ class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object): _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams) +_AZURE_AD_TOKEN_PROVIDER_ADAPTER: Final = TypeAdapter[Callable[[], str | Awaitable[str]] | None]( + Callable[[], str | Awaitable[str]] | None +) + + +def _callable_azure_ad_token_provider(provider: object) -> Callable[[], str | Awaitable[str]] | None: + return _AZURE_AD_TOKEN_PROVIDER_ADAPTER.validate_python(provider) if callable(provider) else None class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): @@ -81,6 +89,160 @@ class AzureOpenAIError(BaseLLMException): ) +class EntraRejectionError(AzureOpenAIError): + pass + + +_GUID: Final = re.compile(r"[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}") +_AADSTS_REASON: Final = re.compile(r"\bAADSTS(\d+)\b.*?(?:\.(?=[\s'\"\\]|$)|$)", re.MULTILINE) +_ENTRA_CREDENTIAL_REJECTION_CODES: Final = frozenset( + { + 50012, + 50013, + 50027, + 50034, + 50053, + 50055, + 50057, + 50059, + 50076, + 50079, + 50126, + 50128, + 53003, + 65001, + 70001, + 70002, + 70011, + 70021, + 70025, + 70052, + 90002, + 90094, + 500011, + 500014, + 700016, + 700023, + 700024, + 700025, + 700027, + 700211, + 700213, + 700223, + 700238, + 7000112, + 7000215, + 7000218, + 7000222, + } +) + + +def _entra_rejection_reason(message: str) -> str | None: + reason: Final = _AADSTS_REASON.search(message) + if reason is None or int(reason.group(1)) not in _ENTRA_CREDENTIAL_REJECTION_CODES: + return None + return _GUID.sub("", reason.group(0)) + + +def _entra_rejection(reason: str) -> EntraRejectionError: + return EntraRejectionError(status_code=401, message=f"Failed to get Azure AD token: {reason}") + + +def entra_rejection_as_auth_error(error: Exception) -> EntraRejectionError | None: + try: + from azure.core.exceptions import ClientAuthenticationError + except ImportError: + return None + if not isinstance(error, ClientAuthenticationError): + return None + status_code: Final = error.status_code + if status_code is not None and (status_code == 429 or status_code >= 500): + return None + reason: Final = _entra_rejection_reason(str(error)) + return None if reason is None else _entra_rejection(reason) + + +def _raise_entra_rejection_as_auth_error(error: Exception) -> NoReturn: + auth_error: Final = entra_rejection_as_auth_error(error) + if auth_error is None: + raise error + raise auth_error from error + + +def entra_token(provider: Callable[[], str]) -> str: + try: + return provider() + except entra_rejection_types() as error: + _raise_entra_rejection_as_auth_error(error) + + +def entra_rejection_types() -> tuple[type[Exception], ...]: + try: + from azure.core.exceptions import ClientAuthenticationError + except ImportError: + return () + return (ClientAuthenticationError,) + + +_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 + + +def _exception_chain(error: BaseException) -> Iterator[BaseException]: + link: BaseException | None = error # rebind-ok: advances one link of the cause chain per iteration + for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH): + if link is None: + return + yield link + link = link.__cause__ or link.__context__ + + +_REQUEST_FAILURE_TYPES: Final = (APIError, httpx.HTTPError) + + +def _ends_the_failed_attempt(link: BaseException, failed_attempt: BaseException) -> bool: + return not isinstance(link, EntraRejectionError) and ( + isinstance(link, _REQUEST_FAILURE_TYPES) or type(link) is type(failed_attempt) + ) + + +def _links_within_the_failed_attempt(error: BaseException) -> Iterator[BaseException]: + yield error + yield from itertools.takewhile( + lambda link: not _ends_the_failed_attempt(link, error), itertools.islice(_exception_chain(error), 1, None) + ) + + +def entra_rejection_in(error: BaseException) -> EntraRejectionError | None: + return next( + (link for link in _links_within_the_failed_attempt(error) if isinstance(link, EntraRejectionError)), None + ) + + +def caused_by_entra_rejection(error: BaseException) -> bool: + return entra_rejection_in(error) is not None + + +async def _awaited_token(token: Awaitable[str]) -> str: + try: + return await token + except entra_rejection_types() as error: + _raise_entra_rejection_as_auth_error(error) + + +def _entra_rejections_as_auth_errors( + provider: Callable[[], str | Awaitable[str]], +) -> Callable[[], str | Awaitable[str]]: + def call_provider() -> str | Awaitable[str]: + try: + token: Final = provider() + except entra_rejection_types() as error: + _raise_entra_rejection_as_auth_error(error) + return token if isinstance(token, str) else _awaited_token(token) + + return call_provider + + def process_azure_headers(headers: httpx.Headers | dict) -> dict: openai_headers: Final = {} if "x-ratelimit-limit-requests" in headers: @@ -91,7 +253,7 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict: openai_headers["x-ratelimit-limit-tokens"] = headers["x-ratelimit-limit-tokens"] if "x-ratelimit-remaining-tokens" in headers: openai_headers["x-ratelimit-remaining-tokens"] = headers["x-ratelimit-remaining-tokens"] - llm_response_headers: Final = {"{}-{}".format("llm_provider", k): v for k, v in headers.items()} + llm_response_headers: Final = {f"llm_provider-{k}": v for k, v in headers.items()} return {**llm_response_headers, **openai_headers} @@ -257,22 +419,34 @@ def _cached_oidc_token(cache_key: str) -> str | None: return cached if isinstance(cached, str) else None +def _oidc_exchange_rejection_reason(error: MaskedHTTPStatusError) -> str | None: + if error.response.status_code not in (400, 401): + return None + return _entra_rejection_reason(error.text or error.response.text) + + def _exchange_oidc_token(cache_key: str, token_url: str, client_id: str, scope: str, oidc_token: str) -> str: cached_access_token: Final = _cached_oidc_token(cache_key) if cached_access_token is not None: return cached_access_token with closing(HTTPHandler(timeout=litellm.request_timeout)) as client: - req_token: Final = client.post( - token_url, - data={ - "client_id": client_id, - "grant_type": "client_credentials", - "scope": scope, - "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - "client_assertion": oidc_token, - }, - ) + try: + req_token: Final = client.post( + token_url, + data={ + "client_id": client_id, + "grant_type": "client_credentials", + "scope": scope, + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) + except MaskedHTTPStatusError as error: + reason: Final = _oidc_exchange_rejection_reason(error) + if reason is None: + raise + raise _entra_rejection(reason) from error if req_token.status_code != 200: raise AzureOpenAIError( @@ -483,6 +657,9 @@ def get_azure_ad_token( raise except Exception as e: verbose_logger.error("Error calling Azure AD token provider: %s", e) + auth_error: Final = entra_rejection_as_auth_error(e) + if auth_error is not None: + raise auth_error from e raise RuntimeError(f"Failed to get Azure AD token: {e}") from e return azure_ad_token @@ -696,7 +873,7 @@ class BaseAzureLLM(BaseOpenAILLM): api_version: str | None, is_async: bool, ) -> dict: - azure_ad_token_provider = litellm_params.get("azure_ad_token_provider") + azure_ad_token_provider = _callable_azure_ad_token_provider(litellm_params.get("azure_ad_token_provider")) # If we have api_key, then we have higher priority azure_ad_token = litellm_params.get("azure_ad_token") @@ -775,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM): azure_client_params["timeout"] = timeout if azure_ad_token_provider is not None: - azure_client_params["azure_ad_token_provider"] = azure_ad_token_provider + azure_client_params["azure_ad_token_provider"] = _entra_rejections_as_auth_errors(azure_ad_token_provider) # this decides if we should set azure_endpoint or base_url on Azure OpenAI Client # required to support GPT-4 vision enhancements, since base_url needs to be set on Azure OpenAI Client diff --git a/litellm/llms/azure/decisions/transformation.py b/litellm/llms/azure/decisions/transformation.py index 3fb40e39702..bdbd784185f 100644 --- a/litellm/llms/azure/decisions/transformation.py +++ b/litellm/llms/azure/decisions/transformation.py @@ -8,6 +8,7 @@ import litellm from litellm.llms.azure.common_utils import ( BaseAzureLLM, cached_azure_ad_token_refresh_provider, + entra_token, ) from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig from litellm.secret_managers.main import get_secret_str @@ -61,7 +62,7 @@ class AzureDecisionsConfig(OpenAIDecisionsConfig): scope: Final = ( params.azure_scope or get_secret_str("AZURE_SCOPE") or "https://cognitiveservices.azure.com/.default" ) - azure_headers["Authorization"] = f"Bearer {cached_azure_ad_token_refresh_provider(scope)()}" + azure_headers["Authorization"] = f"Bearer {entra_token(cached_azure_ad_token_refresh_provider(scope))}" return azure_headers async def avalidate_environment( diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 3d4576fd7ac..16ec4249244 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -5,9 +5,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ from collections.abc import Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import Final, Protocol, cast +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.url_utils import join_api_base @@ -48,17 +50,55 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe pass +def azure_realtime_requires_ga(model: str) -> bool: + try: + azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + try: + openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + openai_entry: Final = openai_model_info.get("provider_specific_entry") + return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1 + azure_entry: Final = azure_model_info.get("provider_specific_entry") + return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 + + +def _is_ga_protocol(protocol: str | None) -> bool: + return protocol is not None and protocol.upper() in ("GA", "V1") + + +@dataclass(frozen=True, slots=True) +class AzureRealtimeProtocol: + name: str + bridges_beta_client_to_ga: bool = False + + def azure_realtime_protocol_for_client( configured_protocol: object, *, + model: str, + base_model: str | None = None, + realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, -) -> str: +) -> AzureRealtimeProtocol: + serves_a_ga_only_session: Final = ( + realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription" + ) + client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket) if isinstance(configured_protocol, str) and configured_protocol: - return configured_protocol - if (query_params or {}).get("intent") == "transcription": - return "GA" - return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" + return AzureRealtimeProtocol( + name=configured_protocol, + bridges_beta_client_to_ga=client_wants_beta + and not serves_a_ga_only_session + and _is_ga_protocol(configured_protocol), + ) + if serves_a_ga_only_session: + return AzureRealtimeProtocol(name="GA") + if azure_realtime_requires_ga(base_model or model): + return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta) + return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA") class _ProxyClientWebSocket(Protocol): @@ -92,6 +132,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version: str | None, realtime_protocol: str | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", ) -> str: """ Construct Azure realtime WebSocket URL. @@ -104,6 +145,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): - "GA" or "v1": Uses /openai/v1/realtime (GA path) - "beta" or None: Uses /openai/realtime (beta path, default) query_params: Extra query params to forward (e.g. intent=transcription). + realtime_mode: Realtime surface to target, "realtime" or "translation". Returns: WebSocket URL string @@ -117,24 +159,29 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_base = api_base.replace("https://", "wss://") # Determine path based on realtime_protocol (case-insensitive) - _is_ga: Final = realtime_protocol is not None and realtime_protocol.upper() in ( - "GA", - "V1", - ) + _is_ga: Final = _is_ga_protocol(realtime_protocol) intent: Final = (query_params or {}).get("intent") - if _is_ga: - path = "/openai/v1/realtime" - query_parts = [] - if intent != "transcription" and (query_params is None or "model" in query_params): - query_parts.append(urlencode({"model": model})) - else: - # Default to beta path for backwards compatibility - path = "/openai/realtime" - query_parts = [urlencode({"api-version": api_version, "deployment": model})] + path: Final = ( + "/openai/v1/realtime/translations" + if realtime_mode == "translation" + else "/openai/v1/realtime" + if _is_ga + else "/openai/realtime" + ) + base_query_parts: Final = ( + (urlencode((("model", model),)),) + if realtime_mode == "translation" + else ( + (urlencode((("model", model),)),) + if intent != "transcription" and (query_params is None or "model" in query_params) + else () + ) + if _is_ga + else (urlencode((("api-version", api_version), ("deployment", model))),) + ) - if intent: - query_parts.append(urlencode({"intent": intent})) + query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts qs: Final = "&".join(query_parts) endpoint_url: Final = join_api_base(api_base, path) @@ -165,13 +212,16 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", + bridges_beta_client_to_ga: bool = False, + drop_params: bool = False, ): import websockets from websockets.asyncio.client import ClientConnection if api_base is None: raise ValueError("api_base is required for Azure OpenAI calls") - backend_uses_beta_protocol: Final = realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1") + backend_uses_beta_protocol: Final = not _is_ga_protocol(realtime_protocol) if api_version is None and backend_uses_beta_protocol: raise ValueError("api_version is required for Azure OpenAI calls") @@ -181,6 +231,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, ) auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token) @@ -204,6 +255,9 @@ class AzureOpenAIRealtime(AzureChatCompletion): force_transcription_model=( model if (query_params or {}).get("intent") == "transcription" else None ), + translation_session=realtime_mode == "translation", + bridges_beta_client_to_ga=bridges_beta_client_to_ga, + drop_params=drop_params, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 545c3348182..64051b2c3c9 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -1,11 +1,20 @@ """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" +from collections.abc import Mapping from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.azure.common_utils import get_azure_ad_token from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +_CALLER_AUTH_HEADERS: Final = frozenset({"authorization", "api-key"}) + + +def _azure_resource_base(api_base: str) -> str: + return api_base.rstrip("/").removesuffix("/openai/v1").removesuffix("/openai") def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: @@ -21,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_api_key(self, api_key: str | None, **kwargs: object) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if api_key or any(name.lower() in _CALLER_AUTH_HEADERS for name in extra_headers or {}): + return extra_headers + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token is None: + return extra_headers + return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"} + def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets") @@ -30,20 +52,30 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - return { + validated_headers: Final = { **headers, - "api-key": api_key or "", "Content-Type": "application/json", } + if api_key: + validated_headers["api-key"] = api_key + return validated_headers def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: - base: Final = self.get_api_base(api_base) + base: Final = _azure_resource_base(self.get_api_base(api_base)) version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets") + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls") + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]: return { "Authorization": f"Bearer {ephemeral_key}", diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 6c954d00e64..f95dc1e4388 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -3,11 +3,10 @@ from collections.abc import Callable, Mapping from copy import deepcopy from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, cast from urllib.parse import urlsplit import httpx -from openai.types.responses import ResponseReasoningItem from pydantic import TypeAdapter from litellm._logging import verbose_logger @@ -16,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cach from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * @@ -72,6 +72,21 @@ def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[st return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)}) +_AZURE_REJECTED_INPUT_FIELDS: Final = MappingProxyType( + {"message": frozenset({"status"}), "reasoning": frozenset({"status"})} +) +_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS: Final = MappingProxyType( + {**_AZURE_REJECTED_INPUT_FIELDS, "reasoning": frozenset({"status", "content"})} +) + + +def _without_fields_azure_rejects(item: object, rejected_fields: Mapping[str, frozenset[str]]) -> object: + if not is_str_keyed_mapping(item): + return item + rejected: Final = rejected_fields.get(str(item.get("type")), frozenset()) + return {key: value for key, value in item.items() if key not in rejected} if rejected else item + + class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Parameters not supported by Azure Responses API AZURE_UNSUPPORTED_PARAMS = ["context_management"] @@ -122,68 +137,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_stripped_model_name(self, model: str) -> str: return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") - def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: - """ - Handle reasoning items to filter out the status field. - Issue: https://github.com/BerriAI/litellm/issues/13484 - - Azure OpenAI API does not accept 'status' field in reasoning input items. - """ - if item.get("type") == "reasoning": - try: - # Ensure required fields are present for ResponseReasoningItem - item_data: Final = dict(item) - if "summary" not in item_data: - item_data["summary"] = ( - item_data.get("reasoning_content", "")[:100] + "..." - if len(item_data.get("reasoning_content", "")) > 100 - else item_data.get("reasoning_content", "") - ) - - # Create ResponseReasoningItem object from the item data - reasoning_item: Final = ResponseReasoningItem.model_validate(item_data) - - # Convert back to dict with exclude_none=True to exclude None fields - dict_reasoning_item: Final = reasoning_item.model_dump(exclude_none=True) - dict_reasoning_item.pop("status", None) - - return dict_reasoning_item - except Exception as e: - verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e) - # Fallback: manually filter out known None fields - filtered_item: Final = { - k: v - for k, v in item.items() - if v is not None or k not in {"status", "content", "encrypted_content"} - } - return filtered_item - return item - def _validate_input_param( - self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + self, input: str | ResponseInputParam, *, drop_params: bool = False ) -> str | ResponseInputParam: - """ - Override parent method to also filter out 'status' field from message items. - Azure OpenAI API does not accept 'status' field in input messages. - """ - from typing import cast - - # First call parent's validation - validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by) - - # Then filter out status from message items - if isinstance(validated_input, list): - filtered_input: Final[list[object]] = [] - for item in validated_input: - if isinstance(item, dict) and item.get("type") == "message": - # Filter out status field from message items - filtered_item = {k: v for k, v in item.items() if k != "status"} - filtered_input.append(filtered_item) - else: - filtered_input.append(item) - return cast(ResponseInputParam, filtered_input) - - return validated_input + validated_input: Final = super()._validate_input_param(input, drop_params=drop_params) + if not isinstance(validated_input, list): + return validated_input + rejected_fields: Final = ( + _AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS if drop_params else _AZURE_REJECTED_INPUT_FIELDS + ) + sanitized_items: Final = [_without_fields_azure_rejects(item, rejected_fields) for item in validated_input] + return cast("ResponseInputParam", sanitized_items) # cast-ok: items keep their shape minus rejected fields def transform_responses_api_request( self, diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py index e255a667606..6d0241d4d3b 100644 --- a/litellm/llms/azure/search/transformation.py +++ b/litellm/llms/azure/search/transformation.py @@ -36,7 +36,7 @@ import httpx from pydantic import ConfigDict, ValidationError from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider +from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider, entra_rejection_as_auth_error from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.responses import ( ResponsesSearchAnnotation, @@ -270,6 +270,9 @@ class BingGroundingSearchConfig(BaseSearchConfig): try: return minter() except Exception as e: + rejection: Final = entra_rejection_as_auth_error(e) + if rejection is not None: + raise rejection from e raise ValueError( f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} " f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / " diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 97ee44bd3bb..3609871fdf8 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -16,6 +16,7 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.azure.common_utils import EntraRejectionError from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) @@ -135,6 +136,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): except AnthropicError: # Re-raise Anthropic exceptions as-is raise + except EntraRejectionError as rejection: + raise AnthropicError(status_code=rejection.status_code, message=rejection.message) from rejection except httpx.HTTPStatusError as e: # HTTP errors - preserve the actual status code verbose_logger.error("HTTP error in CountTokens handler: %s", e) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 5214d31fb2e..487958d6d55 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -10,7 +10,11 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + declared_prompt_cache_breakpoint, + routed_model_info, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( content_parts, convert_content_list_to_str, @@ -125,18 +129,14 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: + model_info: Final = routed_model_info(litellm_params.get("model_info")) + declared: Final = declared_prompt_cache_breakpoint(model_info) + if declared is not None: + return declared base_model: Final = litellm_params.get("base_model") named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,) - model_info: Final = litellm_params.get("model_info") - if not isinstance(model_info, Mapping): - return any( - AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models - ) - deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint") - if isinstance(deployment_flag, bool): - return deployment_flag return any( - AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai", model_info=model_info) for name in named_models ) diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index 147bf69e91e..1635b9b5679 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -1,5 +1,7 @@ import asyncio -from collections.abc import Mapping +import os +from collections.abc import Callable, Mapping +from functools import partial from types import MappingProxyType from typing import Final, Literal from urllib.parse import urlparse @@ -52,6 +54,28 @@ AZURE_ENTRA_CREDENTIAL_PARAM_KEYS: Final = frozenset({"azure_ad_token", "client_ AZURE_ENTRA_LITELLM_PARAM_KEYS: Final = AZURE_ENTRA_CREDENTIAL_PARAM_KEYS | frozenset( {"tenant_id", "client_id", "azure_username", "azure_scope"} ) +AZURE_DEFAULT_SCOPE: Final = "https://cognitiveservices.azure.com/.default" +_AZURE_ENTRA_ENVIRONMENT_VARIABLES: Final = MappingProxyType( + { + "tenant_id": "AZURE_TENANT_ID", + "client_id": "AZURE_CLIENT_ID", + "client_secret": "AZURE_CLIENT_SECRET", + "azure_username": "AZURE_USERNAME", + "azure_password": "AZURE_PASSWORD", + "azure_ad_token": "AZURE_AD_TOKEN", + } +) + + +def azure_entra_environment_defaults() -> Mapping[str, object]: + return MappingProxyType( + { + **{key: os.getenv(variable) for key, variable in _AZURE_ENTRA_ENVIRONMENT_VARIABLES.items()}, + "azure_scope": os.getenv("AZURE_SCOPE", AZURE_DEFAULT_SCOPE), + } + ) + + AZURE_ENTRA_CREDENTIAL_HELP: Final = ( "Set `tenant_id` + `client_id` + `client_secret`, `azure_ad_token` (an `oidc/` token also needs " "`tenant_id` + `client_id`), or `client_id` + `azure_username` + `azure_password` in the agent's `litellm_params`" @@ -70,6 +94,17 @@ def _resolve_config_secret(value: object) -> str | None: return get_secret_str(value) if value.startswith("os.environ/") else value +def _agent_entra_token(provider: Callable[[], str], model: str) -> str: + from litellm.llms.azure.common_utils import EntraRejectionError, entra_token + + try: + return entra_token(provider) + except EntraRejectionError as rejection: + raise litellm.AuthenticationError( + message=rejection.message, llm_provider="azure_ai", model=model + ) from rejection + + def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str: """Mints the Entra bearer from the agent's own litellm_params, never from process-wide AZURE_* env vars.""" from litellm.llms.azure.common_utils import ( @@ -88,18 +123,32 @@ def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str: azure_username: Final = resolved["azure_username"] azure_password: Final = resolved["azure_password"] azure_ad_token: Final = resolved["azure_ad_token"] + model: Final = str(litellm_params.get("model") or "") if tenant_id and client_id and client_secret: - return get_azure_ad_token_from_entra_id( - tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope - )() + return _agent_entra_token( + get_azure_ad_token_from_entra_id( + tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope + ), + model, + ) if client_id and azure_username and azure_password: - return get_azure_ad_token_from_username_password( - client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope - )() + return _agent_entra_token( + get_azure_ad_token_from_username_password( + client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope + ), + model, + ) federated: Final = azure_ad_token is not None and azure_ad_token.startswith("oidc/") if azure_ad_token and federated and tenant_id and client_id: - return get_azure_ad_token_from_oidc( - azure_ad_token=azure_ad_token, azure_client_id=client_id, azure_tenant_id=tenant_id, scope=scope + return _agent_entra_token( + partial( + get_azure_ad_token_from_oidc, + azure_ad_token=azure_ad_token, + azure_client_id=client_id, + azure_tenant_id=tenant_id, + scope=scope, + ), + model, ) if azure_ad_token and not federated: return azure_ad_token diff --git a/litellm/llms/azure_ai/rerank/transformation.py b/litellm/llms/azure_ai/rerank/transformation.py index 4c26ba58dd8..2131a8c7cfa 100644 --- a/litellm/llms/azure_ai/rerank/transformation.py +++ b/litellm/llms/azure_ai/rerank/transformation.py @@ -111,10 +111,4 @@ class AzureAIRerankConfig(CohereRerankConfig): return rerank_response def _get_base_model(self, azure_model_group: str | None) -> str | None: - if azure_model_group is None: - return None - if azure_model_group == "offer-cohere-rerank-mul-paygo": - return "azure_ai/cohere-rerank-v3-multilingual" - if azure_model_group == "offer-cohere-rerank-eng-paygo": - return "azure_ai/cohere-rerank-v3-english" return azure_model_group diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..68365372e7e 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,9 +1,30 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Final +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Final, + cast, # noqa: TID251 # SDK file handles have no runtime validation schema +) import httpx +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import ( + Transcription, + TranscriptionDiarized, + TranscriptionStreamEvent, + TranscriptionVerbose, +) +from openai.types.audio.transcription_create_params import ( + TranscriptionCreateParamsBase, + TranscriptionCreateParamsStreaming, +) +from pydantic import BaseModel, TypeAdapter +from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -11,6 +32,31 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse +_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object]) +_TRANSCRIPTION_TEXT: Final = TypeAdapter(str) +_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object]) +_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType( + {"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized} +) + + +def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]: + request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data) + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data} + if not extension_body: + return request_data + + existing_extra_body: Final = request_data.get("extra_body") + return { + **{key: value for key, value in request_data.items() if key not in extension_keys}, + "extra_body": { + **(as_str_mapping(existing_extra_body) or {}), + **extension_body, + }, + } + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -20,6 +66,77 @@ else: LiteLLMLoggingObj = Any +def sdk_compatible_transcription_response(response: object) -> BaseModel | str: + if isinstance(response, BaseModel): + return response + return _TRANSCRIPTION_TEXT.validate_python(response, strict=True) + + +def _content_type(http_response: httpx.Response) -> str: + return next(iter(http_response.headers.get_list("content-type")), "") + + +def _is_event_stream(http_response: httpx.Response) -> bool: + return _content_type(http_response).startswith("text/event-stream") + + +def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str: + if not _content_type(http_response).startswith("application/json"): + return http_response.text + transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription) + return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content)) + + +def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"} + ) + + +def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, data + ) + + +def create_sdk_transcription( + client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]: + if not transcription_stream_requested(data.get("stream")): + raw_response: Final = client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = client.audio.transcriptions.with_raw_response.create( + **_streaming_params({**data, "stream": True}), timeout=timeout + ) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent]) + raw_stream.http_response.read() + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) + + +async def acreate_sdk_transcription( + client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]: + if not transcription_stream_requested(data.get("stream")): + raw_response: Final = await client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = await client.audio.transcriptions.with_raw_response.create( + **_streaming_params({**data, "stream": True}), timeout=timeout + ) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent]) + await raw_stream.http_response.aread() + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) + + @dataclass class AudioTranscriptionRequestData: """ diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index c562c8d0f04..254783edf39 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -431,6 +431,10 @@ class DecisionsResponseValidationError(BaseLLMException): pass +def _lacks_confidence(answer: DecisionsIRAnswer) -> bool: + return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None + + def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return systemone_response_to_ir( _SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request @@ -534,6 +538,8 @@ class BaseDecisionsConfig(ABC): max_total_image_bytes: int | None = None image_media_types: frozenset[str] | None = None inlines_remote_images: bool = False + openai_evidence_fields: frozenset[str] = frozenset() + confidence_optional: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) @@ -682,13 +688,20 @@ class BaseDecisionsConfig(ABC): ) -> DecisionsIRResponse: try: payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) - return self.parse_response(payload, request) + response: Final = self.parse_response(payload, request) except ValidationError as error: raise DecisionsResponseValidationError( status_code=500, message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " f"{error.title} {_validation_summary(error)}", ) from error + if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers): + raise DecisionsResponseValidationError( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " + "a choice or score answer has no confidence", + ) + return response def get_error_class( self, diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index a02054ec8cb..c1c943c2cd6 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Final import httpx from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.types.router import GenericLiteLLMParams class BaseRealtimeHTTPConfig(ABC): @@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC): explicit api_key → litellm.api_key → env var → "" """ + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + return extra_headers + # ------------------------------------------------------------------ # # client_secrets endpoint # # ------------------------------------------------------------------ # @@ -64,6 +75,12 @@ class BaseRealtimeHTTPConfig(ABC): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/client_secrets" + @abstractmethod def validate_environment( self, @@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC): """Return the full URL for POST /realtime/calls (SDP exchange).""" return join_api_base(api_base or "", "v1/realtime/calls") + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ Build headers for the realtime_calls POST. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2d185068815..03fbc35a97f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -22,6 +22,12 @@ from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_hea from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, +) +from litellm.integrations.anthropic_cache_control_hook import ( + MAX_CACHE_CONTROL_BLOCKS, + AnthropicCacheControlHook, + configured_injection_points, ) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, @@ -131,6 +137,7 @@ from ..common_utils import ( bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, + bedrock_model_known_to_accept_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, get_anthropic_beta_from_headers, @@ -204,6 +211,11 @@ def _has_native_text_format(output_config: object) -> bool: return False +def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]: + cache_control: Final = block.get("cache_control") + return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {} + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -243,7 +255,7 @@ class AmazonConverseConfig(BaseConfig): @staticmethod def _convert_consecutive_user_messages_to_guarded_text( - messages: list[AllMessageValues], optional_params: dict + messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False ) -> list[AllMessageValues]: """ Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present @@ -281,7 +293,11 @@ class AmazonConverseConfig(BaseConfig): new_content = [] for item in content: if isinstance(item, dict) and item.get("type") == "text": - new_item = {"type": "guarded_text", "text": item["text"]} + new_item = { + "type": "guarded_text", + "text": item["text"], + **_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks), + } new_content.append(new_item) else: new_content.append(item) @@ -289,7 +305,13 @@ class AmazonConverseConfig(BaseConfig): messages_copy[user_message_index]["content"] = new_content elif isinstance(content, str): # If content is a string, convert it to guarded_text - messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}] + messages_copy[user_message_index]["content"] = [ + { + "type": "guarded_text", + "text": content, + **_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks), + } + ] return messages_copy @@ -929,19 +951,24 @@ class AmazonConverseConfig(BaseConfig): def _supports_native_structured_outputs( model: str, custom_llm_provider: str | None, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and - ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the - request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot - force the json tool gets it unless that key is false. + ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude + will think, which the json tool path would drop. When the request does not ask for thinking, a model + that cannot force the json tool gets it unless that key is false. Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html """ return bedrock_chat_uses_native_structured_output( - model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info + model, + will_think=will_think, + thinking_requested=thinking_requested, + custom_llm_provider=custom_llm_provider or "bedrock_converse", + model_info=model_info, ) @staticmethod @@ -984,11 +1011,13 @@ class AmazonConverseConfig(BaseConfig): self, model: str, json_schema: Mapping[str, object], + *, + will_think: bool, thinking_requested: bool, model_info: Mapping[str, object] | None, ) -> Mapping[str, object] | None: if not self._supports_native_structured_outputs( - model, self.custom_llm_provider, thinking_requested, model_info + model, self.custom_llm_provider, will_think, thinking_requested, model_info ): return None if BedrockModelInfo.get_base_model(model).startswith("anthropic"): @@ -1074,7 +1103,6 @@ class AmazonConverseConfig(BaseConfig): request_params: Final = tool_params_without_hosted_mcp( non_default_params, model=model, route="bedrock", drop_params=drop_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) base_model: Final = BedrockModelInfo.get_base_model(model) drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model) drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param( @@ -1088,7 +1116,6 @@ class AmazonConverseConfig(BaseConfig): model=model, optional_params=optional_params, non_default_params=request_params, - is_thinking_enabled=is_thinking_enabled, model_info=model_info, ) if param == "max_tokens" or param == "max_completion_tokens": @@ -1293,7 +1320,6 @@ class AmazonConverseConfig(BaseConfig): model: str, optional_params: dict, non_default_params: dict, - is_thinking_enabled: bool, model_info: Mapping[str, object] | None = None, ) -> dict: """ @@ -1318,9 +1344,16 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params - thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)) + request_params: Final = REQUEST_PARAMS.validate_python(non_default_params) + thinking_requested: Final = self.is_thinking_requested(request_params) native_schema: Final = ( - self._native_output_schema(model, json_schema, thinking_requested, model_info) + self._native_output_schema( + model, + json_schema, + will_think=AnthropicModelInfo.will_think(model, "bedrock", request_params), + thinking_requested=thinking_requested, + model_info=model_info, + ) if json_schema is not None else None ) @@ -1338,17 +1371,24 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_is_forced: Final = litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) and AnthropicModelInfo.response_format_tool_can_be_forced(model, "bedrock", request_params) + schema_description: Final = description if isinstance(description, str) else None + instructs_unforced_tool: Final = not tool_is_forced and BedrockModelInfo.get_base_model(model).startswith( + "anthropic" + ) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, schema_description))) + if instructs_unforced_tool + else schema_description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) - if ( - litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) - ): + if tool_is_forced: optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) ) @@ -1359,8 +1399,8 @@ class AmazonConverseConfig(BaseConfig): # returns {} instead of the requested JSON. The model already # returns JSON when the prompt asks for it. - # A request that goes native only because the model can't force the json tool keeps the json tool's faked - # stream, so it still needs only bedrock:InvokeModel. + # A request that goes native without asking for thinking keeps the json tool's faked stream, so it still + # needs only bedrock:InvokeModel, unless the model takes native output on every request. streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output( model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info ) @@ -1420,6 +1460,7 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["system"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> SystemContentBlock | None: pass @@ -1433,6 +1474,7 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["content_block"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> ContentBlock | None: pass @@ -1445,9 +1487,10 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["system", "content_block"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> SystemContentBlock | ContentBlock | None: cache_control: Final = message_block.get("cache_control", None) - if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model): + if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model, model_info): return None cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model) @@ -1592,8 +1635,80 @@ class AmazonConverseConfig(BaseConfig): return ChatCompletionTextObject(type="text", text=text) return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control) + def _messages_with_cache_marks( + self, + messages: Sequence[AllMessageValues], + model: str, + deployment_model: str, + drop_params: bool, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> Sequence[AllMessageValues]: + if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info): + return messages + markable: Final = ( + AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model) + if "nova" in model.lower() + else messages + ) + return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + markable, + drop_params, + AnthropicCacheControlHook.count_converse_cache_points, + lambda: self._tool_cache_points_on_the_wire( + model, deployment_model, optional_params, litellm_params, model_info + ), + ) + + def _tool_cache_points_on_the_wire( + self, + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> int: + tools: Final = optional_params.get("tools") + bedrock_tools, _ = self._process_tools_and_beta( + tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info + ) + injects_tool_config_point: Final = bool(bedrock_tools) and any( + point.get("location") == "tool_config" + for point in configured_injection_points(optional_params.get("cache_control_injection_points")) + ) + return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point) + + def _guarded_text_keeps_cache_marks( + self, + messages: Sequence[AllMessageValues], + system_content_blocks: Sequence[SystemContentBlock], + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> bool: + if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points( + model, deployment_model, model_info + ): + return False + message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages)) + if message_cache_points == 0: + return False + cache_points: Final = ( + sum(1 for block in system_content_blocks if "cachePoint" in block) + + message_cache_points + + self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info) + ) + return cache_points <= MAX_CACHE_CONTROL_BLOCKS + def _transform_system_message( - self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None + self, + messages: list[AllMessageValues], + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> tuple[list[AllMessageValues], list[SystemContentBlock]]: leading_count: Final = next( (i for i, m in enumerate(messages) if not is_system_message(m)), @@ -1609,7 +1724,7 @@ class AmazonConverseConfig(BaseConfig): if isinstance(content, str) and content: system_content_blocks.append(SystemContentBlock(text=content)) cache_block = self.get_cache_point_block( - message, block_type="system", model=model, deployment_model=deployment_model + message, block_type="system", model=model, deployment_model=deployment_model, model_info=model_info ) if cache_block: system_content_blocks.append(cache_block) @@ -1618,7 +1733,11 @@ class AmazonConverseConfig(BaseConfig): if m.get("type") == "text" and m.get("text"): system_content_blocks.append(SystemContentBlock(text=m["text"])) cache_block = self.get_cache_point_block( - m, block_type="system", model=model, deployment_model=deployment_model + m, + block_type="system", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) if cache_block: system_content_blocks.append(cache_block) @@ -1816,6 +1935,7 @@ class AmazonConverseConfig(BaseConfig): additional_request_params: dict, litellm_params: Mapping[str, object] | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> tuple[list[ToolBlock], list]: """Process tools and collect anthropic_beta values.""" bedrock_tools: list[ToolBlock] = [] @@ -1877,6 +1997,7 @@ class AmazonConverseConfig(BaseConfig): _tools_the_model_accepts(regular_tools, model, litellm_params), model=model, deployment_model=deployment_model, + model_info=model_info, ) # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) @@ -1946,6 +2067,7 @@ class AmazonConverseConfig(BaseConfig): _tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params), model=model, deployment_model=deployment_model, + model_info=model_info, ) # Append pre-formatted tools (systemTool etc.) after transformation @@ -2021,6 +2143,7 @@ class AmazonConverseConfig(BaseConfig): drop_params: bool = False, litellm_params: Mapping[str, object] | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> CommonRequestObject: # Drop thinking param if thinking is enabled but thinking_blocks are missing # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" @@ -2059,7 +2182,7 @@ class AmazonConverseConfig(BaseConfig): # Process tools and collect beta values bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( - original_tools, model, headers, additional_request_params, litellm_params, deployment_model + original_tools, model, headers, additional_request_params, litellm_params, deployment_model, model_info ) requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict @@ -2077,7 +2200,7 @@ class AmazonConverseConfig(BaseConfig): if ( cache_injection_points and len(bedrock_tools) > 0 - and bedrock_model_accepts_cache_points(model, deployment_model) + and bedrock_model_accepts_cache_points(model, deployment_model, model_info) ): for point in cache_injection_points: if point.get("location") == "tool_config": @@ -2115,8 +2238,8 @@ class AmazonConverseConfig(BaseConfig): } if additional_request_params: data["additionalModelRequestFields"] = additional_request_params - if "thinking" in additional_request_params: - data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) + if "thinking" in additional_request_params or AnthropicModelInfo.thinks_by_default(model, "bedrock"): + data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",) if system_content_blocks: data["system"] = system_content_blocks @@ -2137,22 +2260,27 @@ class AmazonConverseConfig(BaseConfig): return data - @staticmethod def _claude_messages_within_cache_control_cap( + self, messages: Sequence[AllMessageValues], capability_model: str, + deployment_model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, ) -> Sequence[AllMessageValues]: if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): return messages from litellm.integrations.anthropic_cache_control_hook import ( - messages_within_cache_control_cap, + messages_within_reserved_cache_control_cap, tool_message_with_single_cache_mark, ) - return messages_within_cache_control_cap( - tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + return messages_within_reserved_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), + self._tool_cache_points_on_the_wire( + capability_model, deployment_model, optional_params, litellm_params, model_info + ), ) async def _async_transform_request( @@ -2163,27 +2291,53 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: + from litellm.integrations.anthropic_cache_control_hook import routed_model_info + base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) + model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, deployment_model=model, + model_info=model_info, ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) ## TRANSFORMATION ## _data: Final[CommonRequestObject] = self._transform_request_helper( model=capability_model, deployment_model=model, + model_info=model_info, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2198,6 +2352,7 @@ class AmazonConverseConfig(BaseConfig): messages=messages, model=capability_model, deployment_model=model, + model_info=model_info, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), @@ -2250,26 +2405,52 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: + from litellm.integrations.anthropic_cache_control_hook import routed_model_info + base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) + model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, deployment_model=model, + model_info=model_info, ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) _data: Final[CommonRequestObject] = self._transform_request_helper( model=capability_model, deployment_model=model, + model_info=model_info, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2285,6 +2466,7 @@ class AmazonConverseConfig(BaseConfig): messages=messages, model=capability_model, deployment_model=model, + model_info=model_info, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index e2f12b123de..62e1bee5217 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -13,8 +13,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( + RemoteMedia, async_inline_remote_media, convert_url_to_base64, + inline_every_remote_url, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.common_utils import ( @@ -107,8 +109,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): self._read_configured_max_tokens: Final = read_configured_max_tokens @classmethod - def get_config(cls, *, model: str | None = None) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict - return {key: value for key, value in super().get_config(model=model).items() if key != "max_tokens"} + def get_config( + cls, *, model: str | None = None, configured_max_output_tokens: int | None = None + ) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict + return { + key: value + for key, value in super() + .get_config(model=model, configured_max_output_tokens=configured_max_output_tokens) + .items() + if key != "max_tokens" + } @property def custom_llm_provider(self) -> str | None: @@ -117,6 +127,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + def inlines_remote_media(self, media: RemoteMedia) -> bool: + return inline_every_remote_url(media) + def get_supported_openai_params(self, model: str) -> list[str]: return AnthropicConfig.get_supported_openai_params(self, model) @@ -129,14 +142,19 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - return self.chat_uses_native_structured_output( - model, self.is_thinking_requested(request_params), self._deployment_model_info - ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + return self.chat_uses_native_structured_output(model, request_params, self._deployment_model_info) and ( + json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None + ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_uses_native_structured_output( + model, + will_think=self.will_think(model, "bedrock", request_params), + thinking_requested=self.is_thinking_requested(request_params), + model_info=model_info, + ) def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] @@ -346,7 +364,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): request_body=anthropic_request, model_info=model_info, chat_uses_native_output=lambda: self.chat_uses_native_structured_output( - model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info + model, _NON_DEFAULT_PARAMS.validate_python(filtered_params), model_info ), ) strip_unsupported_bedrock_invoke_output_config_keys( diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index 952ca7a2cd4..7e635a61d4d 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -55,9 +55,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): ) def chat_uses_native_structured_output( - self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None + self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None ) -> bool: - return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info) + return bedrock_chat_opts_into_native_structured_output( + model, self.will_think(model, "bedrock", request_params), model_info=model_info + ) def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]: return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params)) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 3126450b287..dfee28a7cc9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -33,6 +33,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.core_helpers import lax_bool from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -355,13 +356,15 @@ def bedrock_chat_opts_into_native_structured_output( def bedrock_chat_uses_native_structured_output( model: str, + *, + will_think: bool, thinking_requested: bool, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> bool: - return bedrock_chat_opts_into_native_structured_output( - model, thinking_requested, custom_llm_provider, model_info - ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info)) + return bedrock_chat_opts_into_native_structured_output(model, will_think, custom_llm_provider, model_info) or ( + not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info) + ) def bedrock_native_output_is_deployment_opt_in( @@ -1515,7 +1518,9 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: ) -def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool: +def bedrock_model_accepts_cache_points( + model: str | None, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: """ Whether Converse ``cachePoint`` blocks may be sent to this model. @@ -1535,43 +1540,102 @@ def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here: it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the model as configured when ``model`` is its ``base_model``: its own declaration wins. + A routed call passes its deployment's ``model_info``, whose breakpoint flag wins outright and + whose ``supports_prompt_caching`` wins over the catalog's for a known model. Other flags come + from the catalog as loaded, since the registered keys are shared by every deployment of the + same model string and carry whichever sibling registered last. """ if model is None: return True deployment_declared: Final = ( - None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model) + None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model, model_info) ) declared: Final = ( - deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model) + deployment_declared + if deployment_declared is not None + else bedrock_declared_cache_point_support(model, model_info) ) return declared is not False -def bedrock_declared_cache_point_support(model: str) -> bool | None: +def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None: if _OPENAI_FAMILY_MODEL_RE.search(model): return False - map_keys: Final = (model, get_bedrock_base_model(model)) + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + return _mapped_prompt_caching_support(model, model_info) + + +def bedrock_model_known_to_accept_cache_points( + model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: + """Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents + as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI + ``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for + Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field.""" + deployment_declared: Final = ( + None if deployment_model is None else _declared_marker_support(deployment_model, model_info) + ) + declared: Final = ( + deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info) + ) + return declared is True + + +def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + if _OPENAI_FAMILY_MODEL_RE.search(model): + return False + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + mapped: Final = _mapped_prompt_caching_support(model, model_info) + if not _documents_cache_checkpoints(model): + return None if mapped is None else False + return mapped is not False + + +def _documents_cache_checkpoints(model: str) -> bool: + return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova") + + +def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") + if deployment_flag is not None: + return deployment_flag is True + catalog_entry: Final = _catalog_entry_reader(model_info) registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/")) - explicit_marker_support: Final = next( + return next( ( entry.get("supports_prompt_cache_breakpoint") is True - for key in (*registered_keys, *map_keys) - if (entry := litellm.model_cost.get(key)) is not None - and entry.get("supports_prompt_cache_breakpoint") is not None + for key in (*registered_keys, model, get_bedrock_base_model(model)) + if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None ), None, ) - if explicit_marker_support is not None: - return explicit_marker_support - if not any(key in litellm.model_cost for key in map_keys): + + +def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + catalog_entry: Final = _catalog_entry_reader(model_info) + map_keys: Final = (model, get_bedrock_base_model(model)) + if not any(catalog_entry(key) is not None for key in map_keys): return None + deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) + if deployment_caching is not None: + return deployment_caching return any( - entry.get("supports_prompt_caching") is True - for key in map_keys - if (entry := litellm.model_cost.get(key)) is not None + entry.get("supports_prompt_caching") is True for key in map_keys if (entry := catalog_entry(key)) is not None ) +def _catalog_entry_reader(model_info: Mapping[str, object] | None) -> Callable[[str], Mapping[str, object] | None]: + from litellm.utils import catalog_with_runtime_registrations + + if model_info is not None: + return catalog_with_runtime_registrations().get + return _model_cost_entry + + def bedrock_supports_tool_search(model: str) -> bool: """ Whether Bedrock InvokeModel admits the ``tool_search_tool_*`` tool types on ``model``. diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 5eb46944f9a..1d293bdc99c 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -455,9 +455,7 @@ def _converse_request( if not all(_text_only(message) for message in messages): return None try: - chat_tools: Final = tuple( - chat_tool for tool in tools or () for chat_tool in _as_converse_input(tool) - ) # comprehension-ok: flattens each tool's translation + chat_tools: Final = tuple(chain.from_iterable(_as_converse_input(tool) for tool in tools or ())) return _ConverseRequest.model_validate( litellm.AmazonConverseConfig().transform_request( model=model, diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index 3a65a8928bf..3514a5e6e5c 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -14,6 +14,7 @@ from pydantic import InstanceOf, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -60,6 +61,8 @@ if TYPE_CHECKING: _CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"}) _INVOKE_STREAM_ACTION: Final = "invoke-with-response-stream" +_INVOKE_ACTIONS: Final = frozenset({"invoke", _INVOKE_STREAM_ACTION}) +_BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN: Final = frozenset({"document", "search_result"}) _EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream" _EVENT_STREAM_MEDIA_TYPE: Final = "application/vnd.amazon.eventstream" @@ -73,6 +76,83 @@ def _is_invoke_stream_endpoint(endpoint: str) -> bool: return endpoint.rstrip("/").split("/")[-1] == _INVOKE_STREAM_ACTION +def _anthropic_invoke_body( + data: Mapping[str, object], +) -> dict[str, object] | None: # mutable-ok: the body guardrails write back into + """The body of an InvokeModel call in Anthropic Messages format, which Bedrock requires to carry anthropic_version""" + endpoint: Final = data.get("endpoint") + body: Final = data.get("data") + if not isinstance(endpoint, str) or endpoint.rstrip("/").split("/")[-1] not in _INVOKE_ACTIONS: + return None + if not isinstance(body, dict): + return None + invoke_body: Final = _BODY_BY_REFERENCE.validate_python(body) + if not isinstance(invoke_body.get("anthropic_version"), str) or not isinstance(invoke_body.get("messages"), list): + return None + messages: Final = _ARRAY_BY_REFERENCE.validate_python(invoke_body["messages"]) + if not all(_is_well_formed_message(message) for message in messages): + return None + blocks: Final = _anthropic_message_blocks(messages) + if any(block.get("type") in _BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN for block in blocks): + return None + return invoke_body + + +def _body_level_request_data( + passthrough_request: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the request dict guardrails receive + """What `/v1/messages` hands a guardrail as `request_data`: the Anthropic body at the top level, with the proxy's own keys (model, metadata, headers) on top""" + body: Final = as_str_mapping(passthrough_request.get("data")) or {} + proxy_keys: Final = {key: value for key, value in passthrough_request.items() if key != "data"} + return {**body, **proxy_keys} + + +def _is_well_formed_message(message: object) -> bool: + """Whether the Messages translation can read `message`, so a malformed body still reaches Bedrock to be rejected there""" + fields: Final = as_str_mapping(message) + if fields is None or not isinstance(fields.get("role"), str): + return False + content: Final = fields.get("content") + if isinstance(content, str): + return True + return isinstance(content, list) and all( + as_str_mapping(block) is not None for block in _ARRAY_BY_REFERENCE.validate_python(content) + ) + + +def _anthropic_message_blocks(messages: Sequence[object]) -> tuple[Mapping[str, object], ...]: + fields: Final = (as_str_mapping(message) for message in messages) + return tuple(chain.from_iterable(_anthropic_content_blocks(field.get("content")) for field in fields if field)) + + +def _anthropic_content_blocks(content: object) -> tuple[Mapping[str, object], ...]: + blocks: Final[list[Mapping[str, object]]] = [] # mutable-ok: collected while walking the content + pending: Final[list[object]] = [content] # mutable-ok: content lists still to walk, nested tool results included + while pending: + current = pending.pop() + if not isinstance(current, list): + continue + for item in _ARRAY_BY_REFERENCE.validate_python(current): + block = as_str_mapping(item) + if block is None: + continue + blocks.append(block) + if block.get("type") == "tool_result": + pending.append(block.get("content")) + return tuple(blocks) + + +def _anthropic_invoke_reply( + response: object, + request_data: object, +) -> dict[str, object] | None: # mutable-ok: the reply guardrails write back into + if not isinstance(response, dict) or not isinstance(request_data, dict): + return None + if _anthropic_invoke_body(_OBJECT_BY_REFERENCE.validate_python(request_data)) is None: + return None + return _BODY_BY_REFERENCE.validate_python(response) + + def _generic_passthrough_handler() -> "PassThroughEndpointHandler": """ Fallback for non-Converse Bedrock routes (e.g. invoke). The generic @@ -1285,6 +1365,17 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): body: Final = data.get("data") if not _is_converse_endpoint(endpoint): + passthrough_request: Final = _BODY_BY_REFERENCE.validate_python(data) + anthropic_body: Final = _anthropic_invoke_body(passthrough_request) + if anthropic_body is not None: + await AnthropicMessagesHandler().process_input_messages( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict + data=anthropic_body, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + request_data=_body_level_request_data(passthrough_request), + keep_cache_control=True, + ) + return passthrough_request return await _generic_passthrough_handler().process_input_messages( data=data, guardrail_to_apply=guardrail_to_apply, @@ -1410,6 +1501,15 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): and not _is_converse_endpoint(endpoint) and not _is_invoke_stream_endpoint(endpoint) ): + anthropic_reply: Final = _anthropic_invoke_reply(response, request_data) # pyright: ignore[reportUnknownArgumentType] # the base signature types request_data as a bare dict + if anthropic_reply is not None: + return await AnthropicMessagesHandler().process_output_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict + response=anthropic_reply, # pyright: ignore[reportArgumentType] # the Anthropic reply dict, written back in place + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=_body_level_request_data(_OBJECT_BY_REFERENCE.validate_python(request_data or {})), + ) return await _generic_passthrough_handler().process_output_response( response=response, guardrail_to_apply=guardrail_to_apply, diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 00622a1e287..49939399ec8 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,7 +9,7 @@ import contextlib import importlib.metadata import json from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable @@ -34,12 +34,15 @@ from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateR from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import StandardLoggingPayloadErrorInformation from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata") _EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) @@ -61,6 +64,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType( ) +def _stream_ended_before_session_closed() -> BedrockError: + return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def _as_bedrock_error(error: BaseException) -> BaseException: status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__) if status_code is None: @@ -165,6 +172,12 @@ class _BridgeOutcome: client_disconnected: bool +@dataclass(frozen=True, slots=True) +class SessionEndSignals: + input_closing: asyncio.Event = field(default_factory=asyncio.Event) + output_failed: asyncio.Event = field(default_factory=asyncio.Event) + + async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]: if initial_message is not None: yield initial_message @@ -188,8 +201,8 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: raise BedrockError( status_code=400, message=( - "Bedrock realtime session already committed to a provider stream; it cannot be replayed" - + (f". The committed stream failed with: {committed_failure}" if committed_failure else "") + (f"{committed_failure}. " if committed_failure else "") + + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) @@ -201,11 +214,48 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None +def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None: + return next( + ( + (key, metadata) + for key in _SPEND_LOG_METADATA_KEYS + if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key))) + ), + None, + ) + + +def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: + """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" + litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python( + logging_obj.model_call_details.get("litellm_params") + ) + if litellm_params is None: + return + spend_log_metadata: Final = _spend_log_metadata(litellm_params) + if spend_log_metadata is None: + return + status_code: Final[object] = getattr(error, "status_code", None) + error_information: Final[StandardLoggingPayloadErrorInformation] = { + "error_code": "" if status_code is None else str(status_code), + "error_class": type(error).__name__, + "error_message": redact_string(str(error)), + "llm_provider": "bedrock", + } + metadata_key, metadata = spend_log_metadata + logging_obj.model_call_details["litellm_params"] = { # rebind-ok: the spend row reads the logging object + **litellm_params, + metadata_key: {**metadata, "status": "failure", "error_information": error_information}, + } + + def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: error: Final = _as_bedrock_error(failure) verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) if _is_session_committed(scope): scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error)) + if error is failure: + raise failure raise error from failure @@ -267,6 +317,25 @@ async def _reject_session_update( return True +async def _close_bedrock_input( + bedrock_stream: BedrockBidirectionalStream, + close_messages: Sequence[str], + send_to_bedrock: Callable[[str], Awaitable[None]], + timeout: float, +) -> None: + async def flush_and_close() -> None: + for close_message in close_messages: + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): + await bedrock_stream.input_stream.close() + + try: + await asyncio.wait_for(flush_and_close(), timeout) + except (asyncio.TimeoutError, TimeoutError): + verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout) + + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -278,6 +347,18 @@ class BedrockRealtime(BaseAWSLLM): super().__init__() self._sdk_version_lookup: Final = sdk_version_lookup self._close_timeout: Final = close_timeout + self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes + + def _close_failed_input_in_background( + self, + bedrock_stream: BedrockBidirectionalStream, + send_to_bedrock: Callable[[str], Awaitable[None]], + ) -> None: + close_task: Final = asyncio.create_task( + _close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout) + ) + self._failed_input_closes.add(close_task) + close_task.add_done_callback(self._failed_input_closes.discard) async def async_realtime( self, @@ -433,23 +514,26 @@ class BedrockRealtime(BaseAWSLLM): if _should_log_event(leftover_event) ), ) + if outcome.provider_failure is not None and outcome.client_disconnected: + verbose_proxy_logger.debug( + "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure + ) + provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure if logged_events: + if provider_failure is not None: + _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) + usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy() GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - logging_obj.dispatch_success_handlers( + usage_logging_obj.dispatch_success_handlers( list(logged_events), prefer_async_handlers=True, ) ) logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True - if outcome.provider_failure is None: + if provider_failure is None: return - if outcome.client_disconnected: - verbose_proxy_logger.debug( - "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure - ) - return - _raise_provider_failure(websocket.scope, outcome.provider_failure) + _raise_provider_failure(websocket.scope, provider_failure) async def _bridge( self, @@ -463,16 +547,34 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend + signals: Final = SessionEndSignals() async def collect_logged_events() -> None: - async for event in self._forward_bedrock_to_client( - bedrock_stream, websocket, transformation_config, model, logging_obj, session_state - ): - logged.append(event) + try: + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + signals=signals, + ): + logged.append(event) + except Exception: + signals.output_failed.set() + raise client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( - websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message + websocket, + bedrock_stream, + transformation_config, + model, + session_state, + logging_obj, + initial_message, + signals=signals, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -506,6 +608,8 @@ class BedrockRealtime(BaseAWSLLM): session_state: RealtimeResponseTransformInput, logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, + *, + signals: SessionEndSignals, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -521,7 +625,10 @@ class BedrockRealtime(BaseAWSLLM): async def send_to_bedrock(bedrock_message: str) -> None: event: Final = build_input_chunk(bedrock_message.encode("utf-8")) - await bedrock_stream.input_stream.send(event) + try: + await bedrock_stream.input_stream.send(event) + except Exception as e: + raise _stream_ended_before_session_closed() from e verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: @@ -553,20 +660,13 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break + signals.input_closing.set() finally: - - async def close_bedrock_input() -> None: - for close_message in transformation_config.session_close_messages(): - with contextlib.suppress(Exception): - await send_to_bedrock(close_message) - with contextlib.suppress(Exception): - await bedrock_stream.input_stream.close() - - try: - await asyncio.wait_for(close_bedrock_input(), self._close_timeout) - except (asyncio.TimeoutError, TimeoutError): - verbose_proxy_logger.debug( - "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout + if signals.output_failed.is_set(): + self._close_failed_input_in_background(bedrock_stream, send_to_bedrock) + else: + await _close_bedrock_input( + bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) async def _forward_bedrock_to_client( @@ -577,6 +677,8 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, + *, + signals: SessionEndSignals, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -597,6 +699,8 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: + if not signals.input_closing.is_set(): + raise _stream_ended_before_session_closed() verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") with contextlib.suppress(Exception): await client_ws.close() diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index b237116c5f1..d286d572a35 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -66,6 +66,13 @@ def get_cached_authenticator(auth_file: str) -> "Authenticator": return Authenticator(auth_file=auth_file) +def resolve_chatgpt_authenticator( + litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None", default: "Authenticator" +) -> "Authenticator": + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else default + + def _token_refresh_timeout() -> httpx.Timeout: configured: Final = get_configured_request_timeout() seconds: Final = ( diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index 9005185931f..e8f057b4d3d 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -6,7 +6,7 @@ from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams -from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file +from ..authenticator import Authenticator, get_chatgpt_auth_file, resolve_chatgpt_authenticator from ..common_utils import ( GetAccessTokenError, chatgpt_client_api_base, @@ -30,12 +30,6 @@ class ChatGPTConfig(OpenAIConfig): super().__init__() self.authenticator = authenticator if authenticator is not None else Authenticator() - def _resolve_authenticator( - self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None - ) -> Authenticator: - auth_file: Final = get_chatgpt_auth_file(litellm_params) - return get_cached_authenticator(auth_file) if auth_file else self.authenticator - @staticmethod def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str: try: @@ -63,7 +57,7 @@ class ChatGPTConfig(OpenAIConfig): return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider if api_key: return api_base or self.api_base_without_login(), api_key, custom_llm_provider - authenticator: Final = self._resolve_authenticator(litellm_params) + authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator) dynamic_api_base: Final = authenticator.get_api_base() dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider) return dynamic_api_base, dynamic_api_key, custom_llm_provider @@ -98,7 +92,7 @@ class ChatGPTConfig(OpenAIConfig): ), **strip_chatgpt_client_credential_headers(headers), ) - authenticator: Final = self._resolve_authenticator(litellm_params) + authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator) resolved_api_key: Final = ( self._get_access_token_or_raise(authenticator, model, "chatgpt") if get_chatgpt_auth_file(litellm_params) diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index cd2ea6b9ccc..88f2eecec0c 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -7,7 +7,7 @@ import hashlib import json import os import platform -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeAlias @@ -20,14 +20,10 @@ from pydantic import JsonValue, TypeAdapter from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.provider_affinity import get_caller_litellm_session_id, get_requested_session_id from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.types.utils import ProviderSpecificHeader if TYPE_CHECKING: from litellm.types.router import GenericLiteLLMParams -_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( - ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None -) _STRING_HEADERS: Final = TypeAdapter(Mapping[str, str]) _PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -357,8 +353,8 @@ def get_chatgpt_request_credential( normalized: Final = _normalize_litellm_params(params) scoped: Final = normalized.get("provider_specific_header") extra: Final = normalized.get("extra_headers") - headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( - provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt" + headers: Final = ProviderSpecificHeaderUtils.validated_provider_specific_headers( + provider_specific_header=scoped, custom_llm_provider="chatgpt" ) return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential( _STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({}) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 26a5a3f150b..3c7d0b8d673 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -28,7 +28,7 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file +from ..authenticator import Authenticator, get_chatgpt_auth_file, resolve_chatgpt_authenticator from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, @@ -64,10 +64,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.CHATGPT - def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator: - auth_file: Final = get_chatgpt_auth_file(litellm_params) - return get_cached_authenticator(auth_file) if auth_file else self.authenticator - def validate_environment( self, headers: dict[str, str], @@ -81,7 +77,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): **get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id), **strip_chatgpt_client_credential_headers(headers), } - authenticator: Final = self._resolve_authenticator(litellm_params) + authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator) safe_headers: Final = strip_chatgpt_client_credential_headers(headers) configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None if configured_api_key and not get_chatgpt_auth_file(litellm_params): diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 2f60d94c08a..34fde444880 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -82,7 +82,7 @@ async def async_embedding( ) try: - response: Final = await client.post(api_base, headers=headers, data=json.dumps(data)) + response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout) except httpx.HTTPStatusError as e: ## LOGGING logging_obj.post_call( @@ -166,7 +166,7 @@ def embedding( if client is None or not isinstance(client, HTTPHandler): client = HTTPHandler(concurrent_limit=1) - response: Final = client.post(embed_url, headers=headers, data=json.dumps(data)) + response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout) return CohereEmbeddingConfig().transform_response( response=response, diff --git a/litellm/llms/cohere/embed/v1_transformation.py b/litellm/llms/cohere/embed/v1_transformation.py index ad5bfa507e5..056b5bce742 100644 --- a/litellm/llms/cohere/embed/v1_transformation.py +++ b/litellm/llms/cohere/embed/v1_transformation.py @@ -9,6 +9,7 @@ import httpx from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.http_handler import header_value from litellm.types.llms.bedrock import ( CohereEmbeddingRequest, CohereEmbeddingRequestWithModel, @@ -23,6 +24,11 @@ class _SupportsEncode(Protocol): def encode(self, text: str, /) -> Sized: ... +def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None: + count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count") + return int(count) if count is not None and count.isdigit() else None + + class CohereEmbeddingConfig: """ Reference: https://docs.cohere.com/v2/reference/embed @@ -76,32 +82,30 @@ class CohereEmbeddingConfig: ) -> CohereEmbeddingRequestWithModel: return self._transform_request(model, input, inference_params) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - + def _calculate_usage( + self, input: list[str], encoding: _SupportsEncode, meta: dict, billed_input_tokens: int | None = None + ) -> Usage: text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - prompt_tokens_details: PromptTokensDetailsWrapper | None = None if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, + counted_tokens: Final = ( + billed_input_tokens + if billed_input_tokens is not None + else sum(len(encoding.encode(text)) for text in input) ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens + return Usage(prompt_tokens=counted_tokens, completion_tokens=0, total_tokens=counted_tokens) + input_tokens: Final = (image_tokens or 0) + (text_tokens or 0) return Usage( prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + image_tokens=image_tokens, + text_tokens=text_tokens, + ), ) def _transform_response( @@ -129,6 +133,7 @@ class CohereEmbeddingConfig: model=model, encoding=encoding, input=input, + billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None, ) def transform_response( @@ -151,6 +156,7 @@ class CohereEmbeddingConfig: model: str, encoding: _SupportsEncode, input: list, + billed_input_tokens: int | None = None, ) -> EmbeddingResponse: """ Parse a Cohere embed response body into an OpenAI-style EmbeddingResponse. @@ -195,7 +201,7 @@ class CohereEmbeddingConfig: setattr( model_response, "usage", - self._calculate_usage(input, encoding, response_json.get("meta", {})), + self._calculate_usage(input, encoding, response_json.get("meta", {}), billed_input_tokens), ) return model_response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 437944f8e03..4a10db31606 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -37,6 +37,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm._internal_context import internal_sub_call from litellm._logging import redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import ( aload_beta_headers_config, @@ -46,6 +47,12 @@ from litellm.anthropic_beta_headers_manager import ( from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs +from litellm.litellm_core_utils.agentic_loop_refusal import ( + every_claimed_tool_call, + finalize_refused_responses_turn, + is_refused_item, + refused_tool_call_identifiers, +) from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -55,11 +62,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import ( - as_str_mapping, - qualify_agentic_followup_model, - set_provider_response_headers_in_hidden_params, +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, + run_chat_completion_agentic_hooks, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -149,6 +156,7 @@ from litellm.types.extraction import ExtractionResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + AgenticLoopLimitError, AgenticLoopPlan, AgenticLoopRequestPatch, AgenticLoopSafetyError, @@ -6004,7 +6012,7 @@ class BaseLLMHTTPHandler: @staticmethod def _check_agentic_loop_safety( tool_calls: object, - fingerprints: list[str], + fingerprints: Sequence[str], depth: int, max_loops: int, model: str, @@ -6023,7 +6031,7 @@ class BaseLLMHTTPHandler: if fingerprint in fingerprints: raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @staticmethod @@ -6033,26 +6041,6 @@ class BaseLLMHTTPHandler: except Exception: return str(tools) - @staticmethod - def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: - """ - Collect the ids and names of the tool calls a safety rail just refused. - - Callbacks hand back either a bare list of tool calls or a dict wrapping - that list under ``tool_calls``, and both the anthropic and responses - shapes carry an ``id`` (or ``call_id``) plus a ``name``. - """ - calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls - if not isinstance(calls, list): - return frozenset(), frozenset() - dict_calls: Final = (call for call in calls if isinstance(call, dict)) - fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls) - ids: Final = frozenset( - value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str) - ) - names: Final = frozenset(name for _, _, name in fields if isinstance(name, str)) - return ids, names - @staticmethod def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: """ @@ -6063,22 +6051,13 @@ class BaseLLMHTTPHandler: share a name with a refused one. The name is only consulted for tool call shapes that arrive without an id. """ - if not isinstance(block, dict) or block.get("type") != "tool_use": - return False - block_id: Final = block.get("id") - if isinstance(block_id, str) and refused_ids: - return block_id in refused_ids - return block.get("name") in refused_names + return is_refused_item(block, "tool_use", "id", refused_ids, refused_names) @staticmethod def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool: """ Whether a refused rerun can still be answered with a finalized turn. - Only the anthropic messages surface can. The responses surface carries a - pydantic model the finalizer does not rewrite, so it keeps raising, which - is what every surface did before this path learned to end the turn. - The messages and responses call sites pass ``stream=False``, because interception converts an intercepted stream to non-streaming before the loop runs and rebuilds the SSE stream from the finalized turn @@ -6087,7 +6066,7 @@ class BaseLLMHTTPHandler: finalized turn would reach the client as a second message rather than as a replacement. """ - return not stream and api_surface == "anthropic_messages" + return not stream and api_surface in ("anthropic_messages", "responses") @staticmethod def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object: @@ -6104,10 +6083,12 @@ class BaseLLMHTTPHandler: one is still in the response the stop reason stays ``tool_use`` so the client knows to answer it. """ + if isinstance(response, ResponsesAPIResponse): + return finalize_refused_responses_turn(response, tool_calls) if not isinstance(response, dict): return response - refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls) + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) finalized: Final = dict(response) content: Final = finalized.get("content") if isinstance(content, list): @@ -6134,7 +6115,7 @@ class BaseLLMHTTPHandler: kwargs: dict, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, stream: bool = False, callback: Optional["CustomLogger"] = None, @@ -6175,7 +6156,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup.update(patch.kwargs) kwargs_for_followup["_agentic_loop_depth"] = depth + 1 kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] + kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint] response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( **{ @@ -6214,7 +6195,7 @@ class BaseLLMHTTPHandler: kwargs: dict, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, callback: Optional["CustomLogger"] = None, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: @@ -6252,12 +6233,13 @@ class BaseLLMHTTPHandler: ) try: - response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( - model=patch.model or model, - input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime - **optional_params, - **kwargs_for_followup, - ) + with internal_sub_call(logging_obj.litellm_call_id): + response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( + model=patch.model or model, + input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime + **optional_params, + **kwargs_for_followup, + ) if callback is not None: try: @@ -6334,64 +6316,6 @@ class BaseLLMHTTPHandler: custom_llm_provider=custom_llm_provider, ) - async def _execute_chat_completion_agentic_plan( - self, - plan: AgenticLoopPlan, - model: str, - messages: list[dict], - optional_params: dict, - kwargs: dict, - custom_llm_provider: str, - depth: int, - max_loops: int, - fingerprints: list[str], - fingerprint: str, - ) -> ModelResponse | CustomStreamWrapper: - patch: Final = plan.request_patch or AgenticLoopRequestPatch() - if patch.messages is None: - raise ValueError("Agentic loop plan missing patched messages") - - full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) - - optional_params_for_followup: Final = dict(optional_params) - optional_params_for_followup.update(patch.optional_params) - if patch.tools is not None: - optional_params_for_followup["tools"] = patch.tools - - internal_params: Final = { - "_websearch_interception", - "acompletion", - "litellm_logging_obj", - "custom_llm_provider", - "model_alias_map", - "stream_response", - "custom_prompt_dict", - } - kwargs_for_followup: Final = build_agentic_followup_kwargs( - request_kwargs=MappingProxyType( - { - k: v - for k, v in kwargs.items() - if not k.startswith("_websearch_interception") - and not k.startswith("_compression_interception") - and k not in internal_params - } - ), - patch_kwargs=patch.kwargs, - request_params=frozenset((*optional_params_for_followup, "model", "messages")), - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) - - return await litellm.acompletion( - model=full_model_name, - messages=patch.messages, - **optional_params_for_followup, - **kwargs_for_followup, - ) - def _maybe_wrap_in_fake_stream( self, response: _ResponseT, @@ -6508,26 +6432,38 @@ class BaseLLMHTTPHandler: except AgenticLoopSafetyError as e: if not self._can_replace_turn_with_terminal_response(stream, api_surface): raise - verbose_logger.warning( - "LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s", - logging_obj.litellm_call_id, - model, - str(e), - ) + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) return self._maybe_wrap_in_fake_stream( - self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), + self._finalize_refused_agentic_response( + response=response, + tool_calls=await every_claimed_tool_call( + tool_calls, + ( + other.async_should_run_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=hook_kwargs, + ) + for other in callbacks + if isinstance(other, CustomLogger) and other is not callback + ), + ), + ), logging_obj, api_surface, ) - try: - kwargs_with_provider = hook_kwargs.copy() - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - build_plan_overridden = ( - callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan - ) - if not build_plan_overridden: - agentic_result: object = await callback.async_run_agentic_loop( + kwargs_with_provider = {**hook_kwargs, "custom_llm_provider": custom_llm_provider} + build_plan_overridden = ( + callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan + ) + if not build_plan_overridden: + agentic_result: object = await bill_first_round_on_failure( + callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -6537,10 +6473,15 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, stream=stream, kwargs=kwargs_with_provider, - ) - return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) + ), + logging_obj=logging_obj, + response=response, + stream=stream, + ) + return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) - plan = await callback.async_build_agentic_loop_plan( + plan = await bill_first_round_on_failure( + callback.async_build_agentic_loop_plan( tools=tool_calls, model=model, messages=messages, @@ -6550,30 +6491,35 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, stream=stream, kwargs=kwargs_with_provider, + ), + logging_obj=logging_obj, + response=response, + stream=stream, + ) + + if plan.response_override is not None: + return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface) + if plan.terminate: + verbose_logger.debug( + "Agentic loop terminated by callback=%s reason=%s", + callback.__class__.__name__, + plan.stop_reason, ) + return self._maybe_wrap_in_fake_stream( + await callback.async_post_agentic_loop_response_hook( + response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), + plan=plan, + kwargs=kwargs_with_provider, + ), + logging_obj, + api_surface, + ) + if not plan.run_agentic_loop: + continue - if plan.response_override is not None: - return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface) - if plan.terminate: - verbose_logger.debug( - "Agentic loop terminated by callback=%s reason=%s", - callback.__class__.__name__, - plan.stop_reason, - ) - return self._maybe_wrap_in_fake_stream( - await callback.async_post_agentic_loop_response_hook( - response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), - plan=plan, - kwargs=kwargs_with_provider, - ), - logging_obj, - api_surface, - ) - if not plan.run_agentic_loop: - continue - - if api_surface == "responses": - return await self._execute_responses_agentic_plan( + if api_surface == "responses": + return await bill_first_round_on_failure( + self._execute_responses_agentic_plan( plan=plan, model=model, response_api_optional_request_params=anthropic_messages_optional_request_params, @@ -6584,10 +6530,15 @@ class BaseLLMHTTPHandler: fingerprints=fingerprints, fingerprint=fingerprint, callback=callback, - ) + ), + logging_obj=logging_obj, + response=response, + stream=stream, + ) - return self._maybe_wrap_in_fake_stream( - await self._execute_anthropic_agentic_plan( + return self._maybe_wrap_in_fake_stream( + await bill_first_round_on_failure( + self._execute_anthropic_agentic_plan( plan=plan, model=model, messages=messages, @@ -6601,17 +6552,13 @@ class BaseLLMHTTPHandler: stream=stream, callback=callback, ), - logging_obj, - api_surface, - ) - except Exception as e: - _call_id = getattr(logging_obj, "litellm_call_id", "unknown") - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in agentic completion hooks [call_id=%s model=%s]: %s", - _call_id, - model, - str(e), - ) + logging_obj=logging_obj, + response=response, + stream=stream, + ), + logging_obj, + api_surface, + ) # Check if we need to convert response to fake stream # This happens when: @@ -6643,110 +6590,23 @@ class BaseLLMHTTPHandler: Returns the response from agentic loop, or None if no hook runs. """ - from litellm._logging import verbose_logger - from litellm.integrations.custom_logger import CustomLogger - - callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or []) - tools: Final = optional_params.get("tools", []) - depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs) - - for callback in callbacks: - if not isinstance(callback, CustomLogger): - continue - if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"): - continue - - should_run: bool = False - tool_calls: object = None - try: - ( - should_run, - tool_calls, - ) = await callback.async_should_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - tools=tools, - stream=stream, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s", - str(e), - ) - continue - - if not should_run: - continue - - # Safety guards must run OUTSIDE the callback try/except — they are - # bounded-loop / cycle-break rails that must propagate to the caller. - fingerprint = self._check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, + agentic_response: Final = await bill_first_round_on_failure( + run_chat_completion_agentic_hooks( + response=response, model=model, - ) - - try: - kwargs_with_provider = kwargs.copy() if kwargs else {} - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - build_plan_overridden = ( - callback.__class__.async_build_chat_completion_agentic_loop_plan - is not CustomLogger.async_build_chat_completion_agentic_loop_plan - ) - if not build_plan_overridden: - return await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - - plan = await callback.async_build_chat_completion_agentic_loop_plan( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - - if plan.response_override is not None: - return plan.response_override - if plan.terminate: - verbose_logger.debug( - "Agentic chat loop terminated by callback=%s reason=%s", - callback.__class__.__name__, - plan.stop_reason, - ) - return response - if not plan.run_agentic_loop: - continue - - return await self._execute_chat_completion_agentic_plan( - plan=plan, - model=model, - messages=messages, - optional_params=optional_params, - kwargs=kwargs_with_provider, - custom_llm_provider=custom_llm_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) - except Exception as e: - verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e) + messages=messages, + optional_params=optional_params, + kwargs=kwargs or {}, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + stream=stream, + ), + logging_obj=logging_obj, + response=response, + stream=stream, + ) + if agentic_response is not None: + return agentic_response # Check if we need to convert response to fake stream for chat completions # This happens when: @@ -7091,7 +6951,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7124,7 +6984,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7143,9 +7003,8 @@ class BaseLLMHTTPHandler: api_version=api_version, ) - async def _async_realtime_session_post( + async def async_realtime_translation_client_secret_handler( self, - endpoint: Literal["client_secrets", "transcription_sessions"], api_base: str, api_key: str, request_data: dict[str, object], @@ -7153,7 +7012,35 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, + api_version: str | None = None, + ) -> httpx.Response: + return await self._async_realtime_session_post( + endpoint="translation_client_secrets", + api_base=api_base, + api_key=api_key, + request_data=request_data, + logging_obj=logging_obj, + timeout=timeout, + provider_config=provider_config, + model=model, + extra_headers=extra_headers, + client=client, + api_version=api_version, + ) + + async def _async_realtime_session_post( + self, + endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"], + api_base: str, + api_key: str, + request_data: dict[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + provider_config: BaseRealtimeHTTPConfig | None = None, + model: str | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7176,13 +7063,20 @@ class BaseLLMHTTPHandler: url = provider_config.get_transcription_session_url( api_base=api_base, model=model or "", api_version=api_version ) + elif endpoint == "translation_client_secrets": + url = provider_config.get_translation_client_secret_url( + api_base=api_base, model=model or "", api_version=api_version + ) else: url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) else: - url = join_api_base(api_base, f"v1/realtime/{endpoint}") + endpoint_path: Final = ( + "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint + ) + url = join_api_base(api_base, f"v1/realtime/{endpoint_path}") headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -7216,6 +7110,12 @@ class BaseLLMHTTPHandler: ) raise + @staticmethod + def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: + if isinstance(client, AsyncHTTPHandler): + return client + return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI) + async def async_realtime_calls_handler( self, api_base: str, @@ -7225,10 +7125,11 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - session_config: dict[str, object] | None = None, - extra_headers: dict[str, object] | None = None, + session_config: Mapping[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, + translation: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -7240,18 +7141,30 @@ class BaseLLMHTTPHandler: - sdp: the SDP offer (text) - session: JSON string with {"type": "realtime", "model": "...", ...} """ - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders.OPENAI, - ) - else: - async_httpx_client = client + session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally + **(session_config or {}) + } + if "type" not in session_data: + session_data["type"] = "translation" if translation else "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + url = ( + provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version) + if translation + else provider_config.get_realtime_calls_url( + api_base=api_base, model=model or "", api_version=api_version + ) + ) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = join_api_base(api_base, "v1/realtime/calls") + path: Final = "translations/calls" if translation else "calls" + url = join_api_base(api_base, f"v1/realtime/{path}") headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } @@ -7259,14 +7172,8 @@ class BaseLLMHTTPHandler: if extra_headers: headers.update(extra_headers) - # Build multipart form data: sdp + session JSON - session_data: Final = session_config or {} - if "type" not in session_data: - session_data["type"] = "realtime" - if "model" not in session_data and model: - session_data["model"] = model - - sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + if translation: + headers["Content-Type"] = "application/sdp" files: Final = { "sdp": (None, sdp_text, "text/plain"), @@ -7284,12 +7191,14 @@ class BaseLLMHTTPHandler: ) try: - return await async_httpx_client.post( - url=url, - headers=headers, - files=files, - timeout=timeout, - ) + if translation: + return await async_httpx_client.post( + url=url, + headers=headers, + content=sdp_text, + timeout=timeout, + ) + return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout) except Exception as e: if provider_config is not None: raise self._handle_error( @@ -8099,7 +8008,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8295,7 +8204,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8476,7 +8385,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8640,7 +8549,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8781,7 +8690,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8954,7 +8863,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9141,7 +9050,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9274,7 +9183,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9356,7 +9265,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9541,7 +9450,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index b5393fa1603..859211bd40d 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict ) +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.databricks import ( AllDatabricksContentValues, @@ -395,7 +396,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls, *, model: str | None = None): + def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None): return super().get_config() def get_json_schema_from_pydantic_object( @@ -541,12 +542,12 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): model: str, value: dict | None, optional_params: dict, - is_thinking_enabled: bool, + tool_is_forced: bool, ) -> DatabricksTool | None: if value is None: return None - tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled) + tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, tool_is_forced) databricks_tool: Final = self.convert_anthropic_tool_to_databricks_tool(tool) return databricks_tool @@ -579,7 +580,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): if "claude" in model else non_default_params ) - is_thinking_enabled: Final = self.is_thinking_enabled(request_params) mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params) if "claude" in model: AnthropicConfig.translate_legacy_thinking_for_adaptive_model( @@ -596,17 +596,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): mapped_params.pop("max_completion_tokens", None) if "response_format" in request_params and "claude" in model: + tool_is_forced: Final = AnthropicConfig.response_format_tool_can_be_forced( + model, "databricks", REQUEST_PARAMS.validate_python(request_params) + ) _tool: Final = self.map_response_format_to_databricks_tool( model, request_params["response_format"], mapped_params, - is_thinking_enabled, + tool_is_forced, ) if _tool is not None: self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) optional_params["json_mode"] = True - if not is_thinking_enabled: + if tool_is_forced: _tool_choice: Final = ChatCompletionToolChoiceObjectParam( type="function", function=ChatCompletionToolChoiceFunctionParam(name=RESPONSE_FORMAT_TOOL_NAME), diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index 90b63d860d3..85f44181bc3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -192,7 +192,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) headers=headers, ) self._normalize_system_role_messages( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict - anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + anthropic_messages_request, model=model, drop_params=litellm_params.drop_params ) self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict return anthropic_messages_request diff --git a/litellm/llms/empiriolabs/decisions/transformation.py b/litellm/llms/empiriolabs/decisions/transformation.py new file mode 100644 index 00000000000..ce3464310c4 --- /dev/null +++ b/litellm/llms/empiriolabs/decisions/transformation.py @@ -0,0 +1,10 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class EmpirioLabsDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("EMPIRIOLABS_API_KEY",) + api_base_env = ("EMPIRIOLABS_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.empiriolabs.ai" diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 3134dfa1e4b..a9f31d1abff 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -218,7 +218,7 @@ def _system_instruction_text(system_instruction: HttpxContentType | None) -> str class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping - def __init__(self, drop_params: bool = False): + def __init__(self, drop_params: bool = False) -> None: super().__init__() self.drop_params = drop_params self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict() diff --git a/litellm/llms/github_copilot/github_host.py b/litellm/llms/github_copilot/github_host.py index 4fc921d490d..cfe122ebac4 100644 --- a/litellm/llms/github_copilot/github_host.py +++ b/litellm/llms/github_copilot/github_host.py @@ -21,8 +21,10 @@ class GithubHostEndpoints: device_code_url: str access_token_url: str api_key_url: str + copilot_user_url: str user_url: str client_id: str + exchanges_copilot_token: bool copilot_api_hosts: frozenset[str] copilot_api_host_suffix: str | None default_copilot_api_base: str @@ -49,14 +51,17 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints: if normalized is None: raise ValueError("host must be github.com or a single-subdomain GHE.com hostname") client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID) + exchanges_copilot_token: Final = client_id == DEFAULT_GITHUB_CLIENT_ID if normalized == GITHUB_COPILOT_DEFAULT_HOST: return GithubHostEndpoints( host=normalized, device_code_url=os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL), access_token_url=os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL), api_key_url=os.getenv("GITHUB_COPILOT_API_KEY_URL", DEFAULT_GITHUB_API_KEY_URL), + copilot_user_url="https://api.github.com/copilot_internal/user", user_url="https://api.github.com/user", client_id=client_id, + exchanges_copilot_token=exchanges_copilot_token, copilot_api_hosts=frozenset({"githubcopilot.com"}), copilot_api_host_suffix=".githubcopilot.com", default_copilot_api_base="https://api.githubcopilot.com", @@ -70,8 +75,10 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints: device_code_url=f"https://{normalized}/login/device/code", access_token_url=f"https://{normalized}/login/oauth/access_token", api_key_url=f"https://{api_host}/copilot_internal/v2/token", + copilot_user_url=f"https://{api_host}/copilot_internal/user", user_url=f"https://{api_host}/user", client_id=client_id, + exchanges_copilot_token=exchanges_copilot_token, copilot_api_hosts=frozenset({copilot_api_host}), copilot_api_host_suffix=None, default_copilot_api_base=f"https://{copilot_api_host}", diff --git a/litellm/llms/github_copilot/per_user_auth.py b/litellm/llms/github_copilot/per_user_auth.py index 75a10f2e671..e210414ce8f 100644 --- a/litellm/llms/github_copilot/per_user_auth.py +++ b/litellm/llms/github_copilot/per_user_auth.py @@ -46,6 +46,7 @@ _LLM_PROVIDER: Final = "github_copilot" _DEVICE_FLOW_SCOPE: Final = "read:user" _SESSION_CACHE_MAX_SIZE: Final = 1000 _SYNC_LOCK_COUNT: Final = 64 +_ACCOUNT_LOOKUP_TTL_SECONDS: Final = 1800 class _SessionCache(Protocol): @@ -125,12 +126,20 @@ class _CopilotEndpointsPayload(TypedDict): api: ReadOnly[NotRequired[object]] +@with_config(ConfigDict(extra="allow", strict=True)) +class _CopilotAccountPayload(TypedDict): + access_type_sku: ReadOnly[NotRequired[object]] + chat_enabled: ReadOnly[NotRequired[object]] + endpoints: ReadOnly[NotRequired[object]] + + @with_config(ConfigDict(extra="allow", strict=True)) class _GithubUserPayload(TypedDict): login: ReadOnly[str] _COPILOT_TOKEN: Final = TypeAdapter(_CopilotTokenPayload) +_COPILOT_ACCOUNT: Final = TypeAdapter(_CopilotAccountPayload) _DEVICE_CODE: Final = TypeAdapter(_DeviceCodePayload) _ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPollPayload) _GITHUB_USER: Final = TypeAdapter(_GithubUserPayload) @@ -273,6 +282,8 @@ def _session_from_response( llm_provider=_LLM_PROVIDER, model="", ) + if not endpoints.exchanges_copilot_token: + return _session_from_account(response, github_token, credential_name, cache_key, endpoints) try: payload: Final = _COPILOT_TOKEN.validate_python(response.json()) except Exception as e: @@ -282,13 +293,7 @@ def _session_from_response( model="", ) from e - payload_endpoints: Final[object] = payload.get("endpoints") - endpoints_api: Final[object] = ( - _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api") - if isinstance(payload_endpoints, Mapping) - else None - ) - api_base: Final = validated_copilot_api_base(endpoints_api, endpoints) + api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints) session: Final = GithubCopilotUserSession(token=payload["token"], api_base=api_base) expires_at: Final = _expires_at_seconds(payload.get("expires_at")) if expires_at > 0: @@ -296,6 +301,41 @@ def _session_from_response( return session +def _session_from_account( + response: httpx.Response, + github_token: str, + credential_name: str, + cache_key: str, + endpoints: GithubHostEndpoints, +) -> GithubCopilotUserSession: + try: + payload: Final = _COPILOT_ACCOUNT.validate_python(response.json()) + except Exception as e: + raise ServiceUnavailableError( + message="GitHub Copilot account lookup returned an invalid response", + llm_provider=_LLM_PROVIDER, + model="", + ) from e + if payload.get("chat_enabled") is False or payload.get("access_type_sku") == "no_access": + _SESSION_CACHE.delete_cache(cache_key) + raise _reject_connection(credential_name) + + api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints) + session: Final = GithubCopilotUserSession(token=github_token, api_base=api_base) + _cache_session(cache_key, github_token, session, time.time() + _ACCOUNT_LOOKUP_TTL_SECONDS) + return session + + +def _endpoints_api(payload_endpoints: object) -> object: + if not isinstance(payload_endpoints, Mapping): + return None + return _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api") + + +def _session_lookup_url(endpoints: GithubHostEndpoints) -> str: + return endpoints.api_key_url if endpoints.exchanges_copilot_token else endpoints.copilot_user_url + + def _to_int(raw: object) -> int: if isinstance(raw, bool): return 0 @@ -337,7 +377,7 @@ def _do_exchange( _SyncGetClient, litellm.module_level_client ) response: Final = client.get( - endpoints.api_key_url, + _session_lookup_url(endpoints), headers=github_api_headers(github_token), ) except Exception as e: @@ -367,7 +407,7 @@ async def _do_aexchange( client: Final = _new_async_github_client() try: response: Final = await client.get( - endpoints.api_key_url, + _session_lookup_url(endpoints), headers=github_api_headers(github_token), ) except Exception as e: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index a1d1a3f6a19..5d997babc6e 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -38,7 +38,9 @@ class _SupportsTokenEncode(Protocol): def encode(self, text: str) -> list[int]: ... -def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: +def get_hf_task_embedding_for_model( + model: str, task_type: str | None, api_base: str, timeout: float | httpx.Timeout | None = None +) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -46,7 +48,7 @@ def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: raise Exception(f"Invalid task_type={task_type}. Expected one of={hf_tasks_embeddings}") http_client: Final = HTTPHandler(concurrent_limit=1) - model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}") + model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}", timeout=timeout) model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) @@ -148,6 +150,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type: Literal["sync", "async"], optional_params: dict, embed_url: str, + timeout: float | httpx.Timeout | None = None, ) -> dict: data: dict = {} @@ -165,7 +168,9 @@ class HuggingFaceEmbedding(BaseLLM): task_type: Final = optional_params.pop("input_type", None) if call_type == "sync": - hf_task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL) + hf_task: Final = get_hf_task_embedding_for_model( + model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout + ) elif call_type == "async": return self._async_transform_input(model=model, task_type=task_type, embed_url=embed_url, input=input) @@ -253,6 +258,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type="sync", optional_params=optional_params, embed_url=api_base, + timeout=timeout, ) ## LOGGING @@ -269,9 +275,10 @@ class HuggingFaceEmbedding(BaseLLM): if client is None: client = get_async_httpx_client( llm_provider=litellm.LlmProviders.HUGGINGFACE, + params={"timeout": timeout}, ) - response: Final = await client.post(api_base, headers=headers, data=json.dumps(data)) + response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout) ## LOGGING logging_obj.post_call( @@ -321,7 +328,9 @@ class HuggingFaceEmbedding(BaseLLM): litellm_params=litellm_params, ) task_type: Final = optional_params.get("input_type", None) - task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL) + task: Final = get_hf_task_embedding_for_model( + model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout + ) # print_verbose(f"{model}, {task}") embed_url = "" if model.startswith(("http://", "https://")): @@ -359,6 +368,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type="sync", optional_params=optional_params, embed_url=embed_url, + timeout=timeout, ) ## LOGGING @@ -374,7 +384,7 @@ class HuggingFaceEmbedding(BaseLLM): ## COMPLETION CALL if client is None or not isinstance(client, HTTPHandler): client = HTTPHandler(concurrent_limit=1) - response: Final = client.post(embed_url, headers=headers, data=json.dumps(data)) + response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout) ## LOGGING logging_obj.post_call( diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index cafeece0a4e..d2ef33b89dd 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -13,24 +13,26 @@ The handler is intentionally a thin orchestration layer: 2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs from the structured dict produced by :class:`NvidiaRivaAudioTranscriptionConfig`. -3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS) - vs self-hosted (any host:port, optional TLS) modes. -4. Stream the audio through Riva's ``streaming_response_generator`` and - aggregate ``is_final`` results into a single transcript, or, when - ``riva_offline`` is set, send the whole buffer through the unary - ``offline_recognize`` RPC for NIMs that only ship offline engines. +3. Open a gRPC channel honoring NVCF (TLS by default) vs self-hosted (any + host:port, optional TLS) modes, and attach the ``authorization`` and NVCF + ``function-id`` metadata to each call exactly once. +4. Stream the audio through Riva's ``StreamingRecognize`` RPC and aggregate + ``is_final`` results into a single transcript, or, when ``riva_offline`` + is set, send the whole buffer through the unary ``Recognize`` RPC for NIMs + that only ship offline engines. 5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed on ``_hidden_params`` so cost calculation works. -``riva-client`` is imported lazily so ``litellm`` core remains usable -without the optional STT extras installed. +``grpcio`` and ``protobuf`` are imported lazily so ``litellm`` core remains +usable without the optional STT extras installed. """ +from __future__ import annotations + import asyncio -import inspect -from collections.abc import Callable, Iterable, Mapping, Sequence -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol +from collections.abc import Iterable, Sequence +from contextlib import closing +from typing import TYPE_CHECKING, Final import httpx @@ -64,89 +66,19 @@ if TYPE_CHECKING: Logging as LiteLLMLoggingObj, ) -# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches -# NVIDIA's recommended chunk size for streaming ASR — small enough for -# responsive endpointing, large enough to keep per-RPC overhead low. -_DEFAULT_CHUNK_SAMPLES: Final = 1600 -_DEFAULT_CHUNK_BYTES: Final = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample - - _RIVA_INSTALL_HINT = "NVIDIA Riva client is not installed. Install with `pip install 'litellm[stt-nvidia-riva]'`." -class _RivaAuth(Protocol): - """Opaque ``riva.client.Auth`` handle.""" - - -class _RecognizeFuture(Protocol): - def result(self, timeout: float | None = None) -> object: ... - - def done(self) -> bool: ... - - def cancel(self) -> bool: ... - - -class _AsrService(Protocol): - @property - def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ... - - def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ... - - -class _EndpointingConfig(Protocol): - """Opaque ``EndpointingConfig`` protobuf message.""" - - -class _EndpointingConfigField(Protocol): - CopyFrom: Callable[[_EndpointingConfig], None] - - -class _RecognitionConfig(Protocol): - @property - def endpointing_config(self) -> _EndpointingConfigField: ... - - -class _StreamingRecognitionConfig(Protocol): - """Opaque ``StreamingRecognitionConfig`` protobuf message.""" - - -class _AudioEncoding(Protocol): - @property - def LINEAR_PCM(self) -> object: ... - - -class _RivaClientModule(Protocol): - """The ``riva.client`` entry points this handler calls.""" - - @property - def Auth(self) -> Callable[..., _RivaAuth]: ... - - @property - def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ... - - -class _RivaAsrModule(Protocol): - """The protobuf constructors this handler calls, from whichever module exposes them.""" - - @property - def AudioEncoding(self) -> _AudioEncoding: ... - - @property - def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ... - - @property - def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ... - - @property - def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ... - - -def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]: - return riva_module.Auth - - -def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding: - return riva_asr_module.AudioEncoding +def _use_ssl(optional_params: dict) -> bool: + """ + NVCF endpoints are TLS-only, so ``nvcf_function_id`` defaults ``use_ssl`` + to True. Self-hosted Riva defaults to plaintext but honors an explicit + ``use_ssl``, since TLS-terminating ingresses in front of it are common. + """ + use_ssl_override: Final = optional_params.get("use_ssl") + if use_ssl_override is not None: + return bool(use_ssl_override) + return bool(optional_params.get("nvcf_function_id")) def _parse_riva_offline(value: object, drop_params: bool) -> bool: @@ -192,7 +124,7 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, atranscription: bool = False, @@ -237,13 +169,11 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, provider_config: NvidiaRivaAudioTranscriptionConfig | None = None, ) -> TranscriptionResponse: - # ``riva-client`` exposes a sync streaming generator, so we offload - # the blocking call to a worker thread to keep the event loop free. return await asyncio.to_thread( self._run_sync, model=model, @@ -267,7 +197,7 @@ class NvidiaRivaAudioTranscription: litellm_params: dict, model_response: TranscriptionResponse, timeout: float | str | httpx.Timeout | None, - logging_obj: "LiteLLMLoggingObj", + logging_obj: LiteLLMLoggingObj, api_key: str | None, api_base: str | None, provider_config: NvidiaRivaAudioTranscriptionConfig, @@ -317,18 +247,12 @@ class NvidiaRivaAudioTranscription: ) or litellm.drop_params is True riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params) - riva_module, riva_asr_module = _import_riva() - auth_obj: Final = self._construct_auth( - riva_module=riva_module, - api_base=api_base, - api_key=api_key, - optional_params=optional_params, - ) + try: + from litellm.llms.nvidia_riva.audio_transcription import riva_grpc + except ImportError as e: + raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - recognition_config: Final = self._build_recognition_config_proto( - riva_asr_module=riva_asr_module, - recognition_config_dict=recognition_config_dict, - ) + recognition_config: Final = riva_grpc.recognition_config(recognition_config_dict) logging_obj.pre_call( input=None, api_key=api_key, @@ -344,26 +268,18 @@ class NvidiaRivaAudioTranscription: }, ) + metadata: Final = riva_grpc.call_metadata(api_key, optional_params.get("nvcf_function_id")) try: - asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj) - responses: Final = ( - self._offline_responses( - asr_service=asr_service, - pcm_bytes=resampled.pcm_bytes, - recognition_config=recognition_config, - timeout=None if timeout is None else _deadline_seconds(timeout, drop_params), + deadline: Final = None if timeout is None else _deadline_seconds(timeout, drop_params) + with closing(riva_grpc.open_channel(api_base, _use_ssl(optional_params))) as channel: + responses: Final = ( + riva_grpc.recognize(channel, recognition_config, resampled.pcm_bytes, metadata, deadline) + if riva_offline + else riva_grpc.streaming_recognize( + channel, recognition_config, resampled.pcm_bytes, metadata, deadline + ) ) - if riva_offline - else self._streaming_responses( - asr_service=asr_service, - riva_asr_module=riva_asr_module, - pcm_bytes=resampled.pcm_bytes, - recognition_config=recognition_config, - timeout=timeout, - drop_params=drop_params, - ) - ) - final_results: Final = self._collect_final_results(responses, require_final=not riva_offline) + final_results: Final = self._collect_final_results(responses, require_final=not riva_offline) except NvidiaRivaException: raise except Exception as e: @@ -400,142 +316,6 @@ class NvidiaRivaAudioTranscription: return final_response - def _construct_auth( - self, - riva_module: _RivaClientModule, - api_base: str, - api_key: str | None, - optional_params: dict, - ) -> _RivaAuth: - """ - Build a ``riva.client.Auth`` object. - - - When ``nvcf_function_id`` is provided we attach the NVCF - ``function-id`` and bearer ``authorization`` metadata, and default - ``use_ssl`` to True (NVCF endpoints are TLS-only). - - Otherwise (self-hosted) we default ``use_ssl`` to False but still - honor an explicit override — self-hosted Riva behind an ingress - with TLS termination is a real deployment topology. - """ - nvcf_function_id: Final = optional_params.get("nvcf_function_id") - use_ssl_override: Final = optional_params.get("use_ssl") - use_ssl: Final = bool(use_ssl_override) if use_ssl_override is not None else bool(nvcf_function_id) - - metadata: Final[list[tuple[str, str]]] = [] - if nvcf_function_id: - metadata.append(("function-id", str(nvcf_function_id))) - if api_key: - metadata.append(("authorization", f"Bearer {api_key}")) - - try: - return _auth_factory(riva_module)(uri=api_base, use_ssl=use_ssl, metadata_args=metadata) - except TypeError: - # Older riva-client signatures used positional-only args. - return _auth_factory(riva_module)(None, use_ssl, api_base, metadata) - - def _build_recognition_config_proto( - self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any] - ) -> _RecognitionConfig: - encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper() - encoding_enum: Final[object] = getattr( - _audio_encoding(riva_asr_module), - encoding_name, - _audio_encoding(riva_asr_module).LINEAR_PCM, - ) - - config: Final[_RecognitionConfig] = riva_asr_module.RecognitionConfig( - encoding=encoding_enum, - sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]), - language_code=recognition_config_dict["language_code"], - audio_channel_count=int(recognition_config_dict["audio_channel_count"]), - enable_automatic_punctuation=bool(recognition_config_dict.get("enable_automatic_punctuation", True)), - enable_word_time_offsets=bool(recognition_config_dict.get("enable_word_time_offsets", False)), - max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)), - model=recognition_config_dict.get("model", "") or "", - verbatim_transcripts=bool(recognition_config_dict.get("verbatim_transcripts", False)), - profanity_filter=bool(recognition_config_dict.get("profanity_filter", False)), - ) - - endpointing: Final = recognition_config_dict.get("endpointing_config") - if isinstance(endpointing, dict) and endpointing: - try: - ep: Final[_EndpointingConfig] = riva_asr_module.EndpointingConfig(**endpointing) - config.endpointing_config.CopyFrom(ep) - except Exception: - # If the user supplied an unknown EndpointingConfig field - # (older Riva server), fall back to Riva's defaults rather - # than failing the whole request. - pass - - return config - - def _streaming_responses( - self, - asr_service: _AsrService, - riva_asr_module: _RivaAsrModule, - pcm_bytes: bytes, - recognition_config: _RecognitionConfig, - timeout: float | str | httpx.Timeout | None, - drop_params: bool, - ) -> Iterable[object]: - streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig( - config=recognition_config, interim_results=False - ) - return asr_service.streaming_response_generator( - audio_chunks=self._iter_audio_chunks(pcm_bytes), - streaming_config=streaming_config, - **self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params), - ) - - def _offline_responses( - self, - asr_service: _AsrService, - pcm_bytes: bytes, - recognition_config: _RecognitionConfig, - timeout: float | None, - ) -> Iterable[object]: - pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True) - try: - return (pending.result(timeout=timeout),) - except Exception as e: - if pending.done(): - raise - pending.cancel() - raise NvidiaRivaException( - status_code=408, - message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds", - ) from e - - @classmethod - def _timeout_kwargs( - cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool - ) -> Mapping[str, float]: - # Forward the deadline so the call cannot block forever if the - # server stalls. Older riva-client versions do not accept a - # ``timeout`` kwarg, so pass it only when supported. - if timeout is None or not cls._supports_timeout_kwarg(rpc): - return MappingProxyType({}) - return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)}) - - @staticmethod - def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool: - try: - sig: Final = inspect.signature(callable_obj) - except (TypeError, ValueError): - return False - params: Final = sig.parameters - if "timeout" in params: - return True - return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) - - @staticmethod - def _iter_audio_chunks(pcm_bytes: bytes): - for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES): - chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES] - if not chunk: - continue - yield chunk - @staticmethod def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]: """ @@ -570,27 +350,3 @@ class NvidiaRivaAudioTranscription: } ) return final_results - - -def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]: - """ - Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``. - - We try the SDK first (preferred) and fall back to importing the proto - module separately when the SDK packaging changes between versions. - """ - try: - import riva.client as riva_client - except ImportError as e: - raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - - riva_asr_module = riva_client - if not hasattr(riva_asr_module, "RecognitionConfig"): - try: - from riva.client.proto import riva_asr_pb2 - - riva_asr_module = riva_asr_pb2 - except ImportError as e: - raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e - - return riva_client, riva_asr_module diff --git a/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py b/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py new file mode 100644 index 00000000000..13af5820f07 --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py @@ -0,0 +1,199 @@ +""" +The part of NVIDIA Riva's speech recognition gRPC schema that LiteLLM sends and reads. + +Names and numbers match ``riva/proto/riva_asr.proto`` and ``riva/proto/riva_audio.proto`` in +https://github.com/nvidia-riva/common. Fields LiteLLM never sets or reads are left out, and the +wire format skips them. The messages live in their own descriptor pool so they never collide +with nvidia-riva-client's copies when both are imported in one process. +""" + +from dataclasses import dataclass +from enum import Enum, IntEnum +from typing import Final + +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from google.protobuf.message import Message + +_PACKAGE: Final = "nvidia.riva.asr" +_SERVICE: Final = f"/{_PACKAGE}.RivaSpeechRecognition" +RECOGNIZE_METHOD: Final = f"{_SERVICE}/Recognize" +STREAMING_RECOGNIZE_METHOD: Final = f"{_SERVICE}/StreamingRecognize" + + +class _Kind(Enum): + FLOAT = "TYPE_FLOAT" + INT32 = "TYPE_INT32" + BOOL = "TYPE_BOOL" + STRING = "TYPE_STRING" + MESSAGE = "TYPE_MESSAGE" + BYTES = "TYPE_BYTES" + ENUM = "TYPE_ENUM" + + +class AudioEncoding(IntEnum): + ENCODING_UNSPECIFIED = 0 + LINEAR_PCM = 1 + + +LINEAR_PCM: Final = AudioEncoding.LINEAR_PCM.value + + +@dataclass(frozen=True, slots=True) +class _Field: + name: str + number: int + kind: _Kind + type_name: str = "" + repeated: bool = False + optional: bool = False + oneof: str = "" + + +_MESSAGES: Final[tuple[tuple[str, tuple[_Field, ...]], ...]] = ( + ( + "RecognitionConfig", + ( + _Field("encoding", 1, _Kind.ENUM, "AudioEncoding"), + _Field("sample_rate_hertz", 2, _Kind.INT32), + _Field("language_code", 3, _Kind.STRING), + _Field("max_alternatives", 4, _Kind.INT32), + _Field("profanity_filter", 5, _Kind.BOOL), + _Field("audio_channel_count", 7, _Kind.INT32), + _Field("enable_word_time_offsets", 8, _Kind.BOOL), + _Field("enable_automatic_punctuation", 11, _Kind.BOOL), + _Field("model", 13, _Kind.STRING), + _Field("verbatim_transcripts", 14, _Kind.BOOL), + _Field("endpointing_config", 25, _Kind.MESSAGE, "EndpointingConfig", optional=True), + ), + ), + ( + "EndpointingConfig", + ( + _Field("start_history", 1, _Kind.INT32, optional=True), + _Field("start_threshold", 2, _Kind.FLOAT, optional=True), + _Field("stop_history", 3, _Kind.INT32, optional=True), + _Field("stop_threshold", 4, _Kind.FLOAT, optional=True), + _Field("stop_history_eou", 5, _Kind.INT32, optional=True), + _Field("stop_threshold_eou", 6, _Kind.FLOAT, optional=True), + ), + ), + ( + "StreamingRecognitionConfig", + ( + _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"), + _Field("interim_results", 2, _Kind.BOOL), + ), + ), + ( + "StreamingRecognizeRequest", + ( + _Field("streaming_config", 1, _Kind.MESSAGE, "StreamingRecognitionConfig", oneof="streaming_request"), + _Field("audio_content", 2, _Kind.BYTES, oneof="streaming_request"), + ), + ), + ( + "WordInfo", + ( + _Field("start_time", 1, _Kind.INT32), + _Field("end_time", 2, _Kind.INT32), + _Field("word", 3, _Kind.STRING), + ), + ), + ( + "SpeechRecognitionAlternative", + ( + _Field("transcript", 1, _Kind.STRING), + _Field("words", 3, _Kind.MESSAGE, "WordInfo", repeated=True), + ), + ), + ( + "StreamingRecognitionResult", + ( + _Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True), + _Field("is_final", 2, _Kind.BOOL), + ), + ), + ( + "StreamingRecognizeResponse", + (_Field("results", 1, _Kind.MESSAGE, "StreamingRecognitionResult", repeated=True),), + ), + ( + "RecognizeRequest", + ( + _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"), + _Field("audio", 2, _Kind.BYTES), + ), + ), + ( + "SpeechRecognitionResult", + (_Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),), + ), + ( + "RecognizeResponse", + (_Field("results", 1, _Kind.MESSAGE, "SpeechRecognitionResult", repeated=True),), + ), +) + + +def _oneof_names(fields: tuple[_Field, ...]) -> tuple[str, ...]: + declared: Final = tuple(dict.fromkeys(field.oneof for field in fields if field.oneof)) + synthetic: Final = tuple(f"_{field.name}" for field in fields if field.optional) + return declared + synthetic + + +def _field_proto(field: _Field, oneofs: tuple[str, ...]) -> descriptor_pb2.FieldDescriptorProto: + oneof: Final = f"_{field.name}" if field.optional else field.oneof + return descriptor_pb2.FieldDescriptorProto( + name=field.name, + number=field.number, + type=descriptor_pb2.FieldDescriptorProto.Type.Value(field.kind.value), + label=( + descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED + if field.repeated + else descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL + ), + type_name=f".{_PACKAGE}.{field.type_name}" if field.type_name else None, + oneof_index=oneofs.index(oneof) if oneof else None, + proto3_optional=field.optional or None, + ) + + +def _message_proto(name: str, fields: tuple[_Field, ...]) -> descriptor_pb2.DescriptorProto: + oneofs: Final = _oneof_names(fields) + return descriptor_pb2.DescriptorProto( + name=name, + field=[_field_proto(field, oneofs) for field in fields], + oneof_decl=[descriptor_pb2.OneofDescriptorProto(name=oneof) for oneof in oneofs], + ) + + +_FILE: Final = descriptor_pb2.FileDescriptorProto( + name="litellm/nvidia_riva/riva_asr.proto", + package=_PACKAGE, + syntax="proto3", + enum_type=[ + descriptor_pb2.EnumDescriptorProto( + name="AudioEncoding", + value=[ + descriptor_pb2.EnumValueDescriptorProto(name=encoding.name, number=encoding.value) + for encoding in AudioEncoding + ], + ) + ], + message_type=[_message_proto(name, fields) for name, fields in _MESSAGES], +) +_POOL: Final = descriptor_pool.DescriptorPool() +_POOL.AddSerializedFile(_FILE.SerializeToString()) + + +def _message_class(name: str) -> type[Message]: + return message_factory.GetMessageClass(_POOL.FindMessageTypeByName(f"{_PACKAGE}.{name}")) + + +RecognitionConfig: Final = _message_class("RecognitionConfig") +EndpointingConfig: Final = _message_class("EndpointingConfig") +StreamingRecognitionConfig: Final = _message_class("StreamingRecognitionConfig") +StreamingRecognizeRequest: Final = _message_class("StreamingRecognizeRequest") +StreamingRecognizeResponse: Final = _message_class("StreamingRecognizeResponse") +RecognizeRequest: Final = _message_class("RecognizeRequest") +RecognizeResponse: Final = _message_class("RecognizeResponse") diff --git a/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py new file mode 100644 index 00000000000..8367853673f --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py @@ -0,0 +1,113 @@ +""" +gRPC calls to NVIDIA Riva's speech recognition service. + +The ``authorization`` and NVCF ``function-id`` metadata ride on each call, never on the channel, so every +request carries them exactly once over both plaintext and TLS. +""" + +from collections.abc import Iterator, Mapping, Sequence +from typing import Any, Final, TypeAlias + +import grpc +from google.protobuf.message import Message +from pydantic import TypeAdapter + +from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException + +# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches +# NVIDIA's recommended chunk size for streaming ASR, small enough for +# responsive endpointing, large enough to keep per-RPC overhead low. +_CHUNK_SAMPLES: Final = 1600 +_CHUNK_BYTES: Final = _CHUNK_SAMPLES * 2 + +CallMetadata: TypeAlias = tuple[tuple[str, str], ...] + +_ENDPOINTING_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def call_metadata(api_key: str | None, nvcf_function_id: object) -> CallMetadata: + function_id: Final = (("function-id", str(nvcf_function_id)),) if nvcf_function_id else () + authorization: Final = (("authorization", f"Bearer {api_key}"),) if api_key else () + return function_id + authorization + + +def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel: + if use_ssl: + return grpc.secure_channel(api_base, grpc.ssl_channel_credentials()) + return grpc.insecure_channel(api_base) + + +def recognition_config(config: Mapping[str, Any]) -> Message: + endpointing: Final = _endpointing_config(config.get("endpointing_config")) + return schema.RecognitionConfig( + encoding=schema.LINEAR_PCM, + sample_rate_hertz=int(config["sample_rate_hertz"]), + language_code=config["language_code"], + audio_channel_count=int(config["audio_channel_count"]), + enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)), + enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)), + max_alternatives=int(config.get("max_alternatives", 1)), + model=config.get("model", "") or "", + verbatim_transcripts=bool(config.get("verbatim_transcripts", False)), + profanity_filter=bool(config.get("profanity_filter", False)), + **({} if endpointing is None else {"endpointing_config": endpointing}), + ) + + +def _endpointing_config(endpointing: object) -> Message | None: + try: + fields: Final = _ENDPOINTING_FIELDS.validate_python(endpointing) + return schema.EndpointingConfig(**fields) if fields else None + except (ValueError, TypeError): + # A missing, unknown or mistyped field falls back to Riva's default + # endpointing rather than failing the whole request. + return None + + +def _streaming_requests(config: Message, pcm_bytes: bytes) -> Iterator[Message]: + yield schema.StreamingRecognizeRequest( + streaming_config=schema.StreamingRecognitionConfig(config=config, interim_results=False) + ) + for offset in range(0, len(pcm_bytes), _CHUNK_BYTES): + yield schema.StreamingRecognizeRequest(audio_content=pcm_bytes[offset : offset + _CHUNK_BYTES]) + + +def streaming_recognize( + channel: grpc.Channel, + config: Message, + pcm_bytes: bytes, + metadata: CallMetadata, + timeout: float | None, +) -> Iterator[Message]: + rpc: Final = channel.stream_stream( + schema.STREAMING_RECOGNIZE_METHOD, + request_serializer=schema.StreamingRecognizeRequest.SerializeToString, + response_deserializer=schema.StreamingRecognizeResponse.FromString, + ) + return rpc(_streaming_requests(config, pcm_bytes), metadata=metadata, timeout=timeout) + + +def recognize( + channel: grpc.Channel, + config: Message, + pcm_bytes: bytes, + metadata: CallMetadata, + timeout: float | None, +) -> Sequence[Message]: + rpc: Final = channel.unary_unary( + schema.RECOGNIZE_METHOD, + request_serializer=schema.RecognizeRequest.SerializeToString, + response_deserializer=schema.RecognizeResponse.FromString, + ) + pending: Final = rpc.future(schema.RecognizeRequest(config=config, audio=pcm_bytes), metadata=metadata) + try: + return (pending.result(timeout=timeout),) + except Exception as e: + if pending.done(): + raise + pending.cancel() + raise NvidiaRivaException( + status_code=408, + message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds", + ) from e diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4bfc9f28939..0479ac5c36c 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -556,18 +556,23 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]: """ A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on - api.openai.com, or one opted in through ``prompt_cache_options`` or the cost - map) gets each Anthropic ``cache_control`` mark rewritten in place, and under - drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat - completions accepts. Any other endpoint keeps or strips the marks as before, and - under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read. + api.openai.com, or one opted in through ``prompt_cache_options``, its own + ``model_info``, or the cost map) gets each + Anthropic ``cache_control`` mark rewritten in place, and under drop_params each + content part's ``prompt_cache_breakpoint`` cut down to what chat completions + accepts. Any other endpoint keeps or strips the marks as before, and under + drop_params also loses every ``prompt_cache_breakpoint`` it cannot read. """ - from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, routed_model_info custom_llm_provider: Final = _litellm_params_str(litellm_params, "custom_llm_provider") api_base: Final = _litellm_params_str(litellm_params, "api_base") reads_breakpoints: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, prompt_cache_options + model, + custom_llm_provider, + api_base, + prompt_cache_options, + routed_model_info(litellm_params.get("model_info")), ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): return ( diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 36d7e4d95de..bc8aaab3185 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -31,9 +31,9 @@ def cost_per_token( Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information - - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), - inferred from api_base. Applies the model's regional-processing - uplift multiplier when set. + - data_residency: optional OpenAI data-residency mode (e.g. "eu", "us", "fedramp"), + inferred from api_base, including gov.api.openai.com. Applies the + model's regional-processing uplift multiplier when set. Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd diff --git a/litellm/llms/openai/data_residency.py b/litellm/llms/openai/data_residency.py index d4859221541..a8831db3517 100644 --- a/litellm/llms/openai/data_residency.py +++ b/litellm/llms/openai/data_residency.py @@ -7,9 +7,12 @@ enabled and rejects requests sent to the wrong host, so the api_base hostname is the authoritative signal of which region a request was processed in. """ +import os from typing import Final from urllib.parse import urlparse +import litellm + # Mapping of OpenAI regional hostnames to the corresponding data-residency # value used by the cost calculator. See # https://developers.openai.com/api/docs/pricing for the regional-processing @@ -17,6 +20,7 @@ from urllib.parse import urlparse _OPENAI_REGIONAL_HOSTS: Final[dict[str, str]] = { "eu.api.openai.com": "eu", "us.api.openai.com": "us", + "gov.api.openai.com": "fedramp", } @@ -25,13 +29,22 @@ def infer_openai_data_residency(custom_llm_provider: str | None, api_base: str | Derive the OpenAI data-residency region from an api_base URL. Returns ``"eu"`` for the EU regional host, ``"us"`` for the US regional - host, and ``None`` for the default global host, any non-OpenAI provider, - or any non-OpenAI URL. + host, ``"fedramp"`` for ``gov.api.openai.com``, and ``None`` for the + default global host, any non-OpenAI provider, or any non-OpenAI URL. + + When ``api_base`` is unset, falls back to the same defaults the OpenAI + handler uses (``litellm.api_base``, ``OPENAI_BASE_URL``, ``OPENAI_API_BASE``) + so env-configured regional endpoints are billed with their uplift. """ - if custom_llm_provider != "openai" or not api_base: + if custom_llm_provider != "openai": + return None + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: return None try: - host: Final = urlparse(api_base).hostname + host: Final = urlparse(resolved_api_base).hostname except (TypeError, ValueError): return None if not host: diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 369b6959411..5aa72579d6a 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -307,10 +307,13 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> def ir_to_openai_response( - response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str + response: DecisionsIRResponse, + request: DecisionsIRRequest, + requested_model: str, + evidence_fields: frozenset[str] = frozenset(), ) -> OpenAIDecisionResponse: usage: Final = response.usage or DecisionsIRUsage() - return OpenAIDecisionResponse( + formatted: Final = OpenAIDecisionResponse( model=response.model or requested_model, answers=tuple( _openai_answer(question, answer) @@ -327,6 +330,7 @@ def ir_to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) + return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()}) class OpenAIDecisionsConfig(BaseDecisionsConfig): diff --git a/litellm/llms/openai/live/handler.py b/litellm/llms/openai/live/handler.py index 297a147a288..5b899ac4162 100644 --- a/litellm/llms/openai/live/handler.py +++ b/litellm/llms/openai/live/handler.py @@ -18,6 +18,7 @@ from litellm._logging import redact_secrets from litellm.constants import ( OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS, OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) @@ -316,12 +317,15 @@ async def _forward_live_backend_frames( backend_websocket: LiveBackendWebSocket, client_gone: asyncio.Event, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed event_loop: Final = asyncio.get_running_loop() - async for message, close_deadline in _live_backend_messages(websocket, backend_websocket): + async for message, close_deadline in _live_backend_messages( + websocket, backend_websocket, upstream_ended_abnormally + ): if (event := _metering_event(message)) is not None: event_retention.retain(event) if client_gone.is_set(): @@ -341,6 +345,7 @@ async def _forward_live_backend_frames( async def _live_backend_messages( websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, + upstream_ended_abnormally: asyncio.Event, ) -> AsyncIterator[tuple[str, float | None]]: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed @@ -350,6 +355,8 @@ async def _live_backend_messages( yield message, None except ConnectionClosed as error: upstream_close: Final = backend_close_from(error) + if not upstream_close.ended_cleanly: + upstream_ended_abnormally.set() with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError): await websocket.close(code=upstream_close.code, reason=upstream_close.reason) return @@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime): "complete_input_dict": {"session_start": rewritten_start}, } event_retention: Final = _LiveEventRetention() + session_ended_abnormally: Final = asyncio.Event() try: logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args) async with self._websocket_connector( @@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime): websocket=websocket, backend_websocket=backend_websocket, event_retention=event_retention, + upstream_ended_abnormally=session_ended_abnormally, ) + except Exception: + session_ended_abnormally.set() + raise finally: session_end_time: Final = self._clock() retained_events: Final = event_retention.results() @@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime): logging_obj.model_call_details[ # rebind-ok: flags the shared logging object as already logged REALTIME_SESSION_SUCCESS_LOGGED_KEY ] = True + if session_ended_abnormally.is_set(): + logging_obj.model_call_details[ # rebind-ok: flags the shared logging object for the router + REALTIME_SESSION_ENDED_ABNORMALLY_KEY + ] = True except InvalidStatus as error: await close_after_upstream_handshake_refusal(websocket, error.response.status_code) except Exception as error: # noqa: BLE001 # any relay failure must still close the client socket with a redacted reason @@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime): websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from websockets.exceptions import ConnectionClosed @@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime): backend_websocket=backend_websocket, client_gone=client_gone, event_retention=event_retention, + upstream_ended_abnormally=upstream_ended_abnormally, ) ) client_gone_task: Final = asyncio.create_task(client_gone.wait()) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 6e22ef9112e..87c944a642a 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -30,6 +30,10 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, + run_chat_completion_agentic_hooks, +) from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing @@ -663,67 +667,23 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): Returns the response from agentic loop, or None if no hook runs. """ - from litellm._logging import verbose_logger - from litellm.integrations.custom_logger import CustomLogger - - callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or []) - # Avoid logging full callback objects to prevent leaking sensitive data - verbose_logger.debug("LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks)) - tools: Final = optional_params.get("tools", []) - # Avoid logging full tools payloads; they may contain sensitive parameters - verbose_logger.debug( - "LiteLLM.AgenticHooks: tools_count=%s", - len(tools) if isinstance(tools, list) else 1 if tools else 0, + if not isinstance(response, ModelResponse): + return None + return await bill_first_round_on_failure( + run_chat_completion_agentic_hooks( + response=response, + model=model, + messages=messages, + optional_params=optional_params, + kwargs=litellm_params or {}, + logging_obj=logging_obj, + custom_llm_provider=litellm_params.get("custom_llm_provider", "openai"), + stream=stream, + ), + logging_obj=logging_obj, + response=response, + stream=stream, ) - # Get custom_llm_provider from litellm_params - custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai") - - for callback in callbacks: - try: - if isinstance(callback, CustomLogger): - # Check if the callback has the chat completion agentic loop methods - if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"): - continue - - # First: Check if agentic loop should run (using chat completion method) - ( - should_run, - tool_calls, - ) = await callback.async_should_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - tools=tools, - stream=stream, - custom_llm_provider=custom_llm_provider, - kwargs=litellm_params, - ) - - if should_run: - # Second: Execute agentic loop - kwargs_with_provider = litellm_params.copy() if litellm_params else {} - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - - # For OpenAI Chat Completions, use the chat completion agentic loop method - agentic_response: object = await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - # First hook that runs agentic loop wins - return agentic_response - - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e - ) - - return None def mock_streaming( self, @@ -1074,8 +1034,28 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): hidden_params={"headers": headers}, _response_headers=headers, ) + except openai.UnprocessableEntityError as e: + if litellm.drop_params is True or drop_params is True: + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) + else: + raise e + except Exception as e: + exception_response: object = getattr(e, "response", None) + status_code = getattr(e, "status_code", 500) + exception_body = getattr(e, "body", None) + error_headers = getattr(e, "headers", None) + if error_headers is None and exception_response: + error_headers = getattr(exception_response, "headers", None) + message = getattr(e, "message", str(e)) - # Call agentic completion hooks (e.g., for websearch_interception) + raise OpenAIError( + status_code=status_code, + message=message, + response=upstream_error_response(e), + headers=error_headers, + body=exception_body, + ) + else: agentic_response = await self._call_agentic_completion_hooks_openai( response=final_response_obj, model=model, @@ -1098,29 +1078,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) return final_response_obj - except openai.UnprocessableEntityError as e: - ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 - if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) - else: - raise e - # e.message - except Exception as e: - exception_response: object = getattr(e, "response", None) - status_code = getattr(e, "status_code", 500) - exception_body = getattr(e, "body", None) - error_headers = getattr(e, "headers", None) - if error_headers is None and exception_response: - error_headers = getattr(exception_response, "headers", None) - message = getattr(e, "message", str(e)) - - raise OpenAIError( - status_code=status_code, - message=message, - response=upstream_error_response(e), - headers=error_headers, - body=exception_body, - ) def streaming( self, diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 5be488f0f6e..40ead28d952 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -141,7 +141,7 @@ class OpenAIRealtime(OpenAIChatCompletion): return ssl_config - def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str: + def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str: """ Construct the backend websocket URL with the client's query parameters. @@ -158,7 +158,8 @@ class OpenAIRealtime(OpenAIChatCompletion): api_base = api_base.replace("http://", "ws://") url = URL(api_base) # Set the correct path - url = url.copy_with(path="/v1/realtime") + path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime" + url = url.copy_with(path=path) upstream_params: Final = tuple( (key, value) for key, value in query_params.items() if not (drops_model and key == "model") ) @@ -197,6 +198,8 @@ class OpenAIRealtime(OpenAIChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", + drop_params: bool = False, **kwargs: object, ): import websockets @@ -210,13 +213,15 @@ class OpenAIRealtime(OpenAIChatCompletion): # Use all query params if provided, else fallback to just model if query_params is None: query_params = {"model": model} - url: Final = self._construct_url(api_base, query_params) + url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode) try: # Get provider-specific SSL configuration ssl_config: Final = self._get_ssl_config(url) - openai_beta_realtime: Final = client_sent_openai_beta_realtime_header(websocket) + openai_beta_realtime: Final = realtime_mode != "translation" and client_sent_openai_beta_realtime_header( + websocket + ) if not openai_beta_realtime: verbose_logger.debug( "OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). " @@ -253,6 +258,8 @@ class OpenAIRealtime(OpenAIChatCompletion): ), event_normalizer=self._make_event_normalizer(), usage_meter=self._make_usage_meter(), + translation_session=realtime_mode == "translation", + drop_params=drop_params, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 49d33582c6f..ab32ae6085e 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -28,6 +28,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/calls" + def validate_environment( self, headers: dict, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9f4a4fd6083..3cb798b7318 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + deployment_model_info, deployment_prompt_cache_breakpoint_support, targets_openai_api, ) @@ -668,7 +669,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): prompt_cache_options: object = None, ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: validated_input: Final = self._validate_input_param( - self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) + self._drop_bridge_minted_reasoning_items(input), drop_params=drops_params(litellm_params) ) reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options @@ -767,6 +768,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): custom_llm_provider=_provider_name(self.custom_llm_provider), api_base=host.api_base or host.base_url, prompt_cache_options=prompt_cache_options, + model_info=deployment_model_info(litellm_params), ) @staticmethod @@ -949,7 +951,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized def _validate_input_param( - self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False + self, input: str | ResponseInputParam, *, drop_params: bool = False ) -> str | ResponseInputParam: """ Ensure all input fields if pydantic are converted to dict @@ -963,9 +965,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): # if it's pydantic, convert to dict if isinstance(item, BaseModel): validated_input.append( - replayable_responses_item( - item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by - ) + replayable_responses_item(item.model_dump(exclude_none=True), drops_set_created_by=drop_params) ) elif isinstance(item, dict): validated_input.append( @@ -973,7 +973,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): self._handle_reasoning_item(cast(dict[str, object], item)) if item.get("type") == "reasoning" else item, - drops_set_created_by=drops_set_created_by, + drops_set_created_by=drop_params, ) ) else: diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index e7f8dd04516..b9eaa204567 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -26,8 +26,8 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): self, model: str, audio_file: FileTypes, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request @@ -37,3 +37,14 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): return AudioTranscriptionRequestData( data=data, ) + + +class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: + return [ + "prompt", + "response_format", + "keywords", + "languages", + "stream", + ] diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 48a44575989..23c5e70a3cc 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,8 +1,9 @@ -from collections.abc import Mapping +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, Optional, cast import httpx -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel if TYPE_CHECKING: @@ -12,6 +13,9 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + acreate_sdk_transcription, + create_sdk_transcription, + sdk_compatible_transcription_request_data, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -28,31 +32,18 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) - headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() - - return headers, response - except Exception as e: - raise e + return await acreate_sdk_transcription(openai_aclient, sdk_compatible_transcription_request_data(data), timeout) def make_sync_openai_audio_transcriptions_request( self, openai_client: OpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) - headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() - return headers, response - except Exception as e: - raise e + return create_sdk_transcription(openai_client, sdk_compatible_transcription_request_data(data), timeout) def audio_transcriptions( self, @@ -70,7 +61,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): atranscription: bool = False, provider_config: BaseAudioTranscriptionConfig | None = None, shared_session: Optional["ClientSession"] = None, - ) -> TranscriptionResponse: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): """ Handle audio transcription request """ @@ -128,6 +123,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): ) logging_obj.model_call_details["response_headers"] = headers + if isinstance(response, Stream): + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -153,7 +151,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def async_audio_transcriptions( self, audio_file: FileTypes, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: LiteLLMLoggingObj, @@ -192,6 +190,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers + if isinstance(response, AsyncStream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() else: diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index d7aafe9f06e..66b41967ca7 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,9 +2,11 @@ Dynamic configuration class generator for JSON-based providers. """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, Literal, overload +from pydantic import TypeAdapter, ValidationError + from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, @@ -20,6 +22,26 @@ from .json_loader import SimpleProviderConfig if TYPE_CHECKING: from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig +_PAID_SEARCH_TOOLS: Final = frozenset({"web_search", "web_search_image", "image_search"}) +_PAYLOAD_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_TOOL_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, int]]] = TypeAdapter(Mapping[str, int]) + + +def web_search_requests_from_tool_usage(payload: object) -> int | None: + """ + Billable search calls in a parsed OpenAI-compatible chat response whose usage carries a + tool_usage {tool_name: count} map, as EmpirioLabs reports built-in tool invocations. + Free tools (code_interpreter, web_extractor) are not counted. + """ + try: + body: Final = _PAYLOAD_ADAPTER.validate_python(payload) + usage: Final = _PAYLOAD_ADAPTER.validate_python(body.get("usage")) + tool_usage: Final = _TOOL_USAGE_ADAPTER.validate_python(usage.get("tool_usage"), strict=True) + except ValidationError: + return None + calls: Final = sum(count for tool, count in tool_usage.items() if tool in _PAID_SEARCH_TOOLS and count > 0) + return calls if calls > 0 else None + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -179,6 +201,25 @@ def create_config_class(provider: SimpleProviderConfig): return optional_params + def transform_parsed_response_dict( + self, + parsed_response: dict[str, object], # mutable-ok: BaseConfig signature + ) -> dict[str, object]: # mutable-ok: BaseConfig signature + """ + Carry the provider's usage.tool_usage search count as usage.server_tool_use.web_search_requests, + the field the built-in tool cost path bills per call. + """ + repaired: Final = _PAYLOAD_ADAPTER.validate_python( + super().transform_parsed_response_dict(parsed_response) # pyright: ignore[reportUnknownMemberType] # base_class is picked at runtime + ) + if not provider.special_handling.get("web_search_requests_from_tool_usage"): + return {**repaired} + web_search_requests: Final = web_search_requests_from_tool_usage(repaired) + if web_search_requests is None: + return {**repaired} + usage: Final = _PAYLOAD_ADAPTER.validate_python(repaired["usage"]) + return {**repaired, "usage": {**usage, "server_tool_use": {"web_search_requests": web_search_requests}}} + @property def custom_llm_provider(self) -> str | None: return provider.slug diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index a71a9c3bb9b..f937e4d2f75 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -178,7 +178,10 @@ "param_mappings": { "max_completion_tokens": "max_tokens" }, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + "special_handling": { + "web_search_requests_from_tool_usage": true + }, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] }, "meta": { "base_url": "https://api.meta.ai/v1", @@ -302,5 +305,14 @@ "preserve_cache_control": true }, "supported_endpoints": ["/v1/chat/completions"] + }, + "wally": { + "base_url": "https://inference.runanywhere.ai/v1", + "api_key_env": "WALLY_API_KEY", + "api_base_env": "WALLY_API_BASE", + "provider_info": { + "supports_function_calling": true, + "supports_reasoning": true + } } } diff --git a/litellm/llms/opencode/chat/transformation.py b/litellm/llms/opencode/chat/transformation.py index 7e16f93d7cb..c6d288b6d84 100644 --- a/litellm/llms/opencode/chat/transformation.py +++ b/litellm/llms/opencode/chat/transformation.py @@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig): def custom_llm_provider(self) -> str: return "opencode_go" + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return "claude" in model.lower() + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/opencode/transformation.py b/litellm/llms/opencode/transformation.py deleted file mode 100644 index 9b62e7aca95..00000000000 --- a/litellm/llms/opencode/transformation.py +++ /dev/null @@ -1,6 +0,0 @@ -from litellm.llms.opencode.common_utils import is_opencode_api_base - - -class OpenCodeProviderAdapter: - def get_provider_affinity_header(self, api_base: str | None) -> str | None: - return "x-opencode-session" if is_opencode_api_base(api_base) else None diff --git a/tests/test_litellm_rust/cache/v2/__init__.py b/litellm/llms/parallel_ai/decisions/__init__.py similarity index 100% rename from tests/test_litellm_rust/cache/v2/__init__.py rename to litellm/llms/parallel_ai/decisions/__init__.py diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py new file mode 100644 index 00000000000..34d03047301 --- /dev/null +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -0,0 +1,89 @@ +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict + +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, ir_to_systemone_request +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + UnsupportedDecisionsRequest, +) + + +class _ParallelResponse(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + mode: str + answers: Mapping[str, Mapping[str, object]] + + +class ParallelAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY") + api_base_env = ("PARALLEL_AI_API_BASE",) + confidence_optional = True + openai_evidence_fields = frozenset({"citations", "decision_id", "mode"}) + + def get_default_api_base(self) -> str: + return "https://api.parallel.ai" + + def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]: + configured_base: Final = self.resolve_api_base(None) + resolved_base: Final = api_base or configured_base + destination: Final = self._decisions_url(resolved_base) + if api_key: + return resolved_base, api_key + if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}: + raise ValueError( + "Parallel Decisions requires an explicit api_key for a custom api_base unless " + "the destination is configured in PARALLEL_AI_API_BASE" + ) + return resolved_base, self.resolve_api_key(None) + + def get_complete_url(self, api_base: str, model: str) -> str: + return super().get_complete_url(api_base.rstrip("/").removesuffix("/v1/search").removesuffix(self.path), model) + + def _decisions_url(self, api_base: str | None) -> httpx.URL: + return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision")) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + return { + **{ + name: value + for name, value in headers.items() + if name.lower() not in {"authorization", "x-api-key", "content-type"} + }, + **({"x-api-key": api_key} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def transform_decisions_request( + self, model: str, request: DecisionsIRRequest, custom_llm_provider: str + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if model != "decision": + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports the decision model") + if len(request.questions) > 100: + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports at most 100 questions") + body: Final = ir_to_systemone_request(model, request) + if isinstance(body, UnsupportedDecisionsRequest): + return body + return {"mode": "fast", **{key: value for key, value in body.items() if key != "model"}} + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = _ParallelResponse.model_validate(payload) + return super().parse_response( + { + **response.model_dump(), + "model": "decision", + "answers": {key: answer for key, answer in response.answers.items() if answer.get("type") != "refusal"}, + }, + request, + ) diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index afd27b0f696..e5e3744dd80 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -89,7 +89,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}.""" input = self._ensure_message_type(input) if model.startswith("preset/"): - input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params)) + input = self._validate_input_param(input, drop_params=drops_params(litellm_params)) data: Final[dict] = { "preset": model[len("preset/") :], "input": input, diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 32d7b39a20e..80244cfc3eb 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -5,9 +5,11 @@ import json import re import threading import time +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from functools import lru_cache, partial +from itertools import chain, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -34,7 +36,11 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning -from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp +from litellm.litellm_core_utils.tool_fields import ( + declared_function_name, + qualified_member_name, + tool_params_without_hosted_mcp, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -544,6 +550,138 @@ def mapped_gemini_speech_config( } +def _as_chat_tool(member: JsonValue) -> JsonValue: + if not isinstance(member, dict) or "function" in member or "name" not in member: + return member + return { + "type": member.get("type", "function"), + "function": {key: value for key, value in member.items() if key != "type"}, + } + + +def _namespace_members_or_tool(tool: JsonValue) -> tuple[JsonValue, ...]: + if not isinstance(tool, dict) or tool.get("type") != "namespace": + return (tool,) + members: Final = tool.get("tools") + return _members_named_once(tuple(map(_as_chat_tool, members))) if isinstance(members, list) else () + + +def _members_named_once( + members: tuple[JsonValue, ...], repeatable: frozenset[str] = frozenset() +) -> tuple[JsonValue, ...]: + names: Final = tuple(map(declared_function_name, members)) + first_index: Final = {name: index for index, name in reversed(tuple(enumerate(names)))} + return tuple( + member + for index, (member, name) in enumerate(zip(members, names, strict=True)) + if name is None or name in repeatable or first_index[name] == index + ) + + +def _is_namespace(tool: JsonValue) -> bool: + return isinstance(tool, dict) and tool.get("type") == "namespace" + + +def _top_level_name(tool: JsonValue) -> str | None: + name: Final = tool.get("name") if isinstance(tool, dict) else None + return name if isinstance(name, str) and name else None + + +def _listed(tools: JsonValue) -> tuple[JsonValue, ...]: + return tuple(tools) if isinstance(tools, list) else () + + +def _declarable_names(tool: JsonValue) -> tuple[str | None, ...]: + members: Final = _namespace_members_or_tool(tool) if _is_namespace(tool) else () + return (declared_function_name(tool), *map(declared_function_name, members)) + + +def _declarable_names_in(tools: tuple[JsonValue, ...]) -> tuple[str | None, ...]: + return tuple(chain.from_iterable(map(_declarable_names, tools))) + + +def _inlines_members(tool: JsonValue, declarable: Counter[str | None]) -> bool: + members: Final = _namespace_members_or_tool(tool) + return _is_namespace(tool) and all( + name is None or declarable[name] == 1 for name in map(declared_function_name, members) + ) + + +def _qualified_member(namespace: str, member: JsonValue) -> JsonValue: + function: Final = member.get("function") if isinstance(member, dict) else None + name: Final = declared_function_name(member) + if not isinstance(member, dict) or not isinstance(function, dict) or name is None: + return member + return {**member, "function": {**function, "name": qualified_member_name(namespace, name)}} + + +def _as_declared( + tool: JsonValue, declarable: Counter[str | None], top_level_names: frozenset[str] +) -> tuple[JsonValue, ...]: + if _inlines_members(tool, declarable): + return _namespace_members_or_tool(tool) + namespace: Final = _top_level_name(tool) + if not _is_namespace(tool) or namespace is None: + return (tool,) + qualified: Final = (_qualified_member(namespace, member) for member in _namespace_members_or_tool(tool)) + return tuple(member for member in qualified if declared_function_name(member) not in top_level_names) + + +def _top_level_names_in(tools: tuple[JsonValue, ...]) -> frozenset[str]: + top_level_tools: Final = (tool for tool in tools if not _is_namespace(tool)) + return frozenset(name for name in map(declared_function_name, top_level_tools) if name is not None) + + +def _with_namespace_members_inlined(tools: JsonValue, outside_declarations: JsonValue = None) -> JsonValue: + if not isinstance(tools, list) or not any(map(_is_namespace, tools)): + return tools + candidates: Final = (*tools, *_listed(outside_declarations)) + declarable: Final = Counter(_declarable_names_in(candidates)) + top_level_names: Final = _top_level_names_in(candidates) + declared: Final = chain.from_iterable(_as_declared(tool, declarable, top_level_names) for tool in tools) + return list(_members_named_once(tuple(declared), top_level_names)) + + +def _names_each_tool_declares(candidates: tuple[JsonValue, ...]) -> tuple[tuple[str, ...], ...]: + declarable: Final = Counter(_declarable_names_in(candidates)) + top_level_names: Final = _top_level_names_in(candidates) + declared: Final = tuple( + tuple(filter(None, map(declared_function_name, _as_declared(tool, declarable, top_level_names)))) + for tool in candidates + ) + owned_names: Final = chain.from_iterable(zip(repeat(index), names) for index, names in enumerate(declared)) + first_owner: Final = {name: index for index, name in reversed(tuple(owned_names))} + return tuple(tuple(name for name in names if first_owner[name] == index) for index, names in enumerate(declared)) + + +def _qualified_names_of( + member_name: str, tool: JsonValue, declarable: Counter[str | None], declared: tuple[str, ...] +) -> tuple[str, ...]: + namespace: Final = _top_level_name(tool) + if not _is_namespace(tool) or namespace is None or _inlines_members(tool, declarable): + return () + qualified: Final = qualified_member_name(namespace, member_name) + return (qualified,) if qualified in declared else () + + +def _forced_function_names(name: str, tools: JsonValue) -> tuple[str, ...]: + candidates: Final = _listed(tools) + if any(not _is_namespace(tool) and declared_function_name(tool) == name for tool in candidates): + return (name,) + declarable: Final = Counter(_declarable_names_in(candidates)) + declared_by_tool: Final = tuple(zip(candidates, _names_each_tool_declares(candidates), strict=True)) + named: Final = next( + (declared for tool, declared in declared_by_tool if _is_namespace(tool) and _top_level_name(tool) == name), + None, + ) + if named is not None: + return named + qualified: Final = chain.from_iterable( + _qualified_names_of(name, tool, declarable, declared) for tool, declared in declared_by_tool + ) + return tuple(qualified) or (name,) + + class VertexAIBaseConfig: def get_mapped_special_auth_params(self) -> dict: """ @@ -815,7 +953,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return supported_params - def map_tool_choice_values(self, model: str, tool_choice: str | dict) -> ToolConfig | None: + def map_tool_choice_values( + self, model: str, tool_choice: str | dict, tools: JsonValue = None, drop_params: bool = False + ) -> ToolConfig | None: if tool_choice == "none": return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="NONE")) elif tool_choice == "required": @@ -825,7 +965,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif isinstance(tool_choice, dict): # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html name: Final = tool_choice.get("function", {}).get("name", "") - return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="ANY", allowed_function_names=[name])) + forced: Final = _forced_function_names(name, tools) + if forced: + return ToolConfig( + functionCallingConfig=FunctionCallingConfig(mode="ANY", allowed_function_names=list(forced)) + ) + if drops_unsupported_params(drop_params): + return None + raise litellm.utils.UnsupportedParamsError( + message=f"VertexAI can't force tool_choice '{name}', since none of that namespace's members can be sent to Gemini as a function. Name one of its functions instead, or set `litellm.drop_params = True` to drop tool_choice.", + status_code=400, + ) else: raise litellm.utils.UnsupportedParamsError( message=f"VertexAI doesn't support tool_choice={tool_choice}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.", @@ -1068,7 +1218,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext - def _map_function(self, value: list[dict], optional_params: dict, send_integer_enums: bool = False) -> list[Tools]: + def _map_function( + self, + value: list[dict], + optional_params: dict, + send_integer_enums: bool = False, + outside_declarations: JsonValue = None, + ) -> list[Tools]: """ Map OpenAI-style tools/functions to Vertex AI format. @@ -1099,7 +1255,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # remove 'strict' from tools remove_strict_from_schema(schema_value) - for tool in cast(list[dict], schema_value): # cast-ok: a deep copy keeps the caller's tool shape + inlined_tools: Final = _with_namespace_members_inlined(schema_value, outside_declarations) + for tool in cast(list[dict], inlined_tools): # cast-ok: a deep copy keeps the caller's tool shape openai_function_object: ChatCompletionToolParamFunctionChunk | None = None if "function" in tool: # tools list _openai_function_object = ChatCompletionToolParamFunctionChunk(**tool["function"]) @@ -1734,7 +1891,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif (param == "tools" or param == "functions") and isinstance(value, list) and value: # Pass optional_params so _map_function can add toolConfig if needed mapped_tools = with_integer_enums_that_fit( - partial(self._map_function, value, optional_params), + partial( + self._map_function, + value, + optional_params, + outside_declarations=request_params.get("functions" if param == "tools" else "tools"), + ), allow_integer_enums and self._is_gemini_3_or_newer(model), ) optional_params = self._add_tools_to_optional_params(optional_params, mapped_tools) @@ -1742,6 +1904,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tool_choice_value = self.map_tool_choice_values( model=model, tool_choice=value, + tools=[*_listed(non_default_params.get("tools")), *_listed(non_default_params.get("functions"))], + drop_params=drop_params, ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 98fa8ed2b77..816301c6637 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -249,7 +249,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert ) self._normalize_system_role_messages( - anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + anthropic_messages_request, model=model, drop_params=litellm_params.drop_params ) self._remove_scope_from_cache_control(anthropic_messages_request) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 014fabd5187..41db96914f5 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -2,10 +2,12 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter -import litellm +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base @@ -37,17 +39,31 @@ class VLLMError(BaseLLMException): ) +_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def _auth_headers(api_key: str | None) -> Mapping[str, str]: + if api_key is None or api_key == PLACEHOLDER_API_KEY: + return {} + return {"Authorization": f"Bearer {api_key}"} + + +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) else None + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: upstream_key: Final = self.upstream_api_key(api_key, api_base) return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers @@ -58,7 +74,7 @@ class VLLMModelInfo(BaseLLMModelInfo): return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key} @staticmethod - def get_api_base(api_base: str | None = None) -> str | None: + def get_api_base(api_base: str | None = None) -> str: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: raise ValueError( @@ -74,25 +90,55 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model - def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: - api_base = VLLMModelInfo.get_api_base(api_base) - api_key = VLLMModelInfo.get_api_key(api_key) - endpoint: Final = "/v1/models" - if api_base is None or api_key is None: - raise ValueError( - "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint." - ) - - url: Final = add_path_to_api_base(api_base, endpoint) - response: Final = litellm.module_level_client.get( - url=url, - ) - - response.raise_for_status() - - models: Final = response.json()["data"] - - return [model["id"] for model in models] + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: mirrors override contract + return [] def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VLLMError(status_code=status_code, message=error_message, headers=headers) + + +class HostedVLLMModelInfo(VLLMModelInfo): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError("HOSTED_VLLM_API_BASE is not set.") + return resolved_api_base + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("HOSTED_VLLM_API_KEY") + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None) + + def discover_models( + self, litellm_params: Mapping[str, object] | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + params: Final = litellm_params or {} + ssl_verify: Final = params.get("ssl_verify") + return self._list_models( + api_key=_str_param(params, "api_key"), + api_base=_str_param(params, "api_base"), + extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}), + ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None, + ) + + def _list_models( + self, + *, + api_key: str | None, + api_base: str | None, + extra_headers: Mapping[str, str], + ssl_verify: bool | str | None, + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") + response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get( + url=url, + headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))}, + timeout=NON_LLM_CONNECTION_TIMEOUT, + ) + response.raise_for_status() + return [model["id"] for model in response.json()["data"]] diff --git a/litellm/llms/vllm/passthrough/transformation.py b/litellm/llms/vllm/passthrough/transformation.py index 2b253e7fa38..10d740e49e0 100644 --- a/litellm/llms/vllm/passthrough/transformation.py +++ b/litellm/llms/vllm/passthrough/transformation.py @@ -67,9 +67,6 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): ) -> tuple["URL", str]: base_target_url: Final = self.get_api_base(api_base) - if base_target_url is None: - raise Exception("VLLM api base not found") - return ( self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params), base_target_url, diff --git a/litellm/main.py b/litellm/main.py index 38dd9f90be9..8cb133bf76b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,6 +37,8 @@ if TYPE_CHECKING: import dotenv import httpx import openai +from openai import AsyncStream, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import assert_never, overload @@ -79,11 +81,13 @@ from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_for_health_check, ) from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, maybe_run_chat_completion_agentic_loop, ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( + AGENTIC_LOOP_CONTROL_KWARGS_KEYS, AWS_CREDENTIAL_KWARGS_KEYS, OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY, @@ -109,6 +113,7 @@ from litellm.litellm_core_utils.mock_functions import ( mock_embedding, mock_image_generation, ) +from litellm.litellm_core_utils.per_call_pricing import per_call_model_info from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export ) @@ -165,7 +170,6 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s from litellm.types.utils import ( AgenticLoopParams, BaseUrlRoutingHint, - CustomPricingLiteLLMParams, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -266,7 +270,7 @@ from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from .llms.custom_llm import CustomLLM, custom_chat_llm_router +from .llms.custom_llm import custom_chat_llm_router from .llms.databricks.common_utils import DatabricksBase from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha @@ -753,6 +757,7 @@ async def acompletion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, ) from litellm.types.llms.openai import AllMessageValues @@ -764,6 +769,7 @@ async def acompletion( tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs api_base=base_url or kwargs.get("api_base"), + model_info=routed_model_info(caller_kwargs.get("model_info")), ) prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs @@ -950,26 +956,31 @@ async def acompletion( # no equivalent: every provider already funnels through one shared # handler where the loop is dispatched. if isinstance(response, litellm.ModelResponse): - looped: Final = await maybe_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - optional_params={ - k: v - for k, v in completion_kwargs.items() - if v is not None - and k - not in ( - "model", - "messages", - "stream", - "acompletion", - "deployment_id", - ) - }, - kwargs=kwargs, + looped: Final = await bill_first_round_on_failure( + maybe_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + optional_params={ + k: v + for k, v in completion_kwargs.items() + if v is not None + and k + not in ( + "model", + "messages", + "stream", + "acompletion", + "deployment_id", + ) + }, + kwargs=kwargs, + logging_obj=kwargs.get("litellm_logging_obj"), + custom_llm_provider=custom_llm_provider, + stream=bool(stream), + ), logging_obj=kwargs.get("litellm_logging_obj"), - custom_llm_provider=custom_llm_provider, + response=response, stream=bool(stream), ) if looped is not None: @@ -1001,7 +1012,7 @@ def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object]) def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse: - return TranscriptionResponse(**cached_response_dict) + return TranscriptionResponse.model_validate(cached_response_dict) def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse: @@ -1319,7 +1330,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1425,46 +1435,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -def _build_custom_pricing_entry( - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None = None, -) -> dict: - """Build a complete model cost entry from kwargs and model_info. - - Collects all CustomPricingLiteLLMParams fields present in kwargs and - merges metadata from model_info (mode, supports_prompt_caching, max_tokens) - so that register_model() receives the full pricing configuration. - """ - entry: Final[dict] = {"litellm_provider": custom_llm_provider} - - for field_name in CustomPricingLiteLLMParams.model_fields: - value = kwargs.get(field_name) - if value is not None: - entry[field_name] = value - - if model_info and isinstance(model_info, dict): - for key in ("mode", "supports_prompt_caching", "max_tokens"): - if key in model_info and model_info[key] is not None: - entry.setdefault(key, model_info[key]) - - return entry - - -def _get_router_deployment_id(kwargs: dict) -> str | None: - for metadata_key in ("litellm_metadata", "metadata"): - metadata = kwargs.get(metadata_key) or {} - if not isinstance(metadata, dict): - continue - deployment_model_info = metadata.get("model_info") or {} - if not isinstance(deployment_model_info, dict): - continue - deployment_id = deployment_model_info.get("id") - if deployment_id is not None: - return str(deployment_id) - return None - - def _record_agentic_loop_params( logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool ) -> None: @@ -1475,53 +1445,6 @@ def _record_agentic_loop_params( ) -def _register_custom_pricing_for_request( - model: str, - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None, -) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) - return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - - def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: return ctx.metadata @@ -5337,10 +5260,7 @@ def _complete_custom_providers( stream: Final = ctx.stream timeout: Final = ctx.timeout - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -5786,9 +5706,9 @@ def completion( ### CUSTOM MODEL COST ### input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) + cost_per_second: Final = kwargs.get("cost_per_second", None) ### CUSTOM PROMPT TEMPLATE ### initial_prompt_value: Final = kwargs.get("initial_prompt_value", None) roles: Final = kwargs.get("roles", None) @@ -5828,6 +5748,7 @@ def completion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, ) from litellm.types.llms.openai import AllMessageValues @@ -5839,6 +5760,7 @@ def completion( tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs api_base=base_url or kwargs.get("api_base"), + model_info=routed_model_info(cast(object, model_info)), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5984,19 +5906,7 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) + logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "chat")) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6159,7 +6069,6 @@ def completion( return resolve_timeout(stream_kind=StreamKind.NOT_STREAMED) return resolve_timeout(stream_kind=StreamKind.FAKED if fakes_stream(attempt_params) else StreamKind.LIVE) - ### TIMEOUT LOGIC ### timeout = timeout_for_optional_params(optional_params) logging.record_applied_timeout(timeout) processed_non_default_params: Final = pre_process_non_default_params( @@ -6202,6 +6111,11 @@ def completion( messages if readable_messages is None else list(readable_messages) ) + supplied_client_api_base: Final = ( + str(client.base_url) + if custom_llm_provider == "openai" and isinstance(client, openai.OpenAI | openai.AsyncOpenAI) + else None + ) # For logging - save the values of the litellm-specific params passed in requested_litellm_params: Final = get_litellm_params( acompletion=acompletion, @@ -6210,7 +6124,7 @@ def completion( logger_fn=logger_fn, verbose=verbose, custom_llm_provider=custom_llm_provider, - api_base=api_base, + api_base=supplied_client_api_base or api_base, litellm_call_id=kwargs.get("litellm_call_id", None), model_alias_map=litellm.model_alias_map, completion_call_id=id, @@ -6266,6 +6180,7 @@ def completion( *ANTHROPIC_WIF_KWARGS_KEYS, *OPENAI_WIF_KWARGS_KEYS, *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, + *AGENTIC_LOOP_CONTROL_KWARGS_KEYS, "github_copilot_auth_type", "github_copilot_host", "github_copilot_user_session", @@ -7003,12 +6918,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -7055,21 +6964,8 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - logging: Final[LiteLLMLoggingObj] = litellm_logging_obj + logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "embedding")) logging.update_environment_variables( model=model, user=user, @@ -7403,6 +7299,7 @@ def embedding( aembedding=aembedding, litellm_params=litellm_params_dict, headers=headers, + timeout=timeout, ) elif custom_llm_provider == "bedrock": if isinstance(input, str): @@ -7911,10 +7808,7 @@ def embedding( litellm_params={}, ) elif custom_llm_provider in litellm._custom_providers: - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -8506,7 +8400,10 @@ async def amoderation( @client -async def atranscription(*args, **kwargs) -> TranscriptionResponse: +async def atranscription( + *args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility + **kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility +) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: """ Calls openai + azure whisper endpoints. @@ -8550,6 +8447,14 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) + if isinstance(response, AsyncStream): + if file is not None: + calculated_duration = calculate_request_duration(file) + if calculated_duration is not None: + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration + ) + return response if not isinstance(response, TranscriptionResponse): raise ValueError( f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}" @@ -8562,9 +8467,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) - if calculated_duration is not None: - response.hidden_params["audio_transcription_duration"] = calculated_duration + sync_calculated_duration: Final = calculate_request_duration(file) + if sync_calculated_duration is not None: + response.set_audio_transcription_duration(sync_calculated_duration) return response except Exception as e: @@ -8578,6 +8483,42 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: ) +def _validate_gpt_transcription_request( + model: str, + pricing_model: str, + custom_llm_provider: str, + response_format: str | None, + drop_params: object, +) -> str | None: + model_cost_key: Final = next( + ( + key + for key in (f"{custom_llm_provider}/{pricing_model}", pricing_model) + if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider + ), + None, + ) + model_info: Final = ( + get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider) + if model_cost_key is not None + else None + ) + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + if ( + provider_specific_entry is None + or provider_specific_entry.get("transcription_json_or_text_only") != 1 + or response_format in (None, "json", "text") + ): + return response_format + if litellm.drop_params is True or normalize_drop_params(drop_params) is True: + return None + raise litellm.UnsupportedParamsError( + message=f"{model} only supports response_format 'json' or 'text'", + model=model, + llm_provider=custom_llm_provider, + ) + + @client def transcription( model: str, @@ -8597,8 +8538,15 @@ def transcription( max_retries: int | None = None, custom_llm_provider=None, base_url: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, + stream: bool | None = None, **kwargs, -) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: +) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] +): """ Calls openai + azure whisper endpoints. @@ -8636,19 +8584,36 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + litellm_params_dict: Final = get_litellm_params(**kwargs) + configured_base_model: Final = litellm_params_dict.get("base_model") + pricing_model: Final = ( + configured_base_model.removeprefix(f"{custom_llm_provider}/") + if isinstance(configured_base_model, str) + else model + ) + + supported_response_format: Final = _validate_gpt_transcription_request( + model=model, + pricing_model=pricing_model, + custom_llm_provider=custom_llm_provider, + response_format=response_format, + drop_params=kwargs.get("drop_params"), + ) + optional_params: Final = get_optional_params_transcription( model=model, language=language, + languages=languages, + keywords=keywords, prompt=prompt, - response_format=response_format, + response_format=supported_response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, + stream=stream, custom_llm_provider=custom_llm_provider, **non_default_params, ) - litellm_params_dict: Final = get_litellm_params(**kwargs) - litellm_logging_obj.update_environment_variables( model=model, user=user, @@ -8665,11 +8630,17 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] + response: ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | AsyncStream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ) provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, provider=LlmProviders(custom_llm_provider), + base_model=pricing_model, ) uses_openai_transport: Final = custom_llm_provider in OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS and not ( @@ -8680,7 +8651,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") @@ -8699,7 +8670,7 @@ def transcription( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - api_version=api_version, + api_version=azure_api_version, azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, @@ -8836,15 +8807,17 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). - if not isinstance(response, Coroutine): - existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) + if not isinstance(response, Coroutine) and ( + isinstance(response, (Stream, AsyncStream)) or getattr(response, "duration", None) is None + ): + calculated_duration: Final = calculate_request_duration(file) + if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration ) - response_hidden_params["audio_transcription_duration"] = calculated_duration + elif calculated_duration is not None: + response.set_audio_transcription_duration(calculated_duration) return response diff --git a/litellm/mcp_client/client.py b/litellm/mcp_client/client.py index 107b7155303..0461f1f06ff 100644 --- a/litellm/mcp_client/client.py +++ b/litellm/mcp_client/client.py @@ -478,6 +478,13 @@ def _notify_cleanup(callback: Callable[[], None] | None) -> None: callback() +class _UnvalidatedClientSession(ClientSession): + """Skips the SDK's output check, which lists the upstream's tools on a session with no schema cached.""" + + async def validate_tool_result(self, name: str, result: MCPCallToolResult) -> None: + return None + + class MCPClient: """ MCP Client supporting: @@ -505,12 +512,15 @@ class MCPClient: elicitation_callback: "ElicitationCallback | None" = None, logging_callback: Callable | None = None, protocol_version: MCPUpstreamProtocol = "auto", + skip_tool_output_validation: bool = False, upstream_transport: httpx2.AsyncBaseTransport | None = None, ): self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol]( MCPUpstreamProtocol ).validate_python(protocol_version) validate_mcp_protocol_transport(self.protocol_version, transport_type) + # 2026-07-28 lists tools before every call, so its output check adds no list + self.skip_tool_output_validation: Final = skip_tool_output_validation and self.protocol_version != "2026-07-28" self.server_url: str = server_url self.transport_type: MCPTransport = transport_type self.auth_type: MCPAuthType = auth_type @@ -546,7 +556,7 @@ class MCPClient: "response_hooks": _mcp_response_hooks(), } - def session_settings(self) -> tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool]: + def session_settings(self) -> tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool, bool]: callbacks: Final = self.session_callbacks() return ( self.transport_type, @@ -555,6 +565,7 @@ class MCPClient: callbacks["sampling_callback"] is not None, callbacks["elicitation_callback"] is not None, callbacks["logging_callback"] is not None, + self.skip_tool_output_validation, ) def route_session_callbacks(self, callbacks: _SessionCallbackMap) -> None: @@ -776,7 +787,10 @@ class MCPClient: ) # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else # ever fails the request. - session_ctx: Final = ClientSession( + session_type: Final = ( + _UnvalidatedClientSession if self.skip_tool_output_validation else ClientSession + ) + session_ctx: Final = session_type( read_stream, write_stream, read_timeout_seconds=self.timeout, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 032b7d17423..76fd85d001b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,172 @@ { + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -40,6 +208,7 @@ }, "1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -189,6 +358,7 @@ }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md", "input_cost_per_token": 2e-06, "litellm_provider": "bedrock", "max_input_tokens": 256000, @@ -199,6 +369,7 @@ }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, @@ -368,6 +539,7 @@ }, "amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -376,6 +548,7 @@ }, "us.amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -384,6 +557,7 @@ }, "us.writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -396,6 +570,7 @@ }, "us.writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -408,6 +583,7 @@ }, "writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -420,6 +596,7 @@ }, "writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -800,6 +977,7 @@ }, "twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -814,6 +992,7 @@ }, "us.twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -828,6 +1007,7 @@ }, "eu.twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -881,6 +1061,7 @@ }, "twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -890,6 +1071,7 @@ }, "us.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -899,6 +1081,7 @@ }, "eu.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -908,6 +1091,7 @@ }, "global.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -1011,7 +1195,8 @@ "supports_native_streaming": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": {"model":"anthropic.claude-haiku-4-5-20251001-v1:0","litellm_provider":"bedrock"} }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -1111,6 +1296,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -1939,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1971,12 +2158,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2025,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2057,12 +2247,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2111,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2143,12 +2336,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2197,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2229,12 +2425,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2278,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2310,12 +2509,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2364,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2396,7 +2598,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2760,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2804,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2848,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2892,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2936,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2980,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3233,6 +3443,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -3276,6 +3488,8 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -3510,6 +3724,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "apac.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -3565,6 +3781,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "au.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -3598,6 +3816,7 @@ }, "azure/ada": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -3608,6 +3827,7 @@ "azure/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -3635,19 +3855,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/command-r-plus": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true - }, "azure_ai/claude-haiku-4-5": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-haiku-5-5", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -3671,6 +3882,8 @@ }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-opus-5-5", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -3695,6 +3908,7 @@ }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, @@ -3726,6 +3940,7 @@ }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3757,6 +3972,7 @@ }, "azure_ai/claude-fable-5": { "deprecation_date": "2027-12-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -3785,7 +4001,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3819,14 +4035,47 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-12-05" + "deprecation_date": "2027-12-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" + }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3893,6 +4142,7 @@ }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, @@ -3925,6 +4175,8 @@ }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-sonnet-5-5", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3948,6 +4200,7 @@ }, "azure_ai/claude-sonnet-5": { "deprecation_date": "2027-06-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -3964,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -3980,6 +4234,7 @@ }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, @@ -4040,6 +4295,7 @@ }, "azure_ai/gpt-oss-120b": { "deprecation_date": "2027-09-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "litellm_provider": "azure_ai", @@ -4295,6 +4551,7 @@ }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -4342,6 +4599,7 @@ "azure_ai/gpt-chat-latest": { "cache_read_input_token_cost": 5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -4376,6 +4634,7 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", @@ -4391,6 +4650,7 @@ "azure_ai/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, @@ -4420,6 +4680,7 @@ }, "azure_ai/whisper": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_second": 0.0001, "litellm_provider": "azure_ai", "mode": "audio_transcription", @@ -4473,6 +4734,7 @@ }, "azure_ai/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -4519,6 +4781,8 @@ }, "azure_ai/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -4566,6 +4830,8 @@ }, "azure_ai/gpt-5.4-2026-03-05": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -4613,6 +4879,7 @@ }, "azure_ai/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -4658,6 +4925,7 @@ }, "azure_ai/gpt-5.4-pro-2026-03-05": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -4703,6 +4971,7 @@ }, "azure_ai/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -4744,6 +5013,7 @@ }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -4785,6 +5055,7 @@ }, "azure_ai/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -4826,6 +5097,7 @@ }, "azure_ai/gpt-5.4-nano-2026-03-17": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -4867,6 +5139,7 @@ }, "azure_ai/model_router": { "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", @@ -4878,6 +5151,7 @@ }, "azure_ai/model-router": { "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", @@ -4889,6 +5163,8 @@ }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, @@ -4908,6 +5184,8 @@ }, "azure/eu/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -4927,6 +5205,8 @@ }, "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -4947,6 +5227,7 @@ "azure/eu/gpt-4o-mini": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -4970,6 +5251,7 @@ "azure/eu/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -5008,26 +5290,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/eu/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2.2e-05, - "cache_read_input_token_cost": 2.75e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.00011, - "input_cost_per_token": 5.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.00022, - "output_cost_per_token": 2.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/eu/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_audio_token_cost": 2.5e-06, "cache_read_input_token_cost": 2.75e-06, @@ -5059,6 +5321,8 @@ "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, @@ -5102,7 +5366,9 @@ "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_priority": 4.95e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, "input_cost_per_token_priority": 4.95e-07, @@ -5144,7 +5410,9 @@ "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -5192,6 +5460,8 @@ }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2-codex", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -5223,6 +5493,7 @@ }, "azure/eu/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -5255,6 +5526,8 @@ "azure/eu/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -5296,6 +5569,7 @@ "azure/eu/o1": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -5314,6 +5588,7 @@ "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -5348,6 +5623,8 @@ "azure/eu/o3-mini": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -5365,6 +5642,8 @@ "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -5382,6 +5661,7 @@ "azure/global-standard/gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", @@ -5401,6 +5681,7 @@ "azure/global-standard/gpt-4o-2024-11-20": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", @@ -5418,6 +5699,7 @@ }, "azure/global-standard/gpt-4o-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -5437,6 +5719,8 @@ }, "azure/global/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/global/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -5456,6 +5740,8 @@ }, "azure/global/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/global/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -5474,7 +5760,8 @@ "supports_vision": true }, "azure/global/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -5522,6 +5809,7 @@ }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5553,6 +5841,7 @@ }, "azure/global/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5611,29 +5900,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-16k": { - "deprecation_date": "2025-04-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 16385, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-16k-0613": { - "deprecation_date": "2025-04-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 16385, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-instruct": { "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", @@ -5652,18 +5918,6 @@ "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" }, - "azure/gpt-4": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 3e-05, - "litellm_provider": "azure", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-4-0125-preview": { "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -5675,18 +5929,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-4-0613": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 3e-05, - "litellm_provider": "azure", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-4-1106-preview": { "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -5698,28 +5940,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-4-32k": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 6e-05, - "litellm_provider": "azure", - "max_input_tokens": 32768, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, - "azure/gpt-4-32k-0613": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 6e-05, - "litellm_provider": "azure", - "max_input_tokens": 32768, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, "azure/gpt-4-turbo": { "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -5758,6 +5978,8 @@ }, "azure/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -5800,6 +6022,8 @@ }, "azure/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -5842,6 +6066,8 @@ }, "azure/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -5884,6 +6110,8 @@ }, "azure/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -5926,6 +6154,8 @@ }, "azure/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -5959,6 +6189,8 @@ }, "azure/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -5990,28 +6222,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-4.5-preview": { - "cache_read_input_token_cost": 3.75e-05, - "deprecation_date": "2025-07-14", - "input_cost_per_token": 7.5e-05, - "input_cost_per_token_batches": 3.75e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_batches": 7.5e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6030,7 +6244,9 @@ "supports_vision": true }, "azure/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "azure", @@ -6048,6 +6264,8 @@ }, "azure/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6067,6 +6285,8 @@ }, "azure/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6086,6 +6306,7 @@ }, "azure/gpt-audio": { "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6119,6 +6340,7 @@ }, "azure/gpt-audio-2025-08-28": { "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6151,6 +6373,7 @@ }, "azure/gpt-audio-1.5": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6184,6 +6407,7 @@ }, "azure/gpt-audio-1.5-2026-02-23": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6218,6 +6442,7 @@ }, "azure/gpt-audio-mini": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "azure", @@ -6250,6 +6475,7 @@ }, "azure/gpt-audio-mini-2025-10-06": { "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "azure", @@ -6314,6 +6540,7 @@ }, "azure/gpt-4o-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -6339,6 +6566,7 @@ "azure/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", @@ -6415,6 +6643,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6424,6 +6653,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6449,6 +6681,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6458,6 +6691,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6482,6 +6718,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1-2026-07-07", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6491,6 +6729,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6516,6 +6757,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1-2026-07-07", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6525,6 +6768,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6547,8 +6793,10 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6556,6 +6804,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6575,14 +6826,18 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6590,6 +6845,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6609,6 +6867,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6617,6 +6877,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6626,6 +6887,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6650,6 +6914,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6659,6 +6924,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6680,6 +6948,7 @@ }, "azure/gpt-4o-mini-transcribe": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -6693,6 +6962,7 @@ }, "azure/gpt-4o-mini-tts": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 6e-07, "litellm_provider": "azure", "mode": "audio_speech", @@ -6710,26 +6980,6 @@ "audio" ] }, - "azure/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2e-05, - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.0001, - "input_cost_per_token": 5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.0002, - "output_cost_per_token": 2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, "input_cost_per_audio_token": 4e-05, @@ -6758,6 +7008,7 @@ }, "azure/gpt-4o-transcribe": { "deprecation_date": "2026-10-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 6e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6772,6 +7023,7 @@ }, "azure/gpt-4o-transcribe-diarize": { "deprecation_date": "2027-04-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 6e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6791,6 +7043,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -6807,7 +7062,9 @@ "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", @@ -6855,6 +7112,8 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-codex", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", @@ -6889,6 +7148,7 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", @@ -6921,6 +7181,8 @@ }, "azure/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -6967,6 +7229,8 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, @@ -7010,6 +7274,8 @@ "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1-codex", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7039,7 +7305,9 @@ "supports_vision": true }, "azure/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -7085,7 +7353,9 @@ "azure/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, @@ -7128,6 +7398,8 @@ }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_batches": 2.5e-09, "input_cost_per_token": 5e-08, @@ -7170,6 +7442,8 @@ "azure/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", @@ -7210,6 +7484,8 @@ }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-pro", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", @@ -7245,7 +7521,9 @@ "supports_vision": true }, "azure/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -7294,6 +7572,8 @@ "azure/gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -7318,6 +7598,8 @@ "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7339,6 +7621,8 @@ "azure/gpt-5.2-chat": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7359,6 +7643,8 @@ "azure/gpt-5.3-chat": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7379,6 +7665,8 @@ "azure/us/gpt-5.1-chat": { "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7400,6 +7688,8 @@ "azure/us/gpt-5.2-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7420,6 +7710,8 @@ "azure/us/gpt-5.3-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7440,6 +7732,8 @@ "azure/eu/gpt-5.1-chat": { "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7461,6 +7755,8 @@ "azure/eu/gpt-5.2-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7481,6 +7777,8 @@ "azure/eu/gpt-5.3-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7498,38 +7796,10 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "deprecation_date": "2025-07-28", - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_prompt_caching": true - }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "deprecation_date": "2025-07-28", - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_prompt_caching": true - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-codex", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -7562,6 +7832,7 @@ "azure/gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7592,6 +7863,7 @@ }, "azure/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -7623,6 +7895,8 @@ }, "azure/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4", "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, @@ -7670,6 +7944,8 @@ "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4", "input_cost_per_token": 1.75e-06, "input_cost_per_token_batches": 8.75e-07, "input_cost_per_token_priority": 3.5e-06, @@ -7714,6 +7990,8 @@ "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.3-codex", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7746,6 +8024,7 @@ "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.75e-06, "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", @@ -7859,6 +8138,8 @@ }, "azure/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, @@ -7916,6 +8197,8 @@ }, "azure/us/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.5", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, @@ -7965,6 +8248,8 @@ }, "azure/eu/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.5", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, @@ -8019,6 +8304,8 @@ "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, @@ -8074,6 +8361,8 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.5", "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, @@ -8123,6 +8412,8 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.5", "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, @@ -8169,6 +8460,8 @@ }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5-pro", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -8220,6 +8513,8 @@ "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5-pro", "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, @@ -8332,6 +8627,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-sol", "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, @@ -8395,6 +8692,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-sol", "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, @@ -8458,6 +8757,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6.1-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, @@ -8521,6 +8822,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6.1-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, @@ -8584,6 +8887,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, @@ -8647,6 +8952,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, @@ -8710,11 +9017,7 @@ "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", @@ -8746,7 +9049,11 @@ "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "input_cost_per_token_above_272k_tokens_flex": 1e-05, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "input_cost_per_token_flex": 5e-06, + "output_cost_per_token_flex": 2.5e-05, + "cache_read_input_token_cost_flex": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-06 }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, @@ -8762,11 +9069,7 @@ "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", @@ -8798,7 +9101,11 @@ "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "input_cost_per_token_above_272k_tokens_flex": 1e-05, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "input_cost_per_token_flex": 5e-06, + "output_cost_per_token_flex": 2.5e-05, + "cache_read_input_token_cost_flex": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-06 }, "azure/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, @@ -8814,12 +9121,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, - "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8846,7 +9149,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_flex": 5e-09, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07 }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, @@ -8862,12 +9173,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, - "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8894,7 +9201,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_flex": 5e-09, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07 }, "azure/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, @@ -8946,7 +9261,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6-sol-2026-09-22": { "cache_creation_input_token_cost": 2.5e-06, @@ -8998,7 +9321,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, @@ -9061,11 +9392,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", "supported_endpoints": [ "/v1/chat/completions", @@ -9093,7 +9420,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 5e-08, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, @@ -9109,11 +9444,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", "supported_endpoints": [ "/v1/chat/completions", @@ -9141,7 +9472,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 5e-08, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/us/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, @@ -9431,6 +9770,7 @@ "azure/gpt-chat-latest": { "cache_read_input_token_cost": 5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -9563,6 +9903,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6-sol", "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, @@ -9618,6 +9960,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-07, "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6.1-terra", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "input_cost_per_token_above_272k_tokens_priority": 8.8e-06, @@ -9673,6 +10017,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-08, "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6-luna", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "input_cost_per_token_above_272k_tokens_priority": 8.8e-07, @@ -9869,6 +10215,7 @@ "azure/us/gpt-chat-latest": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -9965,6 +10312,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6-sol", "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, @@ -10020,6 +10369,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-07, "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6.1-terra", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "input_cost_per_token_above_272k_tokens_priority": 8.8e-06, @@ -10075,6 +10426,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-08, "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6-luna", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "input_cost_per_token_above_272k_tokens_priority": 8.8e-07, @@ -10122,6 +10475,8 @@ }, "azure/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, @@ -10182,6 +10537,8 @@ }, "azure/us/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, @@ -10236,6 +10593,8 @@ }, "azure/eu/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, @@ -10337,6 +10696,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_batches": 2.5e-06, @@ -10347,6 +10708,8 @@ }, "azure/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1.25e-06, @@ -10450,6 +10813,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, @@ -10458,6 +10823,8 @@ }, "azure/us/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -10555,6 +10922,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, @@ -10563,6 +10932,8 @@ }, "azure/eu/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -10699,6 +11070,7 @@ }, "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -10753,6 +11125,7 @@ "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -10801,6 +11174,7 @@ }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, @@ -10851,6 +11225,7 @@ "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -10899,6 +11274,8 @@ "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-1.5", "input_cost_per_image_token": 1e-05, "input_cost_per_token": 5e-06, "litellm_provider": "azure", @@ -10933,6 +11310,7 @@ }, "azure/high/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10943,6 +11321,7 @@ }, "azure/high/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10953,6 +11332,7 @@ }, "azure/high/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10963,6 +11343,7 @@ }, "azure/low/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10973,6 +11354,7 @@ }, "azure/low/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10983,6 +11365,7 @@ }, "azure/low/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10993,6 +11376,7 @@ }, "azure/medium/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11003,6 +11387,7 @@ }, "azure/medium/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11013,6 +11398,7 @@ }, "azure/medium/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11025,6 +11411,7 @@ "cache_read_input_image_token_cost": 2.5e-07, "cache_read_input_token_cost": 2e-07, "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_image_token": 2.5e-06, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -11039,6 +11426,8 @@ "azure/gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2026-12-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-2", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11056,6 +11445,8 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-12-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-2", "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", @@ -11071,6 +11462,7 @@ "azure/gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11088,6 +11480,7 @@ "azure/gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-09-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11105,6 +11498,7 @@ "azure/gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-09-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11123,6 +11517,7 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", @@ -11138,6 +11533,7 @@ }, "azure/low/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11148,6 +11544,7 @@ }, "azure/low/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11158,6 +11555,7 @@ }, "azure/low/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11168,6 +11566,7 @@ }, "azure/medium/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11178,6 +11577,7 @@ }, "azure/medium/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11188,6 +11588,7 @@ }, "azure/medium/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11198,6 +11599,7 @@ }, "azure/high/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11208,6 +11610,7 @@ }, "azure/high/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11218,6 +11621,7 @@ }, "azure/high/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11246,6 +11650,7 @@ }, "azure/o1": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -11263,6 +11668,7 @@ "azure/o1-2024-12-17": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", @@ -11315,6 +11721,7 @@ }, "azure/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -11352,6 +11759,7 @@ }, "azure/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -11390,6 +11798,8 @@ "azure/o3-deep-research": { "cache_read_input_token_cost": 2.5e-06, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -11426,6 +11836,8 @@ }, "azure/o3-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, @@ -11445,6 +11857,8 @@ "azure/o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", @@ -11461,6 +11875,8 @@ }, "azure/o3-pro": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -11496,6 +11912,8 @@ }, "azure/o3-pro-2025-06-10": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -11531,6 +11949,8 @@ }, "azure/o4-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, @@ -11569,6 +11989,8 @@ "azure/o4-mini-2025-04-16": { "cache_read_input_token_cost": 2.75e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", @@ -11620,6 +12042,7 @@ }, "azure/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11629,6 +12052,7 @@ }, "azure/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11638,6 +12062,7 @@ }, "azure/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11683,6 +12108,8 @@ }, "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -11725,6 +12152,8 @@ }, "azure/us/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -11767,6 +12196,8 @@ }, "azure/us/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -11800,6 +12231,8 @@ }, "azure/us/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, @@ -11819,6 +12252,8 @@ }, "azure/us/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -11838,6 +12273,8 @@ }, "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -11858,6 +12295,7 @@ "azure/us/gpt-4o-mini": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -11881,6 +12319,7 @@ "azure/us/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -11919,26 +12358,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/us/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2.2e-05, - "cache_read_input_token_cost": 2.75e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.00011, - "input_cost_per_token": 5.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.00022, - "output_cost_per_token": 2.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/us/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_audio_token_cost": 2.5e-06, "cache_read_input_token_cost": 2.75e-06, @@ -11970,6 +12389,8 @@ "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, @@ -12013,7 +12434,9 @@ "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_priority": 4.95e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, "input_cost_per_token_priority": 4.95e-07, @@ -12057,6 +12480,8 @@ "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -12096,7 +12521,9 @@ "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -12144,6 +12571,8 @@ }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2-codex", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -12175,6 +12604,7 @@ }, "azure/us/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -12207,6 +12637,7 @@ "azure/us/o1": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -12225,6 +12656,7 @@ "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -12258,6 +12690,7 @@ }, "azure/us/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -12295,6 +12728,7 @@ }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -12333,6 +12767,8 @@ "azure/us/o3-mini": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12350,6 +12786,8 @@ "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12367,6 +12805,8 @@ "azure/us/o4-mini": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12392,6 +12832,8 @@ "azure/us/o4-mini-2025-04-16": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12423,6 +12865,8 @@ }, "azure_ai/Cohere-embed-v3-english": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, @@ -12434,6 +12878,8 @@ }, "azure_ai/Cohere-embed-v3-multilingual": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, @@ -12445,6 +12891,7 @@ }, "azure_ai/Codestral-2501": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 256000, @@ -12466,6 +12913,7 @@ }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 5000, "mode": "image_generation", @@ -12477,6 +12925,7 @@ }, "azure_ai/FLUX.1-Kontext-pro": { "deprecation_date": "2027-01-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 5000, "mode": "image_generation", @@ -12488,6 +12937,7 @@ }, "azure_ai/flux.2-pro": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 32000, "mode": "image_generation", @@ -12499,6 +12949,7 @@ }, "azure_ai/FLUX.2-flex": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, @@ -12549,6 +13000,7 @@ }, "azure_ai/FW-GLM-5": { "deprecation_date": "2027-07-01", + "successor_model": "azure_ai/FW-GLM-5.2", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -12565,6 +13017,7 @@ }, "azure_ai/FW-GLM-5.1": { "deprecation_date": "2027-07-01", + "successor_model": "azure_ai/FW-GLM-5.2", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -12822,6 +13275,8 @@ "/v1/images/edits" ], "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/MAI-Image-2.6", "max_input_tokens": 32000 }, "azure_ai/MAI-Image-2.5-Flash": { @@ -12837,10 +13292,13 @@ "/v1/images/edits" ], "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/MAI-Image-2.6-Flash", "max_input_tokens": 32000 }, "azure_ai/MAI-Image-2.5-Pro": { "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", @@ -12880,6 +13338,7 @@ }, "azure_ai/Llama-3.3-70B-Instruct": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -12893,6 +13352,7 @@ }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -12907,6 +13367,7 @@ }, "azure_ai/Llama-4-Scout-17B-16E-Instruct": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 10000000, @@ -12919,149 +13380,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure_ai/Meta-Llama-3-70B-Instruct": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 1.1e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 8192, - "max_output_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/Meta-Llama-3.1-70B-Instruct": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 2.68e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.54e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/Phi-3-medium-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.8e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-medium-4k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.8e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-mini-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-mini-4k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-small-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-small-8k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-MoE-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.6e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.4e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-mini-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-vision-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Phi-4": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -13076,6 +13398,8 @@ }, "azure_ai/Phi-4-mini-instruct": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13088,6 +13412,8 @@ }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -13103,6 +13429,8 @@ }, "azure_ai/Phi-4-mini-reasoning": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13115,6 +13443,8 @@ }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -13148,6 +13478,7 @@ }, "azure_ai/Cohere-parse-v5": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "mode": "ocr", "ocr_cost_per_page": 0.0015, @@ -13158,6 +13489,8 @@ }, "azure_ai/cohere-command-a": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "coherelabs-command-a-plus-05-2026-fp81", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13198,6 +13531,7 @@ }, "azure_ai/MAI-DS-R1": { "deprecation_date": "2026-02-27", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13209,30 +13543,10 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/cohere-rerank-v3-english": { - "deprecation_date": "2025-06-30", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" - }, - "azure_ai/cohere-rerank-v3-multilingual": { - "deprecation_date": "2025-06-30", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" - }, "azure_ai/cohere-rerank-v4.0-pro": { "deprecation_date": "2027-09-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "Cohere-rerank-v5-pro", "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -13244,6 +13558,8 @@ }, "azure_ai/cohere-rerank-v4.0-fast": { "deprecation_date": "2027-09-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "Cohere-rerank-v5-fast", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -13255,6 +13571,8 @@ }, "azure_ai/deepseek-v3.2": { "deprecation_date": "2027-11-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4.1-flash", "input_cost_per_token": 5.8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13271,6 +13589,8 @@ }, "azure_ai/deepseek-v3.2-speciale": { "deprecation_date": "2027-11-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-pro", "input_cost_per_token": 5.8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13285,20 +13605,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-v3": { - "deprecation_date": "2025-08-31", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13315,6 +13624,8 @@ }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/DeepSeek-V4-Flash-0731", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13332,6 +13643,7 @@ "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13347,6 +13659,8 @@ }, "azure_ai/embed-v-4-0": { "deprecation_date": "2027-06-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13366,6 +13680,8 @@ }, "azure_ai/grok-4": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4.3", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13424,6 +13740,7 @@ "azure_ai/grok-4-20-reasoning": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13443,6 +13760,7 @@ "azure_ai/grok-4-20-non-reasoning": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13460,6 +13778,7 @@ }, "azure_ai/grok-4-1-fast-non-reasoning": { "deprecation_date": "2027-05-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -13475,6 +13794,7 @@ }, "azure_ai/grok-4-1-fast-reasoning": { "deprecation_date": "2027-05-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -13495,6 +13815,7 @@ }, "azure_ai/grok-code-fast-1": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 256000, @@ -13510,6 +13831,7 @@ }, "azure_ai/jais-30b-chat": { "deprecation_date": "2026-01-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "input_cost_per_token": 0.0032, "litellm_provider": "azure_ai", "max_input_tokens": 8192, @@ -13519,20 +13841,9 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_pdf_input": false }, - "azure_ai/jamba-instruct": { - "deprecation_date": "2025-03-01", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 70000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/kimi-k2.5": { "deprecation_date": "2027-01-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -13550,6 +13861,7 @@ }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -13574,6 +13886,8 @@ }, "azure_ai/ministral-3b": { "deprecation_date": "2026-12-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "ministral-ministral-4b-instruct", "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13585,32 +13899,6 @@ "supports_pdf_input": false, "supports_tool_choice": true }, - "azure_ai/mistral-large": { - "deprecation_date": "2025-04-15", - "input_cost_per_token": 4e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/mistral-large-2407": { - "deprecation_date": "2025-05-13", - "input_cost_per_token": 2e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/mistral-large-latest": { "input_cost_per_token": 2e-06, "litellm_provider": "azure_ai", @@ -13638,6 +13926,8 @@ }, "azure_ai/mistral-medium-2505": { "deprecation_date": "2026-12-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/mistral-medium-3-5", "input_cost_per_token": 4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13651,6 +13941,8 @@ }, "azure_ai/mistral-nemo": { "deprecation_date": "2026-01-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "successor_model": "azure_ai/mistral-small-2503", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13661,21 +13953,10 @@ "supports_function_calling": true, "supports_pdf_input": false }, - "azure_ai/mistral-small": { - "deprecation_date": "2025-07-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/mistral-small-2503": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "mistralai-mistral-small-4-119b-2603", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13689,6 +13970,8 @@ }, "babbage-002": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "litellm_provider": "text-completion-openai", @@ -13853,6 +14136,7 @@ }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -13897,6 +14181,7 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.3e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -13949,6 +14234,7 @@ }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -13982,6 +14268,7 @@ }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -13992,6 +14279,7 @@ }, "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14024,6 +14312,7 @@ }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14068,6 +14357,7 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.1e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -14160,6 +14450,7 @@ }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", @@ -14230,6 +14521,7 @@ }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14240,6 +14532,7 @@ }, "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14272,6 +14565,7 @@ }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14415,6 +14709,7 @@ }, "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14476,6 +14771,7 @@ }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14486,6 +14782,7 @@ }, "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14496,6 +14793,7 @@ }, "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14557,6 +14855,7 @@ }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14567,6 +14866,7 @@ }, "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14577,6 +14877,7 @@ }, "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4.7e-07, "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, @@ -14664,6 +14965,7 @@ }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -14685,6 +14987,7 @@ }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 5.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -14696,6 +14999,7 @@ }, "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", @@ -14766,6 +15070,7 @@ }, "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 4.45e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14776,6 +15081,7 @@ }, "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14808,6 +15114,7 @@ }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14852,6 +15159,7 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.3e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -14983,6 +15291,7 @@ }, "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14993,6 +15302,7 @@ }, "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15003,6 +15313,7 @@ }, "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15024,6 +15335,7 @@ }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15057,6 +15369,7 @@ }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15100,6 +15413,7 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15178,6 +15492,7 @@ }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15222,6 +15537,7 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15296,7 +15612,7 @@ "output_vector_size": 1536 }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0": { - "input_cost_per_token": 2e-07, + "input_cost_per_token": 3e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, "mode": "embedding", @@ -15332,6 +15648,8 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15362,6 +15680,7 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15381,7 +15700,8 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -15409,8 +15729,10 @@ "input_cost_per_image": 7.2e-05, "input_cost_per_video_per_second": 0.00084, "input_cost_per_audio_per_second": 0.000168, + "input_cost_per_token_batches": 8.1e-08, "output_cost_per_token": 0.0, "output_vector_size": 3072, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_embedding_image_input": true, "supports_image_input": true, "supports_video_input": true, @@ -15419,11 +15741,14 @@ "bedrock/us-gov-west-1/amazon.nova-lite-v1:0": { "cache_read_input_token_cost": 1.8e-08, "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -15434,11 +15759,14 @@ "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.05e-08, "input_cost_per_token": 4.2e-08, + "input_cost_per_token_batches": 2.1e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, + "output_cost_per_token_batches": 8.4e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -15446,12 +15774,21 @@ }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, + "cache_read_input_token_cost_flex": 1.2e-07, + "cache_read_input_token_cost_priority": 4.2e-07, "input_cost_per_token": 9.6e-07, + "input_cost_per_token_batches": 4.8e-07, + "input_cost_per_token_flex": 4.8e-07, + "input_cost_per_token_priority": 1.68e-06, "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, + "output_cost_per_token_batches": 1.92e-06, + "output_cost_per_token_flex": 1.92e-06, + "output_cost_per_token_priority": 6.72e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -15504,6 +15841,8 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15534,6 +15873,7 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15553,10 +15893,12 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15568,6 +15910,7 @@ }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15579,6 +15922,7 @@ }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15589,6 +15933,7 @@ }, "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15678,6 +16023,7 @@ }, "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15699,6 +16045,7 @@ }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15732,6 +16079,7 @@ }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15775,6 +16123,7 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15947,17 +16296,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "cerebras/llama3.1-70b": { - "deprecation_date": "2025-01-17", - "input_cost_per_token": 6e-07, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true - }, "cerebras/llama3.1-8b": { "deprecation_date": "2026-05-27", "input_cost_per_token": 1e-07, @@ -16031,6 +16369,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 0.0001, "source": "https://developers.openai.com/api/docs/pricing" }, @@ -16096,6 +16436,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", + "successor_model": "claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -16129,6 +16471,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", + "successor_model": "claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -16177,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -16297,6 +16642,7 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "litellm_provider": "bedrock", "supports_tool_search": true, "max_input_tokens": 200000, @@ -16316,7 +16662,8 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "claude-opus-4-5-20251101": { "supports_web_search": true, @@ -16710,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17508,12 +17856,15 @@ }, "computer-use-preview": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/responses" ], @@ -17537,12 +17888,15 @@ }, "computer-use-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/responses" ], @@ -17567,6 +17921,7 @@ "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, @@ -17588,6 +17943,7 @@ "deepseek-reasoner": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, @@ -21570,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -21618,6 +21975,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", + "successor_model": "databricks/databricks-claude-sonnet-5", "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -21727,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -21741,6 +22100,7 @@ "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-11-05", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21754,6 +22114,7 @@ "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "supported_modalities": [ "text" ], @@ -21767,14 +22128,19 @@ "supports_vision": false }, "databricks/system.ai.deepseek-v4-flash-0731": { + "deprecation_date": "2026-11-05", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", "litellm_provider": "databricks", "mode": "chat", + "source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "token_count_framing": 97 }, "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21787,6 +22153,7 @@ "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "supported_modalities": [ "text" ], @@ -21803,6 +22170,7 @@ "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.0002e-08, "deprecation_date": "2026-10-02", + "successor_model": "databricks/databricks-gemini-3-5-flash", "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, "litellm_provider": "databricks", @@ -22072,7 +22440,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22090,7 +22458,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22104,7 +22474,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22122,7 +22492,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22138,7 +22510,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22157,7 +22529,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22173,7 +22547,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22191,7 +22565,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22207,7 +22583,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22227,7 +22603,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -22841,6 +23219,7 @@ "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22852,6 +23231,7 @@ "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-glm-5-3", "supported_modalities": [ "text", "image" @@ -23076,6 +23456,8 @@ }, "davinci-002": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "text-completion-openai", @@ -23970,7 +24352,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24044,7 +24426,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24417,6 +24799,7 @@ "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -24469,6 +24852,7 @@ "deepseek/deepseek-reasoner": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -24522,6 +24906,7 @@ }, "deepseek.v3-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -25331,6 +25716,7 @@ "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", + "successor_model": "eu.anthropic.claude-haiku-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -25448,6 +25834,7 @@ "supports_vision": true, "prompt_cache_min_tokens": 1024, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "token_count_model": { "model": "anthropic.claude-opus-4-1-20250805-v1:0", "litellm_provider": "bedrock" @@ -25482,6 +25869,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "eu.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -25525,6 +25914,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "eu.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -25558,6 +25949,7 @@ }, "eu.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -25570,6 +25962,7 @@ }, "eu.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -28149,6 +28542,7 @@ "cache_read_input_token_cost": 1.4e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/accounts/fireworks/models/glm-5p3", "input_cost_per_token": 1.4e-06, "input_cost_per_token_priority": 1.75e-06, "litellm_provider": "fireworks_ai", @@ -28524,6 +28918,7 @@ "cache_read_input_token_cost": 1.4e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/glm-5p3", "input_cost_per_token": 1.4e-06, "input_cost_per_token_priority": 1.75e-06, "litellm_provider": "fireworks_ai", @@ -28907,6 +29302,8 @@ }, "ft:babbage-002": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.6e-06, "input_cost_per_token_batches": 8e-07, "litellm_provider": "text-completion-openai", @@ -28919,6 +29316,8 @@ }, "ft:davinci-002": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.2e-05, "input_cost_per_token_batches": 6e-06, "litellm_provider": "text-completion-openai", @@ -28931,6 +29330,8 @@ }, "ft:gpt-3.5-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -28938,6 +29339,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_system_messages": true, @@ -28945,45 +29347,57 @@ }, "ft:gpt-3.5-turbo-0125": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-3.5-turbo-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "ft:gpt-3.5-turbo", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-3.5-turbo-1106": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-4-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", "supports_function_calling": true, "supports_system_messages": true, @@ -28993,6 +29407,8 @@ "cache_read_input_token_cost": 1.875e-06, "cache_read_input_token_cost_batches": 9e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 2.225e-06, "litellm_provider": "openai", @@ -29000,6 +29416,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.25e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29014,12 +29431,15 @@ "ft:gpt-4o-2024-11-20": { "cache_creation_input_token_cost": 1.875e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -29032,6 +29452,8 @@ "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "litellm_provider": "openai", @@ -29039,6 +29461,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29053,6 +29476,8 @@ "cache_read_input_token_cost": 7.5e-07, "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -29060,6 +29485,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29073,6 +29499,8 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 8e-07, "input_cost_per_token_batches": 4e-07, "litellm_provider": "openai", @@ -29080,6 +29508,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.6e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29093,6 +29522,8 @@ "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 2.5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", @@ -29100,6 +29531,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29113,6 +29545,8 @@ "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 4e-06, "input_cost_per_token_batches": 2e-06, "litellm_provider": "openai", @@ -29120,6 +29554,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29131,19 +29566,38 @@ }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, - "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-10-20", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 3e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, + "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":1500,"period":"day","shared_group":"gemini-2-flash"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29160,6 +29614,7 @@ ], "supports_audio_output": false, "supports_function_calling": true, + "supports_image_size": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29169,26 +29624,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_token_batches": 1.5e-07, - "input_cost_per_token_flex": 1.5e-07, - "input_cost_per_token_priority": 5.4e-07, - "input_cost_per_audio_token_priority": 1.8e-06, - "output_cost_per_token_batches": 1.25e-06, - "output_cost_per_token_flex": 1.25e-06, - "output_cost_per_token_priority": 4.5e-06, - "supports_image_size": false + "supports_web_search": true }, "gemini-2.5-flash-image": { "deprecation_date": "2027-03-15", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -29238,12 +29679,13 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -29258,6 +29700,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -29312,6 +29755,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -29352,6 +29796,7 @@ "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29364,6 +29809,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -29410,6 +29856,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -29458,6 +29905,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -29538,17 +29986,28 @@ }, "gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_date": "2027-05-07", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "deprecation_date": "2027-05-07", + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_above_200k_tokens": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 9e-07, + "input_cost_per_audio_token_batches": 2.5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, + "input_cost_per_audio_token_priority": 9e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, - "input_cost_per_audio_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -29558,7 +30017,10 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.5-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29576,6 +30038,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29587,19 +30050,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, @@ -29695,19 +30150,38 @@ }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, - "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-10-20", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_above_200k_tokens": 3e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-08, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-08, + "input_cost_per_audio_token_above_200k_tokens_priority": 5.4e-07, + "input_cost_per_audio_token_batches": 5e-08, + "input_cost_per_audio_token_flex": 5e-08, + "input_cost_per_audio_token_priority": 5.4e-07, "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_flex": 2e-07, + "output_cost_per_token_priority": 7.2e-07, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":1500,"period":"day","shared_group":"gemini-2-flash"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29724,6 +30198,7 @@ ], "supports_audio_output": false, "supports_function_calling": true, + "supports_image_size": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29733,23 +30208,7 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token_batches": 5e-08, - "input_cost_per_token_batches": 5e-08, - "input_cost_per_token_flex": 5e-08, - "input_cost_per_token_priority": 1.8e-07, - "input_cost_per_audio_token_priority": 5.4e-07, - "output_cost_per_token_batches": 2e-07, - "output_cost_per_token_flex": 2e-07, - "output_cost_per_token_priority": 7.2e-07, - "supports_image_size": false + "supports_web_search": true }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -29845,6 +30304,8 @@ }, "gemini-live-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -29978,6 +30439,8 @@ }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -30026,11 +30489,13 @@ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, + "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, @@ -30046,6 +30511,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30058,6 +30524,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30205,11 +30672,17 @@ }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 1e-06, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -30224,7 +30697,7 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30258,6 +30731,8 @@ "vertex_ai/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, @@ -30369,6 +30844,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30431,6 +30908,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2027-01-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30462,7 +30941,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30592,6 +31071,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30604,6 +31084,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30842,6 +31323,8 @@ }, "gemini-embedding-001": { "deprecation_date": "2028-05-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-embedding-2", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -30866,6 +31349,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_embedding_image_input": true, "uses_embed_content": true }, "gemini-embedding-2": { @@ -30883,6 +31367,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30901,6 +31386,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30919,6 +31405,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30936,6 +31423,8 @@ }, "gemini/gemini-embedding-001": { "deprecation_date": "2028-05-14", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-embedding-2", "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, @@ -30964,12 +31453,15 @@ "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_embedding_image_input": true, "supports_multimodal": true, "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-3-pro-image-preview": { "deprecation_date": "2026-06-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3-pro-image", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -31018,6 +31510,8 @@ }, "gemini/gemini-3.1-flash-image-preview": { "deprecation_date": "2026-06-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-nano-banana-2.1", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -31066,6 +31560,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-05-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.1-flash-lite", "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -31123,6 +31619,8 @@ "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-embedding-2-preview": { + "deprecation_date": "2026-08-10", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, @@ -31138,7 +31636,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "successor_model": "gemini/gemini-embedding-2", "supports_audio_input": true, + "supports_embedding_image_input": true, "supports_multimodal": true, "supports_vision": true, "tpm": 10000000 @@ -31224,6 +31724,9 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.5-flash-lite", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -31283,6 +31786,8 @@ "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, "deprecation_date": "2027-03-15", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -31510,6 +32015,8 @@ }, "gemini/deep-research-pro-preview-12-2025": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/deep-research-preview-04-2026", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -31727,6 +32234,8 @@ }, "gemini/gemini-2.5-flash-preview-tts": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -31807,11 +32316,16 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "deprecation_date": "2026-07-28", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash", "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "gemini", @@ -31850,6 +32364,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.5-flash-lite", "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, @@ -31910,7 +32426,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -32315,7 +32831,9 @@ "supports_video_input": true, "supports_vision": true, "tpm": 800000, - "deprecation_date": "2026-10-22" + "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash" }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -32451,11 +32969,17 @@ }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 1e-06, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -32471,7 +32995,7 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32507,13 +33031,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32531,11 +33055,14 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, @@ -32647,6 +33174,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -32709,6 +33238,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2027-01-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -32739,7 +33270,7 @@ "output_cost_per_token_priority": 6.75e-06, "prompt_cache_min_tokens": 4096.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32827,6 +33358,8 @@ "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -33079,6 +33612,8 @@ }, "gemini/veo-3.1-fast-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33095,6 +33630,8 @@ }, "gemini/veo-3.1-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33110,6 +33647,8 @@ }, "gemini/veo-3.1-lite-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33445,6 +33984,7 @@ }, "github_copilot/gpt-4.1": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -33457,6 +33997,7 @@ }, "github_copilot/gpt-4.1-2025-04-14": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -33558,6 +34099,7 @@ "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "deprecation_date": "2026-10-19", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "litellm_provider": "github_copilot", @@ -33600,6 +34142,7 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "deprecation_date": "2026-10-19", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "litellm_provider": "github_copilot", @@ -34453,6 +34996,7 @@ }, "google.gemma-3-12b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 9e-08, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, @@ -34474,6 +35018,7 @@ }, "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.3e-07, "input_cost_per_token_batches": 1.2e-07, "input_cost_per_token_flex": 1.2e-07, @@ -34495,6 +35040,7 @@ }, "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-08, "input_cost_per_token_batches": 2e-08, "input_cost_per_token_flex": 2e-08, @@ -34529,6 +35075,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "global.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -34564,6 +35112,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "global.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -34652,12 +35202,15 @@ }, "gpt-3.5-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 5e-07, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -34666,6 +35219,8 @@ }, "gpt-3.5-turbo-0125": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "openai", @@ -34673,6 +35228,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 7.5e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34683,6 +35239,8 @@ }, "gpt-3.5-turbo-1106": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "openai", @@ -34690,6 +35248,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34700,18 +35259,22 @@ }, "gpt-3.5-turbo-16k": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, + "fedramp_uplift_multiplier": 1.1, "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -34730,12 +35293,15 @@ }, "gpt-4": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_prompt_caching": true, "supports_system_messages": true, @@ -34743,6 +35309,8 @@ }, "gpt-4-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "input_cost_per_token_batches": 1.5e-05, "litellm_provider": "openai", @@ -34750,6 +35318,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34759,12 +35328,15 @@ }, "gpt-4-1106-preview": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -34773,12 +35345,15 @@ }, "gpt-4-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -34789,6 +35364,8 @@ }, "gpt-4-turbo-2024-04-09": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, "litellm_provider": "openai", @@ -34796,6 +35373,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34810,6 +35388,8 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.5e-06, @@ -34818,6 +35398,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_priority": 1.4e-05, "search_context_cost_per_query": { @@ -34853,6 +35434,8 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-06, "input_cost_per_token_priority": 3.5e-06, "input_cost_per_token_batches": 1e-06, @@ -34861,6 +35444,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.4e-05, "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { @@ -34896,6 +35480,8 @@ "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_priority": 7e-07, @@ -34904,6 +35490,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8e-07, "output_cost_per_token_priority": 2.8e-06, "search_context_cost_per_query": { @@ -34939,6 +35526,8 @@ "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 4e-07, "input_cost_per_token_priority": 7e-07, "input_cost_per_token_batches": 2e-07, @@ -34947,6 +35536,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-06, "output_cost_per_token_batches": 8e-07, "search_context_cost_per_query": { @@ -34982,6 +35572,8 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_priority": 2e-07, @@ -34990,6 +35582,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 8e-07, "source": "https://developers.openai.com/api/docs/pricing", @@ -35019,6 +35612,8 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 1e-07, "input_cost_per_token_priority": 2e-07, "input_cost_per_token_batches": 5e-08, @@ -35027,6 +35622,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 8e-07, "output_cost_per_token_batches": 2e-07, "source": "https://developers.openai.com/api/docs/pricing", @@ -35056,6 +35652,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 4.25e-06, @@ -35064,6 +35662,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.7e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35078,6 +35677,8 @@ }, "gpt-4o-2024-05-13": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_priority": 8.75e-06, @@ -35086,6 +35687,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 2.625e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35101,6 +35703,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_priority": 4.25e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35109,6 +35713,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.7e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -35125,6 +35730,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_priority": 4.25e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35133,6 +35740,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.7e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -35147,6 +35755,8 @@ }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35164,6 +35774,8 @@ }, "gpt-4o-audio-preview-2025-06-03": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35181,6 +35793,8 @@ }, "gpt-audio": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35251,6 +35865,8 @@ }, "gpt-audio-2025-08-28": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35288,6 +35904,8 @@ }, "gpt-audio-mini": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35325,6 +35943,8 @@ }, "gpt-audio-mini-2025-10-06": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35395,12 +36015,16 @@ "supports_tool_choice": true, "supports_vision": false, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.5e-07, @@ -35409,6 +36033,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1e-06, "supports_function_calling": true, @@ -35430,6 +36055,8 @@ "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -35438,6 +36065,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { @@ -35457,6 +36085,7 @@ }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -35476,6 +36105,7 @@ "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35500,6 +36130,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35518,6 +36149,8 @@ }, "gpt-4o-mini-search-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -35526,6 +36159,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35555,11 +36189,15 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-4o-mini-tts": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -35580,6 +36218,8 @@ }, "gpt-4o-mini-tts-2025-03-20": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -35607,6 +36247,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35625,6 +36266,8 @@ }, "gpt-4o-search-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35633,6 +36276,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35662,6 +36306,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 0.0001, "source": "https://developers.openai.com/api/docs/pricing" }, @@ -35670,6 +36316,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -35678,6 +36326,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -35691,6 +36340,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -35699,6 +36350,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -35718,6 +36370,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -35753,6 +36406,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -35788,6 +36442,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -35814,6 +36469,8 @@ }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35826,6 +36483,8 @@ }, "low/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35838,6 +36497,8 @@ }, "low/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35850,6 +36511,8 @@ }, "medium/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -35862,6 +36525,8 @@ }, "medium/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -35874,6 +36539,8 @@ }, "medium/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -35886,6 +36553,8 @@ }, "high/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -35898,6 +36567,8 @@ }, "high/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -35910,6 +36581,8 @@ }, "high/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -35922,6 +36595,8 @@ }, "standard/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35934,6 +36609,8 @@ }, "standard/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35946,6 +36623,8 @@ }, "standard/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35958,6 +36637,8 @@ }, "1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35970,6 +36651,8 @@ }, "1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35982,6 +36665,8 @@ }, "1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36002,7 +36687,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "low/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36014,7 +36701,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "low/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36026,7 +36715,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.034, @@ -36038,7 +36729,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36050,7 +36743,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36062,7 +36757,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.133, @@ -36074,7 +36771,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36086,7 +36785,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36098,7 +36799,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -36110,7 +36813,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36122,7 +36827,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36134,7 +36841,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -36146,7 +36855,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36158,7 +36869,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36170,7 +36883,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, @@ -36178,6 +36893,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -36187,6 +36904,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, @@ -36233,12 +36951,15 @@ "gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", "supported_endpoints": [ "/v1/responses" @@ -36273,6 +36994,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2e-05, "reasoning_contexts": [ "auto", @@ -36318,17 +37040,22 @@ "source": "https://developers.openai.com/api/docs/pricing", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol" }, "gpt-5.1-codex-mini": { "cache_read_input_token_cost": 2.5e-08, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", "supported_endpoints": [ "/v1/responses" @@ -36353,12 +37080,15 @@ "gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", "supported_endpoints": [ "/v1/responses" @@ -36384,12 +37114,15 @@ "gpt-5.1-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", "supported_endpoints": [ "/v1/responses" @@ -36415,12 +37148,15 @@ "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -36449,6 +37185,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", @@ -36456,6 +37194,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2e-05, "reasoning_contexts": [ "auto", @@ -36514,6 +37253,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -36565,12 +37305,15 @@ "gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", "supported_endpoints": [ "/v1/responses" @@ -36596,12 +37339,15 @@ "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-08-10", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -36636,6 +37382,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -36692,6 +37439,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8.4e-05, "reasoning_contexts": [ "auto", @@ -36740,6 +37488,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8.4e-05, "reasoning_contexts": [ "auto", @@ -36846,6 +37595,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -36939,6 +37689,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37040,6 +37791,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37132,6 +37884,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37222,6 +37975,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37314,6 +38068,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37405,6 +38160,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37449,10 +38205,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/chat/completions", @@ -37487,10 +38244,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/chat/completions", @@ -37525,10 +38283,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/responses" @@ -37566,6 +38325,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/chat/completions", @@ -37604,6 +38364,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/responses" @@ -37637,6 +38398,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "reasoning_effort_levels": [ "medium" ], @@ -37705,6 +38467,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37775,6 +38538,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37837,6 +38601,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37896,6 +38661,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37963,6 +38729,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38028,6 +38795,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38085,6 +38853,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38145,6 +38914,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38207,6 +38977,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38269,6 +39040,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38328,6 +39100,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38361,13 +39134,17 @@ "source": "https://developers.openai.com/api/docs/pricing", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna" }, "gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, @@ -38388,6 +39165,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38424,6 +39202,8 @@ }, "gpt-5-pro": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -38431,6 +39211,7 @@ "max_output_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-05, "reasoning_contexts": [ "auto", @@ -38473,6 +39254,8 @@ }, "gpt-5-pro-2025-10-06": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -38480,6 +39263,7 @@ "max_output_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-05, "reasoning_contexts": [ "auto", @@ -38526,6 +39310,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -38535,6 +39321,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, @@ -38581,12 +39368,15 @@ "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.2-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38616,12 +39406,15 @@ "gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -38654,6 +39447,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -38693,17 +39487,22 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol" }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-08-10", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -38732,6 +39531,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -38741,6 +39542,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, @@ -38790,6 +39592,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -38799,6 +39603,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, @@ -38847,6 +39652,8 @@ "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -38855,6 +39662,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "reasoning_contexts": [ @@ -38902,6 +39710,8 @@ "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -38910,6 +39720,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "reasoning_contexts": [ @@ -38957,6 +39768,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 1e-05, "input_cost_per_image_token_batches": 5e-06, "input_cost_per_token": 5e-06, @@ -38975,6 +39788,8 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 2.5e-06, "input_cost_per_image_token_batches": 1.25e-06, "input_cost_per_token": 2e-06, @@ -38982,6 +39797,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 8e-06, + "output_cost_per_image_token_batches": 4e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -38994,6 +39810,8 @@ "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -39069,6 +39887,9 @@ "max_input_tokens": 128000, "max_output_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -39124,6 +39945,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39160,6 +39983,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39169,6 +39994,8 @@ "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -39204,6 +40031,8 @@ "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -39527,16 +40356,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-guard-3-8b": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://console.groq.com/docs/model/llama-guard-3-8b" - }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, "litellm_provider": "groq", @@ -39683,6 +40502,8 @@ }, "high/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -39694,6 +40515,8 @@ }, "high/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -39705,6 +40528,8 @@ }, "high/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -40082,6 +40907,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "jp.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -41244,6 +42071,8 @@ }, "low/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -41255,6 +42084,8 @@ }, "low/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -41266,6 +42097,8 @@ }, "low/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -41289,6 +42122,8 @@ }, "medium/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41300,6 +42135,8 @@ }, "medium/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41311,6 +42148,8 @@ }, "medium/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41322,6 +42161,8 @@ }, "low/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -41331,6 +42172,8 @@ }, "low/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -41340,6 +42183,8 @@ }, "low/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -41349,6 +42194,8 @@ }, "medium/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -41358,6 +42205,8 @@ }, "medium/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -41367,6 +42216,8 @@ }, "medium/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -41414,6 +42265,7 @@ }, "meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41426,6 +42278,7 @@ }, "meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock", @@ -41440,6 +42293,7 @@ }, "meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "bedrock", @@ -41454,6 +42308,7 @@ }, "meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41467,6 +42322,7 @@ }, "meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41479,6 +42335,7 @@ }, "meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41491,6 +42348,7 @@ }, "meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41504,6 +42362,7 @@ }, "meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock_converse", @@ -41518,6 +42377,7 @@ }, "meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -41528,6 +42388,7 @@ }, "meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -41538,6 +42399,7 @@ }, "meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md", "input_cost_per_token": 2.4e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "bedrock_converse", @@ -41560,6 +42422,7 @@ }, "meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md", "input_cost_per_token": 1.7e-07, "input_cost_per_token_batches": 8.5e-08, "litellm_provider": "bedrock_converse", @@ -41902,6 +42765,7 @@ }, "minimax.minimax-m2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -41924,6 +42788,7 @@ }, "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -42144,6 +43009,7 @@ }, "mistral.devstral-2-123b": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_flex": 2e-07, @@ -42250,6 +43116,7 @@ }, "mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -42315,6 +43182,7 @@ }, "mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -42326,6 +43194,7 @@ }, "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-08, "input_cost_per_token_batches": 2e-08, "input_cost_per_token_flex": 2e-08, @@ -42345,6 +43214,7 @@ }, "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, @@ -42364,6 +43234,7 @@ }, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4.12e-08, "input_cost_per_token_batches": 2.06e-08, "input_cost_per_token_flex": 2.06e-08, @@ -42383,6 +43254,7 @@ }, "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.03e-07, "input_cost_per_token_batches": 5.15e-08, "input_cost_per_token_flex": 5.15e-08, @@ -42402,6 +43274,7 @@ }, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -42421,6 +43294,7 @@ }, "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.2e-07, "input_cost_per_token_batches": 6e-08, "input_cost_per_token_flex": 6e-08, @@ -42440,6 +43314,7 @@ }, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -42459,6 +43334,7 @@ }, "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "input_cost_per_token_batches": 8e-08, "input_cost_per_token_flex": 8e-08, @@ -42851,6 +43727,7 @@ "mistral/zai-glm-5-2": { "cache_read_input_token_cost": 1.4e-07, "deprecation_date": "2026-10-31", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/models_overview", "input_cost_per_token": 1.4e-06, "litellm_provider": "mistral", "max_input_tokens": 1048576, @@ -43106,18 +43983,18 @@ "mode": "embedding" }, "mistral/mistral-large-4": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43127,18 +44004,18 @@ "supports_vision": true }, "mistral/mistral-large-4-0": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43426,6 +44303,7 @@ }, "moonshot.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_priority": 1.05e-06, @@ -43828,10 +44706,12 @@ "text", "image", "video" - ] + ], + "supports_embedding_image_input": true }, "multimodalembedding@001": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -43851,7 +44731,8 @@ "text", "image", "video" - ] + ], + "supports_embedding_image_input": true }, "nscale/Qwen/QwQ-32B": { "input_cost_per_token": 1.8e-07, @@ -44059,7 +44940,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44285,7 +45167,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44293,7 +45175,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44304,29 +45187,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44349,7 +45235,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44361,7 +45248,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44373,10 +45261,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44385,7 +45274,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44396,7 +45286,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44440,7 +45331,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44463,7 +45355,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44474,7 +45367,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44485,7 +45379,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44496,7 +45391,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44506,7 +45402,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44516,7 +45413,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44526,7 +45424,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44548,7 +45447,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44556,9 +45456,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44569,7 +45470,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44580,21 +45482,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44602,8 +45506,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -44639,6 +45544,7 @@ }, "nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, @@ -44660,6 +45566,7 @@ }, "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -44726,6 +45633,7 @@ "o1": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-10-23", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -44733,6 +45641,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -44748,6 +45657,8 @@ "o1-2024-12-17": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -44755,6 +45666,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -44769,6 +45681,8 @@ }, "o1-pro": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 0.00015, "input_cost_per_token_batches": 7.5e-05, "litellm_provider": "openai", @@ -44776,6 +45690,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 0.0003, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -44802,6 +45717,8 @@ }, "o1-pro-2025-03-19": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 0.00015, "input_cost_per_token_batches": 7.5e-05, "litellm_provider": "openai", @@ -44809,6 +45726,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 0.0003, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -44838,6 +45756,7 @@ "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -44847,6 +45766,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_flex": 4e-06, "output_cost_per_token_priority": 1.4e-05, @@ -44894,6 +45814,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", "supported_endpoints": [ "/v1/batch", @@ -44915,10 +45836,14 @@ "supports_tool_choice": false, "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-07-23" + "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol" }, "o3-deep-research-2025-06-26": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "openai", @@ -44926,6 +45851,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", "supported_endpoints": [ "/v1/batch", @@ -44953,6 +45879,8 @@ "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -44962,6 +45890,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_flex": 4e-06, "output_cost_per_token_priority": 1.4e-05, @@ -45004,6 +45933,8 @@ "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "openai", @@ -45011,6 +45942,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -45024,6 +45956,8 @@ "o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "openai", @@ -45031,6 +45965,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -45043,6 +45978,8 @@ }, "o3-pro": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -45050,6 +45987,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-05, "reasoning_contexts": [ "auto", @@ -45087,6 +46025,8 @@ }, "o3-pro-2025-06-10": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -45094,6 +46034,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-05, "reasoning_contexts": [ "auto", @@ -45134,6 +46075,8 @@ "cache_read_input_token_cost_flex": 1.38e-07, "cache_read_input_token_cost_priority": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "input_cost_per_token_flex": 5.5e-07, @@ -45143,6 +46086,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "output_cost_per_token_flex": 2.2e-06, "output_cost_per_token_priority": 8e-06, @@ -45170,6 +46114,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", "supported_endpoints": [ "/v1/batch", @@ -45191,10 +46136,14 @@ "supports_tool_choice": false, "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-07-23" + "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol" }, "o4-mini-deep-research-2025-06-26": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openai", @@ -45202,6 +46151,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", "supported_endpoints": [ "/v1/batch", @@ -45229,6 +46179,8 @@ "cache_read_input_token_cost_flex": 1.38e-07, "cache_read_input_token_cost_priority": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "input_cost_per_token_flex": 5.5e-07, @@ -45238,6 +46190,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "output_cost_per_token_flex": 2.2e-06, "output_cost_per_token_priority": 8e-06, @@ -45527,6 +46480,7 @@ "mode": "embedding", "output_vector_size": 1024, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", + "supports_embedding_image_input": true, "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -46412,6 +47366,19 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/decision": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", @@ -47029,6 +47996,7 @@ }, "qwen.qwen3-coder-480b-a35b-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -47060,6 +48028,7 @@ }, "qwen.qwen3-coder-30b-a3b-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.625e-07, @@ -48639,6 +49608,7 @@ }, "text-embedding-004": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48650,6 +49620,7 @@ }, "text-embedding-005": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48720,6 +49691,7 @@ }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48834,6 +49806,7 @@ }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -48841,6 +49814,7 @@ "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/meta-models/Muse-Glimmer-30B", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -48900,6 +49874,7 @@ "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-11-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -48907,6 +49882,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/MiniMaxAI/MiniMax-M3.1-Flash", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -48983,6 +49959,7 @@ "together_ai/Qwen/Qwen3.8-2.4T-A95B": { "cache_read_input_token_cost": 2.5e-07, "deprecation_date": "2026-11-04", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, @@ -48990,6 +49967,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/Qwen/Qwen3.7-Max", "supports_prompt_caching": true }, "together_ai/arize-ai/qwen-2-1.5b-instruct": { @@ -49004,6 +49982,7 @@ "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-10-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -49011,6 +49990,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -49095,6 +50075,7 @@ "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -49102,6 +50083,7 @@ "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/meta-models/Muse-Spark-Open-816B-A42B", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -49159,6 +50141,8 @@ }, "tts-1": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_character": 1.5e-05, "litellm_provider": "openai", "mode": "audio_speech", @@ -49169,6 +50153,8 @@ }, "tts-1-hd": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_character": 3e-05, "litellm_provider": "openai", "mode": "audio_speech", @@ -49399,6 +50385,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -49437,6 +50424,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "us.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -49479,6 +50468,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "us-gov.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -49519,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -49594,8 +50586,9 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -49616,7 +50609,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -49715,6 +50710,7 @@ }, "us-gov.nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49730,6 +50726,7 @@ }, "us-gov.nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49985,6 +50982,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "us.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -50019,6 +51018,7 @@ }, "us.deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "input_cost_per_token": 1.35e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -50071,6 +51071,7 @@ }, "us.meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50083,6 +51084,7 @@ }, "us.meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock", @@ -50097,6 +51099,7 @@ }, "us.meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "bedrock", @@ -50111,6 +51114,7 @@ }, "us.meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50124,6 +51128,7 @@ }, "us.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50136,6 +51141,7 @@ }, "us.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50148,6 +51154,7 @@ }, "us.meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50161,6 +51168,7 @@ }, "us.meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock_converse", @@ -50175,6 +51183,7 @@ }, "us.meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md", "input_cost_per_token": 2.4e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "bedrock_converse", @@ -50197,6 +51206,7 @@ }, "us.meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md", "input_cost_per_token": 1.7e-07, "input_cost_per_token_batches": 8.5e-08, "litellm_provider": "bedrock_converse", @@ -51457,6 +52467,7 @@ }, "vertex_ai/claude-haiku-4-5": { "deprecation_date": "2026-10-15", + "successor_model": "vertex_ai/claude-haiku-5-5", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, @@ -51576,32 +52587,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-opus": { - "deprecation_date": "2025-08-01", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-3-opus@20240229": { - "deprecation_date": "2025-08-01", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -51628,6 +52613,7 @@ }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", + "successor_model": "vertex_ai/claude-opus-5-5", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52008,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52046,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52237,6 +53225,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "deprecation_date": "2026-09-29", + "successor_model": "vertex_ai/claude-sonnet-5-5", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -52294,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52524,6 +53514,8 @@ }, "vertex_ai/gemini-2.5-flash-image": { "deprecation_date": "2027-03-15", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -52574,12 +53566,13 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -52594,6 +53587,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -52629,6 +53623,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -52650,6 +53645,7 @@ "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -52662,6 +53658,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -52690,6 +53687,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "supports_reasoning": false, @@ -52719,6 +53717,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -52799,17 +53798,28 @@ }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_date": "2027-05-07", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "deprecation_date": "2027-05-07", + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_above_200k_tokens": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 9e-07, + "input_cost_per_audio_token_batches": 2.5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, + "input_cost_per_audio_token_priority": 9e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, - "input_cost_per_audio_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -52820,7 +53830,10 @@ "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -52838,6 +53851,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -52849,19 +53863,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, @@ -53277,6 +54283,7 @@ "vertex_ai/zai-org/glm-4.7-maas": { "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-10-21", + "successor_model": "vertex_ai/zai-org/glm-5.2-maas", "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -53295,6 +54302,7 @@ "vertex_ai/zai-org/glm-5-maas": { "cache_read_input_token_cost": 1e-07, "deprecation_date": "2026-10-21", + "successor_model": "vertex_ai/zai-org/glm-5.2-maas", "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -53714,6 +54722,7 @@ }, "vertex_ai/veo-3.1-generate-001": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -53729,6 +54738,7 @@ }, "vertex_ai/veo-3.1-fast-generate-001": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -53931,7 +54941,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53947,7 +54957,8 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "supports_embedding_image_input": true }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, @@ -54379,6 +55390,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "source": "https://developers.openai.com/api/docs/pricing" }, "xai/grok-4.20-multi-agent-beta-0309": { @@ -54642,6 +55655,8 @@ }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "zai.glm-5", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, @@ -54686,6 +55701,7 @@ }, "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -54999,6 +56015,7 @@ }, "openai/sora-2": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -55013,6 +56030,7 @@ }, "openai/sora-2-pro": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, @@ -55028,6 +56046,7 @@ "openai/sora-2-pro-high-res": { "litellm_provider": "openai", "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, "source": "https://platform.openai.com/docs/api-reference/videos", @@ -58694,6 +59713,8 @@ }, "gpt-4o-mini-tts-2025-12-15": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -58714,6 +59735,8 @@ }, "gpt-4o-mini-transcribe-2025-03-20": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-4o-mini-transcribe-2025-12-15", "input_cost_per_audio_token": 1.25e-06, "input_cost_per_second": 5e-05, "input_cost_per_token": 1.25e-06, @@ -58729,6 +59752,8 @@ }, "gpt-4o-mini-transcribe-2025-12-15": { "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_audio_token": 1.25e-06, "input_cost_per_second": 5e-05, "input_cost_per_token": 1.25e-06, @@ -58750,6 +59775,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -58777,6 +59803,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -58828,6 +59855,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-realtime-whisper": { @@ -58849,6 +59878,7 @@ }, "sora-2": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -58863,6 +59893,7 @@ }, "sora-2-pro": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, @@ -58878,6 +59909,7 @@ "sora-2-pro-high-res": { "litellm_provider": "openai", "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, "source": "https://developers.openai.com/api/docs/pricing", @@ -58894,6 +59926,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 8e-06, "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, @@ -58901,6 +59935,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -58909,48 +59944,6 @@ "/v1/images/edits" ] }, - "gemini-2.0-flash-exp-image-generation": { - "deprecation_date": "2025-11-14", - "input_cost_per_token": 0.0, - "litellm_provider": "gemini", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.039, - "output_cost_per_token": 0.0, - "source": "https://ai.google.dev/pricing", - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_vision": true - }, - "gemini/gemini-2.0-flash-exp-image-generation": { - "deprecation_date": "2025-11-14", - "input_cost_per_token": 0.0, - "litellm_provider": "gemini", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.039, - "output_cost_per_token": 0.0, - "source": "https://ai.google.dev/pricing", - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_vision": true, - "tpm": 250000, - "rpm": 10 - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -59003,6 +59996,8 @@ }, "gemini-2.5-flash-native-audio-preview-12-2025": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -59034,6 +60029,8 @@ }, "gemini-3.1-flash-live-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -59191,6 +60188,8 @@ }, "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -59225,6 +60224,8 @@ }, "gemini/gemini-3.1-flash-live-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -59264,6 +60265,8 @@ }, "gemini/gemini-3.1-flash-tts-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", @@ -59333,6 +60336,8 @@ }, "gemini-2.5-flash-preview-tts": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -59654,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -63392,6 +64398,10 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -63404,7 +64414,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63422,7 +64433,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63456,7 +64468,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -63547,6 +64560,7 @@ }, "claude-mythos-preview": { "deprecation_date": "2026-06-09", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -63736,7 +64750,8 @@ { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", - "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.", "model_info": { "mode": "chat", "max_input_tokens": 200000, @@ -63779,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -64223,6 +65246,7 @@ "fireworks_ai/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/glm-5p3-fast", "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -64492,6 +65516,7 @@ "fireworks_ai/accounts/fireworks/routers/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast", "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -64583,7 +65608,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64695,13 +65720,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -64804,13 +65829,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66235,9 +67260,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66616,8 +67641,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -66774,10 +67799,12 @@ }, "xai/grok-imagine-image-quality": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66791,10 +67818,12 @@ }, "xai/grok-imagine-image-quality-20260403": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66808,10 +67837,12 @@ }, "xai/grok-imagine-image-quality-latest": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66825,10 +67856,12 @@ }, "xai/grok-imagine-image-pro": { "deprecation_date": "2026-05-15", + "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66946,6 +67979,8 @@ }, "xai/grok-voice-transcribe-1.0": { "deprecation_date": "2026-10-02", + "deprecation_source": "https://docs.x.ai/docs/release-notes", + "successor_model": "xai/grok-voice-transcribe-2.0", "input_cost_per_second": 2.778e-05, "litellm_provider": "xai", "metadata": { @@ -67811,6 +68846,7 @@ }, "together_ai/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-08, "output_cost_per_token": 2.5e-07, "litellm_provider": "together_ai", @@ -67881,6 +68917,7 @@ "azure_ai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "deprecation_date": "2027-04-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -67916,6 +68953,7 @@ "supports_reasoning": true, "supports_tool_choice": true, "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/deepseek-v4-1-flash-is-coming-to-microsoft-foundry/4556431" }, "azure_ai/MAI-Image-2.6": { @@ -68024,6 +69062,8 @@ }, "azure_ai/Cohere-command-a-plus-05-2026": { "deprecation_date": "2026-10-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "coherelabs-command-a-plus-05-2026-fp81", "input_cost_per_token": 8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -68057,6 +69097,8 @@ }, "azure_ai/deepseek-r1": { "deprecation_date": "2026-08-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-pro", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", @@ -68066,6 +69108,8 @@ }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-flash", "input_cost_per_token": 1.14e-06, "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", @@ -68075,6 +69119,8 @@ }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-flash", "input_cost_per_token": 1.23e-06, "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", @@ -68084,6 +69130,8 @@ }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", @@ -68093,6 +69141,8 @@ }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-reasoning", "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", @@ -68102,6 +69152,8 @@ }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-non-reasoning", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -68111,6 +69163,8 @@ }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-reasoning", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -68120,12 +69174,18 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -68136,13 +69196,20 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -68151,13 +69218,20 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -68166,12 +69240,18 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -68183,12 +69263,18 @@ "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68202,12 +69288,18 @@ "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68233,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68308,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68328,7 +69422,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68410,12 +69507,18 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -68426,13 +69529,20 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -68441,13 +69551,20 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -68456,12 +69573,18 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -68473,12 +69596,18 @@ "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68492,12 +69621,18 @@ "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68523,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68588,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68603,7 +69740,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68698,16 +69838,32 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 2.7e-06, - "input_cost_per_token_above_272k_tokens": 5.4e-06, - "cache_creation_input_token_cost": 3.375e-06, - "cache_creation_input_token_cost_above_272k_tokens": 6.75e-06, - "cache_read_input_token_cost": 2.7e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.4e-07, - "output_cost_per_token": 1.62e-05, - "output_cost_per_token_above_272k_tokens": 2.43e-05, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.32e-05, + "cache_creation_input_token_cost_flex": 1.65e-06, + "cache_creation_input_token_cost_priority": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.056e-06, + "cache_read_input_token_cost_flex": 1.32e-07, + "cache_read_input_token_cost_priority": 5.28e-07, + "input_cost_per_token_above_272k_tokens_flex": 2.64e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.056e-05, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 5.28e-06, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.188e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.752e-05, + "output_cost_per_token_flex": 7.92e-06, + "output_cost_per_token_priority": 3.168e-05, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_web_search": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { @@ -68735,16 +69891,32 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 2.7e-07, - "input_cost_per_token_above_272k_tokens": 5.4e-07, - "cache_creation_input_token_cost": 3.375e-07, - "cache_creation_input_token_cost_above_272k_tokens": 6.75e-07, - "cache_read_input_token_cost": 2.7e-08, - "cache_read_input_token_cost_above_272k_tokens": 5.4e-08, - "output_cost_per_token": 1.62e-06, - "output_cost_per_token_above_272k_tokens": 2.43e-06, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.32e-06, + "cache_creation_input_token_cost_flex": 1.65e-07, + "cache_creation_input_token_cost_priority": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.056e-07, + "cache_read_input_token_cost_flex": 1.32e-08, + "cache_read_input_token_cost_priority": 5.28e-08, + "input_cost_per_token_above_272k_tokens_flex": 2.64e-07, + "input_cost_per_token_above_272k_tokens_priority": 1.056e-06, + "input_cost_per_token_flex": 1.32e-07, + "input_cost_per_token_priority": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.188e-06, + "output_cost_per_token_above_272k_tokens_priority": 4.752e-06, + "output_cost_per_token_flex": 7.92e-07, + "output_cost_per_token_priority": 3.168e-06, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_web_search": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { @@ -68771,9 +69943,9 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 3.375e-06, - "cache_read_input_token_cost": 3.375e-07, - "output_cost_per_token": 2.025e-05, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supports_web_search": true @@ -68793,20 +69965,36 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "input_cost_per_token": 1.5e-06, "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 2.4e-07 + "cache_read_input_token_cost": 2.4e-07, + "cache_read_input_token_cost_batches": 1.2e-07, + "cache_read_input_token_cost_flex": 1.2e-07, + "cache_read_input_token_cost_priority": 4.2e-07, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.625e-06, + "output_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06 }, "bedrock_mantle/us-gov-west-1/xai.grok-4.6": { "use_openai_responses_path": true, "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, "mode": "chat", + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -68820,11 +70008,17 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": { "input_cost_per_token": 4.8e-08, + "input_cost_per_token_flex": 2.4e-08, + "input_cost_per_token_priority": 8.4e-08, "output_cost_per_token": 9.6e-08, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 4.8e-08, + "output_cost_per_token_flex": 4.8e-08, + "output_cost_per_token_priority": 1.68e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68838,11 +70032,18 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": { "input_cost_per_token": 1.56e-07, + "input_cost_per_token_batches": 7.8e-08, + "input_cost_per_token_flex": 7.8e-08, + "input_cost_per_token_priority": 2.73e-07, "output_cost_per_token": 4.8e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, "mode": "chat", + "output_cost_per_token_batches": 2.4e-07, + "output_cost_per_token_flex": 2.4e-07, + "output_cost_per_token_priority": 8.4e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68856,11 +70057,18 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-31b": { "input_cost_per_token": 1.68e-07, + "input_cost_per_token_batches": 8.4e-08, + "input_cost_per_token_flex": 8.4e-08, + "input_cost_per_token_priority": 2.94e-07, "output_cost_per_token": 4.8e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, "mode": "chat", + "output_cost_per_token_batches": 2.4e-07, + "output_cost_per_token_flex": 2.4e-07, + "output_cost_per_token_priority": 8.4e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68875,12 +70083,18 @@ "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -68894,12 +70108,18 @@ "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69015,10 +70235,17 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, "mode": "chat", + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69033,12 +70260,18 @@ "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69052,12 +70285,18 @@ "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69070,6 +70309,7 @@ }, "bedrock_mantle/deepseek.v3.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 5.8e-07, "input_cost_per_token_batches": 2.9e-07, "input_cost_per_token_flex": 2.9e-07, @@ -69091,6 +70331,7 @@ }, "bedrock_mantle/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, @@ -69155,6 +70396,7 @@ }, "bedrock_mantle/qwen.qwen3-coder-30b-a3b-instruct": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -69176,6 +70418,7 @@ }, "bedrock_mantle/qwen.qwen3-coder-480b-a35b-instruct": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", "input_cost_per_token": 4.5e-07, "input_cost_per_token_batches": 2.25e-07, "input_cost_per_token_flex": 2.25e-07, @@ -69626,6 +70869,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69753,6 +71004,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -69878,12 +71137,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -69914,6 +71173,7 @@ "litellm_provider": "openai", "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_reasoning": true }, @@ -69923,6 +71183,7 @@ "litellm_provider": "openai", "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_reasoning": true }, @@ -69990,6 +71251,7 @@ }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, @@ -70014,48 +71276,9 @@ "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70064,18 +71287,9 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { "deprecation_date": "2026-01-05", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70086,6 +71300,7 @@ }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { "deprecation_date": "2026-06-04", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70096,6 +71311,7 @@ }, "together_ai/Qwen/Qwen3-Coder-Next-FP8": { "deprecation_date": "2026-05-14", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70106,6 +71322,7 @@ }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70116,6 +71333,7 @@ }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70126,6 +71344,7 @@ }, "together_ai/Qwen/Qwen3-VL-32B-Instruct": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70136,6 +71355,7 @@ }, "together_ai/Qwen/Qwen3-VL-8B-Instruct": { "deprecation_date": "2026-04-16", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70147,6 +71367,7 @@ "together_ai/Qwen/Qwen3.5-397B-A17B": { "cache_read_input_token_cost": 3.5e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70155,38 +71376,9 @@ "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-V3.1": { "deprecation_date": "2026-05-14", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70195,48 +71387,21 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" + "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/zai-org/GLM-5.3-Flash" }, "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -70245,28 +71410,9 @@ "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70277,6 +71423,7 @@ }, "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "deprecation_date": "2026-03-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70285,18 +71432,9 @@ "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70307,6 +71445,7 @@ }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { "deprecation_date": "2026-04-16", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70318,6 +71457,7 @@ "together_ai/moonshotai/Kimi-K2.6": { "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-08-19", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70329,6 +71469,7 @@ "together_ai/moonshotai/Kimi-K2.7-Code": { "cache_read_input_token_cost": 1.9e-07, "deprecation_date": "2026-08-27", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70337,19 +71478,10 @@ "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-08-27", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, @@ -70360,16 +71492,19 @@ }, "together_ai/openai/gpt-oss-20b": { "deprecation_date": "2026-09-15", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" + "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/Qwen/Qwen3.5-9B" }, "together_ai/zai-org/GLM-4.5-Air-FP8": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70380,6 +71515,7 @@ }, "together_ai/zai-org/GLM-4.7": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70390,6 +71526,7 @@ }, "together_ai/zai-org/GLM-5": { "deprecation_date": "2026-06-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70401,6 +71538,7 @@ "together_ai/zai-org/GLM-5.1": { "cache_read_input_token_cost": 2.6e-07, "deprecation_date": "2026-07-10", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70411,6 +71549,7 @@ }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 4.13e-07, "input_cost_per_token": 1.65e-06, "litellm_provider": "azure", @@ -70427,6 +71566,8 @@ }, "azure/eu/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -70446,6 +71587,8 @@ }, "azure/eu/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -70465,6 +71608,8 @@ }, "azure/eu/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -70475,7 +71620,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "input_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "litellm_provider": "azure", @@ -70488,6 +71635,8 @@ }, "azure/eu/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -70508,6 +71657,8 @@ }, "azure/eu/gpt-5-codex": { "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1-codex", "cache_read_input_token_cost": 1.38e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -70516,7 +71667,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, @@ -70537,6 +71690,8 @@ }, "azure/eu/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "cache_read_input_token_cost": 5.5e-09, "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, @@ -70554,6 +71709,8 @@ }, "azure/eu/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2-pro", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -70569,6 +71726,7 @@ }, "azure/eu/gpt-5.1-codex-max": { "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -70578,6 +71736,8 @@ }, "azure/eu/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, @@ -70598,6 +71758,8 @@ }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.3-codex", "cache_read_input_token_cost": 1.925e-07, "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", @@ -70621,6 +71783,7 @@ }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, @@ -70638,6 +71801,7 @@ }, "azure/eu/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, @@ -70658,6 +71822,7 @@ }, "azure/eu/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, @@ -70675,6 +71840,7 @@ }, "azure/eu/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, @@ -70848,6 +72014,7 @@ }, "azure/eu/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -70866,6 +72033,7 @@ }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -70884,6 +72052,8 @@ }, "azure/eu/o3-deep-research": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 2.75e-06, "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -70893,6 +72063,8 @@ }, "azure/eu/o4-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "cache_read_input_token_cost": 3.03e-07, "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, @@ -70911,6 +72083,8 @@ }, "azure/eu/o4-mini-2025-04-16": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "cache_read_input_token_cost": 3.03e-07, "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, @@ -70929,6 +72103,7 @@ }, "azure/eu/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "mode": "embedding", @@ -70936,6 +72111,7 @@ }, "azure/eu/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "mode": "embedding", @@ -70943,6 +72119,7 @@ }, "azure/eu/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "mode": "embedding", @@ -70992,6 +72169,7 @@ }, "azure/us/codex-mini": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 4.13e-07, "input_cost_per_token": 1.65e-06, "litellm_provider": "azure", @@ -71008,6 +72186,8 @@ }, "azure/us/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -71027,6 +72207,8 @@ }, "azure/us/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -71046,6 +72228,8 @@ }, "azure/us/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -71056,7 +72240,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "input_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "litellm_provider": "azure", @@ -71069,6 +72255,8 @@ }, "azure/us/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -71089,6 +72277,8 @@ }, "azure/us/gpt-5-codex": { "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1-codex", "cache_read_input_token_cost": 1.38e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -71097,7 +72287,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, @@ -71118,6 +72310,8 @@ }, "azure/us/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 5.5e-09, "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, @@ -71135,6 +72329,8 @@ }, "azure/us/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2-pro", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -71150,6 +72346,7 @@ }, "azure/us/gpt-5.1-codex-max": { "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -71159,6 +72356,8 @@ }, "azure/us/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, @@ -71179,6 +72378,8 @@ }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.3-codex", "cache_read_input_token_cost": 1.925e-07, "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", @@ -71202,6 +72403,7 @@ }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, @@ -71219,6 +72421,7 @@ }, "azure/us/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, @@ -71239,6 +72442,7 @@ }, "azure/us/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, @@ -71256,6 +72460,7 @@ }, "azure/us/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, @@ -71285,6 +72490,8 @@ }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 2.75e-06, "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -71294,6 +72501,7 @@ }, "azure/us/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "mode": "embedding", @@ -71301,6 +72509,7 @@ }, "azure/us/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "mode": "embedding", @@ -71308,6 +72517,7 @@ }, "azure/us/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "mode": "embedding", @@ -71318,6 +72528,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -71327,6 +72539,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71384,7 +72599,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71394,7 +72613,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71403,7 +72623,13 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", @@ -71421,7 +72647,11 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, @@ -72860,7 +74090,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, @@ -73719,6 +74950,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "deprecation_date": "2026-05-15", + "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", "input_cost_per_image_token": 1e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_above_200k_tokens": 2e-06, @@ -73729,6 +74961,7 @@ "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "source": "https://api.x.ai/v1/language-models", + "successor_model": "xai/grok-build-0.1", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -73818,6 +75051,8 @@ }, "vertex_ai/gemini-2.0-flash": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_character": 3.75e-08, @@ -73831,6 +75066,8 @@ }, "vertex_ai/gemini-2.0-flash-lite": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "input_cost_per_audio_token": 7.5e-08, "input_cost_per_audio_token_batches": 3.75e-08, "input_cost_per_character": 1.875e-08, @@ -73882,6 +75119,8 @@ }, "vertex_ai/veo-3.0-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -73897,6 +75136,8 @@ }, "vertex_ai/veo-3.0-fast-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-fast-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -73913,6 +75154,8 @@ }, "vertex_ai/veo-2.0-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -74233,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -74260,12 +75504,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -74347,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74437,6 +75684,7 @@ }, "deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "input_cost_per_token": 1.35e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -74862,7 +76110,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -74910,6 +76158,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -75288,6 +76539,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -75512,6 +76764,8 @@ }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "successor_model": "azure_ai/kimi-k2.5", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -75613,6 +76867,7 @@ "azure/model-router": { "comment": "The flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it", "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.4e-07, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -75624,6 +76879,7 @@ }, "azure/whisper": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", @@ -75685,6 +76941,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -75731,6 +76988,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -75840,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -75892,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -75936,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75983,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76019,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76075,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76121,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76161,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76202,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76248,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76299,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76345,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76377,6 +77647,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -76402,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76453,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76503,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76547,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76711,23 +77988,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -76757,23 +78030,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77073,6 +78342,7 @@ "litellm_provider": "xai", "mode": "realtime", "source": "https://docs.x.ai/developers/pricing", + "successor_model": "xai/grok-voice-3.0", "supported_endpoints": [ "/v1/realtime" ], @@ -77156,6 +78426,7 @@ "cache_creation_input_token_cost_above_1hr": 1.6e-05, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 8e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -77267,9 +78538,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -78311,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78525,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78600,6 +79874,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "bedrock/claude_platform/claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, @@ -78689,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78812,6 +80089,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -80673,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, @@ -80717,6 +81996,7 @@ "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, "deprecation_date": "2026-11-19", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 3.0002e-07, "input_dbu_cost_per_token": 4.286e-06, "litellm_provider": "databricks", @@ -80744,7 +82024,8 @@ "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token_priority": 2.1e-06, "cache_read_input_token_cost_priority": 5.25e-08, - "cache_creation_input_token_cost_priority": 5.25e-07 + "cache_creation_input_token_cost_priority": 5.25e-07, + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017" }, "databricks/databricks-deepseek-v4-1-flash-1017": { "cache_creation_input_token_cost": 3.0002e-07, @@ -81196,6 +82477,8 @@ "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, + "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", "input_cost_per_token": 9.4997e-07, "input_dbu_cost_per_token": 1.3571e-05, "litellm_provider": "databricks", @@ -81208,6 +82491,7 @@ "output_cost_per_token": 4.00001e-06, "output_dbu_cost_per_token": 5.7143e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "successor_model": "databricks/databricks-kimi-k3", "supported_modalities": [ "text", "image" @@ -81294,6 +82578,23 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/deepseek-v4p1-flash-1017-ultrafast": { + "cache_read_input_token_cost": 4.8e-08, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 9.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing.md", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/deepseek-v4p1-flash-1017-spot": { "cache_read_input_token_cost": 4.2e-09, "input_cost_per_token": 2.1e-07, @@ -81388,17 +82689,17 @@ "supports_native_streaming": true }, "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_priority": 5.625e-08, - "input_cost_per_token": 2.25e-07, - "input_cost_per_token_priority": 2.8125e-07, + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 7.5e-07, - "output_cost_per_token_priority": 9.375e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://docs.fireworks.ai/serverless/pricing.md", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -81488,6 +82789,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -81540,5 +82842,3339 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "empiriolabs/aplomb-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0.0, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages", + "/v1/decisions" + ], + "supports_vision": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/aplomb-1" + }, + "empiriolabs/deepseek-v4-1-flash-1017": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017" + }, + "empiriolabs/deepseek-v4-flash-0731": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 4.24e-07, + "output_cost_per_token": 1.272e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + }, + "empiriolabs/deepseek-v4-flash-0731:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 4.4e-07, + "output_cost_per_token": 1.32e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + }, + "empiriolabs/deepseek-v4-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 3.3e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro-0813": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.32e-06, + "output_cost_per_token": 3.96e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813" + }, + "empiriolabs/deepseek-v4-pro:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 4.8e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro:variant2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 3.301e-06, + "cache_read_input_token_cost": 1.38e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro:variant3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 4.8e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/fugu-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.007, + "search_context_size_medium": 0.007, + "search_context_size_high": 0.007 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-max" + }, + "empiriolabs/fugu-ultra-v1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-05, + "input_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 6.75e-05, + "cache_read_input_token_cost_above_272k_tokens": 3e-06, + "cache_read_input_token_cost": 1.5e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v1-0" + }, + "empiriolabs/fugu-ultra-v1-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost": 5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v1-1" + }, + "empiriolabs/fugu-ultra-v2-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost": 5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v2-0" + }, + "empiriolabs/gemma-4-26b-a4b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.9e-07, + "cache_read_input_token_cost": 2.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/gemma-4-26b-a4b" + }, + "empiriolabs/glm-4-6v-flashx": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.6e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-4-6v-flashx" + }, + "empiriolabs/glm-4-7-flashx": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 6.3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-4-7-flashx" + }, + "empiriolabs/glm-5-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 202745, + "input_cost_per_token": 8.25e-07, + "output_cost_per_token": 3.301e-06, + "input_cost_per_token_above_32k_tokens": 1.1e-06, + "output_cost_per_token_above_32k_tokens": 3.851e-06, + "cache_read_input_token_cost_above_32k_tokens": 2.2e-07, + "cache_read_input_token_cost": 1.65e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-1" + }, + "empiriolabs/glm-5-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-2:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 3.851e-06, + "cache_read_input_token_cost": 2.75e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-2:variant2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-3" + }, + "empiriolabs/glm-5-3-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 2.5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-3-flash" + }, + "empiriolabs/kimi-k2-6": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 16000, + "input_cost_per_token": 8.939e-07, + "output_cost_per_token": 3.7131e-06, + "cache_read_input_token_cost": 1.788e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-6" + }, + "empiriolabs/kimi-k2-7-code": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + }, + "empiriolabs/kimi-k2-7-code-highspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.9e-06, + "output_cost_per_token": 8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed" + }, + "empiriolabs/kimi-k2-7-code:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 8.939e-07, + "output_cost_per_token": 3.7131e-06, + "cache_read_input_token_cost": 1.788e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + }, + "empiriolabs/kimi-k3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k3" + }, + "empiriolabs/mimo-v2-6-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-flash" + }, + "empiriolabs/mimo-v2-6-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 4.35e-07, + "output_cost_per_token": 8.7e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro" + }, + "empiriolabs/mimo-v2-6-pro-ultraspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 4.35e-06, + "output_cost_per_token": 8.7e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed" + }, + "empiriolabs/minimax-m2-7": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m2-7" + }, + "empiriolabs/minimax-m2-7-highspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m2-7-highspeed" + }, + "empiriolabs/minimax-m3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 9e-07, + "input_cost_per_token_above_512k_tokens": 4.5e-07, + "output_cost_per_token_above_512k_tokens": 1.8e-06, + "cache_read_input_token_cost_above_512k_tokens": 9e-08, + "cache_read_input_token_cost": 4.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3" + }, + "empiriolabs/minimax-m3-1-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 1.125e-07, + "output_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 1.125e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3-1-flash" + }, + "empiriolabs/minimax-m3:priority": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048458, + "max_output_tokens": 524288, + "input_cost_per_token": 3.375e-07, + "output_cost_per_token": 1.35e-06, + "input_cost_per_token_above_512k_tokens": 6.75e-07, + "output_cost_per_token_above_512k_tokens": 2.7e-06, + "cache_read_input_token_cost_above_512k_tokens": 1.35e-07, + "cache_read_input_token_cost": 6.75e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3" + }, + "empiriolabs/mistral-small-4": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.084, + "search_context_size_medium": 0.084, + "search_context_size_high": 0.14 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/mistral-small-4" + }, + "empiriolabs/muse-glimmer-30b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, + "cache_read_input_token_cost": 5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/muse-glimmer-30b" + }, + "empiriolabs/muse-spark-1-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-1" + }, + "empiriolabs/muse-spark-1-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-2" + }, + "empiriolabs/muse-spark-1-3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-3" + }, + "empiriolabs/nova-lite-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 317379, + "max_output_tokens": 10000, + "input_cost_per_token": 6.9e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 3.86e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-lite-1-0" + }, + "empiriolabs/nova-lite-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 3.16e-06, + "cache_read_input_token_cost": 2.128e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-lite-2" + }, + "empiriolabs/nova-micro-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 133003, + "max_output_tokens": 10000, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 1.6e-07, + "cache_read_input_token_cost": 2.24e-08, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-micro-1-0" + }, + "empiriolabs/nova-pro-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 317319, + "max_output_tokens": 10000, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 9.6e-06, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://empiriolabs.ai/models/nova-pro-1-0" + }, + "empiriolabs/perplexity-advanced-deep-research": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_output_tokens": 16000, + "input_cost_per_token": 1.2e-05, + "output_cost_per_token": 6e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012, + "search_context_size_high": 0.012 + }, + "source": "https://empiriolabs.ai/models/perplexity-advanced-deep-research" + }, + "empiriolabs/perplexity-deep-research": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919327, + "input_cost_per_token": 4.8e-06, + "output_cost_per_token": 1.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-deep-research" + }, + "empiriolabs/perplexity-pro-search": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_output_tokens": 32768, + "input_cost_per_token": 7.8e-06, + "output_cost_per_token": 3.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-pro-search" + }, + "empiriolabs/perplexity-sonar": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919327, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 2.4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar" + }, + "empiriolabs/perplexity-sonar-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919328, + "max_output_tokens": 32768, + "input_cost_per_token": 7.2e-06, + "output_cost_per_token": 3.6e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar-pro" + }, + "empiriolabs/perplexity-sonar-reasoning-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919330, + "max_output_tokens": 131072, + "input_cost_per_token": 4.8e-06, + "output_cost_per_token": 1.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar-reasoning-pro" + }, + "empiriolabs/qwen3-5-122b-a10b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 1.15e-07, + "output_cost_per_token": 9.17e-07, + "input_cost_per_token_above_128k_tokens": 2.87e-07, + "output_cost_per_token_above_128k_tokens": 2.294e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b" + }, + "empiriolabs/qwen3-5-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 8.6e-08, + "output_cost_per_token": 6.88e-07, + "input_cost_per_token_above_128k_tokens": 2.58e-07, + "output_cost_per_token_above_128k_tokens": 2.064e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-27b" + }, + "empiriolabs/qwen3-5-35b-a3b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 5.7e-08, + "output_cost_per_token": 4.59e-07, + "input_cost_per_token_above_128k_tokens": 2.29e-07, + "output_cost_per_token_above_128k_tokens": 1.835e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b" + }, + "empiriolabs/qwen3-5-397b-a17b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 1.032e-06, + "input_cost_per_token_above_128k_tokens": 4.3e-07, + "output_cost_per_token_above_128k_tokens": 2.58e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b" + }, + "empiriolabs/qwen3-5-4b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 7e-08, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-4b" + }, + "empiriolabs/qwen3-5-9b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.3e-07, + "cache_read_input_token_cost": 4.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-9b" + }, + "empiriolabs/qwen3-5-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 995904, + "max_output_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.68e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-flash" + }, + "empiriolabs/qwen3-5-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 995904, + "max_output_tokens": 65536, + "input_cost_per_token": 2.9e-08, + "output_cost_per_token": 2.87e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-flash" + }, + "empiriolabs/qwen3-5-omni-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash" + }, + "empiriolabs/qwen3-5-omni-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 2.8e-06, + "output_cost_per_token": 1.66e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus" + }, + "empiriolabs/qwen3-5-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 2.21e-06, + "input_cost_per_token_above_256k_tokens": 1.08e-06, + "output_cost_per_token_above_256k_tokens": 6.62e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-plus" + }, + "empiriolabs/qwen3-5-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.15e-07, + "output_cost_per_token": 6.88e-07, + "input_cost_per_token_above_128k_tokens": 2.87e-07, + "output_cost_per_token_above_128k_tokens": 1.72e-06, + "input_cost_per_token_above_256k_tokens": 5.73e-07, + "output_cost_per_token_above_256k_tokens": 3.44e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-plus" + }, + "empiriolabs/qwen3-6-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "input_cost_per_token": 4.12564e-07, + "output_cost_per_token": 2.475384e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-27b" + }, + "empiriolabs/qwen3-6-35b-a3b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 4.2e-07, + "cache_read_input_token_cost": 3.5e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b" + }, + "empiriolabs/qwen3-6-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "input_cost_per_token_above_256k_tokens": 1e-06, + "output_cost_per_token_above_256k_tokens": 4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-flash" + }, + "empiriolabs/qwen3-6-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.65e-07, + "output_cost_per_token": 9.9e-07, + "input_cost_per_token_above_256k_tokens": 6.6e-07, + "output_cost_per_token_above_256k_tokens": 3.961e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-flash" + }, + "empiriolabs/qwen3-6-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "input_cost_per_token_above_256k_tokens": 2e-06, + "output_cost_per_token_above_256k_tokens": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.026, + "search_context_size_medium": 0.026, + "search_context_size_high": 0.026 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-plus" + }, + "empiriolabs/qwen3-6-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.76e-07, + "output_cost_per_token": 1.651e-06, + "input_cost_per_token_above_256k_tokens": 1.101e-06, + "output_cost_per_token_above_256k_tokens": 6.602e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-plus" + }, + "empiriolabs/qwen3-7-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, + "input_cost_per_token_above_32k_tokens": 1e-07, + "output_cost_per_token_above_32k_tokens": 4e-07, + "cache_read_input_token_cost_above_32k_tokens": 2e-08, + "input_cost_per_token_above_256k_tokens": 2e-07, + "output_cost_per_token_above_256k_tokens": 8e-07, + "cache_read_input_token_cost_above_256k_tokens": 4e-08, + "cache_read_input_token_cost": 6e-09, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.03, + "search_context_size_medium": 0.03, + "search_context_size_high": 0.03 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-flash" + }, + "empiriolabs/qwen3-7-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.8e-08, + "output_cost_per_token": 1.1e-07, + "input_cost_per_token_above_32k_tokens": 8.3e-08, + "output_cost_per_token_above_32k_tokens": 3.3e-07, + "cache_read_input_token_cost_above_32k_tokens": 1.7e-08, + "input_cost_per_token_above_256k_tokens": 1.65e-07, + "output_cost_per_token_above_256k_tokens": 6.6e-07, + "cache_read_input_token_cost_above_256k_tokens": 3.3e-08, + "cache_read_input_token_cost": 6e-09, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-flash" + }, + "empiriolabs/qwen3-7-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-max" + }, + "empiriolabs/qwen3-7-max:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-max" + }, + "empiriolabs/qwen3-7-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "input_cost_per_token_above_256k_tokens": 1.2e-06, + "output_cost_per_token_above_256k_tokens": 4.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.03, + "search_context_size_medium": 0.03, + "search_context_size_high": 0.03 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-plus" + }, + "empiriolabs/qwen3-7-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.76e-07, + "output_cost_per_token": 1.101e-06, + "input_cost_per_token_above_256k_tokens": 8.26e-07, + "output_cost_per_token_above_256k_tokens": 3.301e-06, + "cache_read_input_token_cost_above_256k_tokens": 1.66e-07, + "cache_read_input_token_cost": 5.6e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-plus" + }, + "empiriolabs/qwen3-8-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 8e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-27b" + }, + "empiriolabs/qwen3-8-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 1.6e-07, + "output_cost_per_token": 4.7e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-flash" + }, + "empiriolabs/qwen3-8-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max" + }, + "empiriolabs/qwen3-8-max-0902": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max-0902" + }, + "empiriolabs/qwen3-8-max:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max" + }, + "empiriolabs/qwen3-8-omni-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1006592, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9.4e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash" + }, + "empiriolabs/seed-2-0-code": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2.4e-06, + "input_cost_per_token_above_128k_tokens": 8e-07, + "output_cost_per_token_above_128k_tokens": 4.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://empiriolabs.ai/models/seed-2-0-code" + }, + "empiriolabs/seed-2-0-lite": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 3.1e-07, + "output_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": 6.2e-07, + "output_cost_per_token_above_128k_tokens": 5e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-lite" + }, + "empiriolabs/seed-2-0-mini": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229156, + "max_output_tokens": 128000, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, + "input_cost_per_token_above_128k_tokens": 2.4e-07, + "output_cost_per_token_above_128k_tokens": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-mini" + }, + "empiriolabs/seed-2-0-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 3.79e-06, + "input_cost_per_token_above_128k_tokens": 1.26e-06, + "output_cost_per_token_above_128k_tokens": 7.58e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-pro" + }, + "empiriolabs/seed-2-1-turbo": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229093, + "max_output_tokens": 65536, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 3.13e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-1-turbo" + }, + "empiriolabs/skylark-embedding-vision": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 8000, + "input_cost_per_token": 2.5e-07, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/skylark-embedding-vision" + }, + "empiriolabs/step-3-5-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/step-3-5-flash" + }, + "empiriolabs/step-3-5-flash-2603": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/step-3-5-flash-2603" + }, + "empiriolabs/step-3-7-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.15e-06, + "cache_read_input_token_cost": 4e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/step-3-7-flash" + }, + "empiriolabs/step-5-preview": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1024000, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/step-5-preview" + }, + "empiriolabs/stepaudio-2-5-chat": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 3e-07, + "supports_system_messages": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/stepaudio-2-5-chat" + }, + "empiriolabs/stepaudio-3-chat": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 3e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/stepaudio-3-chat" + }, + "empiriolabs/text-embedding-v4": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "source": "https://empiriolabs.ai/models/text-embedding-v4" + }, + "empiriolabs/tongyi-embedding-vision-flash": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 1024, + "input_cost_per_token": 9e-08, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/tongyi-embedding-vision-flash" + }, + "empiriolabs/tongyi-embedding-vision-plus": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 1024, + "input_cost_per_token": 9e-08, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/tongyi-embedding-vision-plus" + }, + "wally/glm-5.3-flash": { + "cache_read_input_token_cost": 7.6e-08, + "input_cost_per_token": 2.72e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048567, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048569, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/deepseek-v4.1-flash-fast": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/glm-5.3-fast": { + "cache_read_input_token_cost": 3.9e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4-pro-fast": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 1.91e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.83e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/minimax-m3": { + "cache_read_input_token_cost": 6.6e-08, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "wally", + "max_input_tokens": 524287, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3.75e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://inference.runanywhere.ai/v1/models", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash-f2": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://inference.runanywhere.ai/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "together_ai/moonshotai/Kimi-K3.1-Pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "input_cost_per_token": 3.6e-06, + "input_cost_per_token_batches": 1.8e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "output_cost_per_token_batches": 9e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { + "cache_creation_input_token_cost": 4.5e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "input_cost_per_token": 3.6e-06, + "input_cost_per_token_batches": 1.8e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "output_cost_per_token_batches": 9e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { + "cache_creation_input_token_cost": 4.5e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.76e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.76e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gmi/MiniMaxAI/MiniMax-M2": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M2.7": { + "cache_read_input_token_cost": 4.2e-08, + "input_cost_per_token": 2.1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 8.4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 4.8e-08, + "input_cost_per_token": 2.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 9.6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.125e-08, + "input_cost_per_token": 1.125e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-FP8": { + "input_cost_per_token": 1.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.09e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8": { + "cache_read_input_token_cost": 1.75e-08, + "input_cost_per_token": 8.75e-08, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8": { + "input_cost_per_token": 3.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262141, + "mode": "chat", + "output_cost_per_token": 4.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-30B-A3B": { + "input_cost_per_token": 8e-08, + "litellm_provider": "gmi", + "max_input_tokens": 40946, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-32B-FP8": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 9e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262128, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Next-80B-A3B-Instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-122B-A10B": { + "input_cost_per_token": 4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-27B": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-35B-A3B": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-397B-A17B": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-35B-A3B": { + "input_cost_per_token": 2.48e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.485e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-Plus": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-Plus-2026-04-02": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.7-Max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 4.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Flash": { + "cache_read_input_token_cost": 1.6e-08, + "input_cost_per_token": 1.6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Max-0902": { + "cache_read_input_token_cost": 2.06e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.951e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/XiaomiMiMo/MiMo-V2.5": { + "cache_read_input_token_cost": 2.38e-09, + "input_cost_per_token": 1.19e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 2.38e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/XiaomiMiMo/MiMo-V2.5-Pro": { + "cache_read_input_token_cost": 2.52e-09, + "input_cost_per_token": 3.045e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 6.09e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-Prover-V2-671B": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.18e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.18e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-0528": { + "cache_read_input_token_cost": 2.3e-07, + "input_cost_per_token": 5.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.29e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Llama-8B": { + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.9e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 9e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.1-Terminus": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.2-Exp": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 4.1e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.2-Speciale": { + "input_cost_per_token": 2.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Flash": { + "cache_read_input_token_cost": 1.82e-08, + "input_cost_per_token": 9.1e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 1.82e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 9.1e-09, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 8.58e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3.52e-08, + "input_cost_per_token": 1.056e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 3.168e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.1-flash-lite-preview": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.1-pro-preview": { + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.5-flash": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 9e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.7-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.8-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-4-argon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1.3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + "input_cost_per_token": 8e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 5.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.29e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.44e-07, + "input_cost_per_token": 8.55e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k2.7-code-highspeed": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k3.1-pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/nemotron-3-ultra-550b-a55b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-pro": { + "input_cost_per_token": 3e-05, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 0.00018, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.5": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-luna": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-sol": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-terra": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-astra": { + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-luna": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-sol": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6.1-sol": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-oss-120b": { + "input_cost_per_token": 5e-08, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/stepfun-ai/Step-3.7-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/stepfun-ai/Step-5-Preview": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.7e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/HunyuanOCR": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/Hy3": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 5.8e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/hy3-preview": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 1.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/hy4-preview": { + "cache_read_input_token_cost": 4.2e-08, + "input_cost_per_token": 8.34e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/x-ai/grok-4.5": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/xai/grok4.6": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131076, + "mode": "chat", + "output_cost_per_token": 1.1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.5-FP8": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.6": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 204800, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.7-Flash": { + "input_cost_per_token": 7e-08, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5-FP8": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 1.92e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.1-FP8": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.2-FP8": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.3e-07, + "input_cost_per_token": 7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3p1-pro": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_priority": 3e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3.1-Pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1000000, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "together_ai/ricursive-ai/Ripple-65B-Preview": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.together.ai/v1/models", + "supports_prompt_caching": true } } diff --git a/litellm/models/plugin_upload.py b/litellm/models/plugin_upload.py new file mode 100644 index 00000000000..41fbf901a03 --- /dev/null +++ b/litellm/models/plugin_upload.py @@ -0,0 +1,29 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True, slots=True) +class PluginUpload: + name: str + version: str | None + description: str | None + manifest: Mapping[str, object] + content: bytes + created_by: str | None + + +@dataclass(frozen=True, slots=True) +class PluginArchiveMetadata: + name: str + enabled: bool + source: Mapping[str, str] + manifest_json: str | None + + +class PluginUploadStore(Protocol): + async def create(self, upload: PluginUpload) -> str | None: ... + + async def get_metadata(self, name: str) -> PluginArchiveMetadata | None: ... + + async def get_content(self, metadata: PluginArchiveMetadata) -> bytes | None: ... diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 700a35edb06..00fee0a0d5a 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3158,7 +3158,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } @@ -5250,13 +5250,13 @@ "generateContent": false } }, - "empiriolabs": { - "display_name": "EmpirioLabs (`empiriolabs`)", - "url": "https://docs.litellm.ai/docs/providers/empiriolabs", + "wally": { + "display_name": "Wally (`wally`)", + "url": "https://docs.litellm.ai/docs/providers/wally", "endpoints": { - "chat_completions": false, - "responses": false, - "text_completion": false, + "chat_completions": true, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, "image_edits": false, @@ -5269,11 +5269,37 @@ "realtime": false, "video_generations": false, "batches": false, - "messages": false, - "interactions": false, + "messages": true, + "interactions": true, "systemone": false, - "a2a": false, - "generateContent": false + "a2a": true, + "generateContent": true + } + }, + "empiriolabs": { + "display_name": "EmpirioLabs AI (`empiriolabs`)", + "url": "https://docs.litellm.ai/docs/providers/empiriolabs", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "openinfer": { diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/0w6kofhv0tntp.js b/litellm/proxy/_experimental/out/_next/static/chunks/0w6kofhv0tntp.js index 0a716cb7115..0b032e1b813 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/0w6kofhv0tntp.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/0w6kofhv0tntp.js @@ -29,4 +29,4 @@ const response = await client.chat.completions.create({ messages: [{ role: "user", content: "Hello!" }], }); -console.log(response);`}];function ed({group:e,baseUrl:t}){return(0,r.jsxs)("div",{className:"border-y bg-muted/40 px-4 py-4",children:[(0,r.jsxs)("div",{className:"mb-2 flex items-center gap-2",children:[(0,r.jsx)(ea.f,{className:"size-4 text-primary"}),(0,r.jsx)("span",{className:"text-sm font-medium text-foreground",children:"How routing works for this group"})]}),(0,r.jsx)("p",{className:"mb-3 text-sm text-muted-foreground",children:"priority"===e.routing_strategy?(0,r.jsxs)(r.Fragment,{children:["Request ",(0,r.jsx)("span",{className:"font-medium text-foreground",children:e.group_name})," to try eligible models in priority order. Direct requests to a member model keep their existing routing behavior."]}):(0,r.jsxs)(r.Fragment,{children:["Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the"," ",(0,r.jsx)("span",{className:"font-medium text-foreground",children:ei(e.routing_strategy)})," strategy."]})}),(0,r.jsxs)(c.i,{defaultValue:"curl",children:[(0,r.jsx)(c.U,{variant:"line",className:"h-auto w-full justify-start rounded-none border-b p-0",children:eo.map(e=>(0,r.jsx)(c.k,{value:e.value,className:"flex-none rounded-none px-4 py-2",children:e.label},e.value))}),eo.map(a=>(0,r.jsx)(c.S,{value:a.value,className:"pt-3",children:(0,r.jsx)(el.f,{language:a.language,code:a.build(e,t)})},a.value))]})]})}let ec=(0,e.i(475254).f)("git-branch",[["line",{x1:"6",x2:"6",y1:"3",y2:"15",key:"17qcm7"}],["circle",{cx:"18",cy:"6",r:"3",key:"1h7g24"}],["circle",{cx:"6",cy:"18",r:"3",key:"fqmcym"}],["path",{d:"M18 9a9 9 0 0 1-9 9",key:"n2h4wq"}]]);var eu=e.i(541071),em=e.i(494862),eg=e.i(997422),ep=e.i(547227),ef=e.i(755146),ex=e.i(196631);function eh({group:e,onEdit:t,onDelete:l}){return(0,r.jsxs)(ef.G,{children:[(0,r.jsx)(ef.X,{"aria-label":`Open actions for ${e.group_name}`,"data-testid":`routing-group-actions-${e.group_name}`,className:(0,ex.c)((0,a.i)({variant:"ghost",size:"icon-sm"}),"text-muted-foreground"),children:(0,r.jsx)(eu.f,{className:"size-4"})}),(0,r.jsxs)(ef.L,{align:"end",className:"w-44",children:[(0,r.jsxs)(ef.r,{"data-testid":"routing-group-action-edit",onClick:()=>t(e),children:[(0,r.jsx)(b.f,{}),"Edit"]}),(0,r.jsxs)(ef.r,{variant:"destructive","data-testid":"routing-group-action-delete",onClick:()=>l(e),children:[(0,r.jsx)(m.f,{}),"Delete"]})]})]})}function ey(){return(0,r.jsxs)("div",{className:"flex flex-col items-center gap-1 py-6",children:[(0,r.jsx)("div",{className:"mb-1 flex size-10 items-center justify-center rounded-lg bg-muted",children:(0,r.jsx)(er.f,{className:"size-5 text-muted-foreground"})}),(0,r.jsx)("div",{className:"text-sm font-medium text-foreground",children:"No routing groups yet"}),(0,r.jsx)("div",{className:"text-sm text-muted-foreground",children:"Create a group to load-balance a set of models behind one name."})]})}let ej=({groups:e,isLoading:a,onEdit:l,onDelete:s,proxyBaseUrl:i})=>{let[n,o]=(0,t.useState)([]),[d,c]=(0,t.useState)({}),u=i&&i.trim()?i:window.location?.origin?window.location.origin:"",m=(0,t.useCallback)(e=>{c(r=>{let t=!0===r?{}:r;return{...t,[e.group_name]:!0!==t[e.group_name]}})},[]),g=(0,t.useMemo)(()=>(({onEdit:e,onDelete:t,onToggleUsage:a})=>[{id:"group_name",accessorKey:"group_name",meta:{title:"Group Name",skeleton:"text"},header:({column:e})=>(0,r.jsx)(em.v,{column:e,title:"Group Name"}),size:240,enableSorting:!0,cell:({row:e})=>(0,r.jsx)(eg.f,{title:e.original.group_name,className:"max-w-60",onClick:()=>a(e.original)})},{id:"models",meta:{title:"Models",skeleton:"chips"},header:"Models",size:320,enableSorting:!1,cell:({row:e})=>(0,r.jsx)(ep.f,{models:e.original.models})},{id:"routing_strategy",accessorKey:"routing_strategy",meta:{title:"Strategy",skeleton:"text"},header:({column:e})=>(0,r.jsx)(em.v,{column:e,title:"Strategy"}),size:180,enableSorting:!0,cell:({row:e})=>(0,r.jsxs)("span",{className:"flex items-center gap-1.5 text-sm",children:[(0,r.jsx)(ec,{className:"size-4 shrink-0 text-muted-foreground"}),ei(e.original.routing_strategy)]})},{id:"actions",meta:{className:"text-right",headerClassName:"text-right"},header:()=>(0,r.jsx)("span",{className:"sr-only",children:"Actions"}),size:64,enableSorting:!1,enableHiding:!1,cell:({row:a})=>(0,r.jsx)("div",{className:"flex justify-end",children:(0,r.jsx)(eh,{group:a.original,onEdit:e,onDelete:t})})}])({onEdit:l,onDelete:s,onToggleUsage:m}),[l,s,m]);return(0,r.jsx)(et.f,{data:e,paginationMode:"client",columns:g,getRowId:e=>e.group_name,sortingMode:"client",sorting:n,onSortingChange:o,expanded:d,onExpandedChange:c,getRowCanExpand:()=>!0,renderSubComponent:({row:e})=>(0,r.jsx)(ed,{group:e.original,baseUrl:u}),isLoading:a,loadingMessage:"Loading routing groups…",noDataMessage:(0,r.jsx)(ey,{}),size:"compact"})};var eb=e.i(653145),e_=e.i(54105),ev=e.i(772325),ek=e.i(411382),eN=e.i(542450),ew=e.i(182668),eS=e.i(131792),eC=e.i(624687),eM=e.i(991326);let eA=new Set(["latency-based-routing","usage-based-routing"]),eE=(e,r)=>e.map((e,t)=>({model:e,priority:r.find(r=>r.model===e)?.priority??String(t+1)})),e$=(e,r)=>({group_name:e?.group_name??"",models:e?.models??[],routing_strategy:e?.routing_strategy??r[0]??"simple-shuffle",routing_strategy_args:e?.routing_strategy_args?JSON.stringify(e.routing_strategy_args,null,2):"",model_priorities:e?.routing_strategy==="priority"?[...e.models.map(r=>({model:r,priority:Object.hasOwn(e.model_priorities??{},r)?String(e.model_priorities?.[r]):""})),...Object.entries(e.model_priorities??{}).filter(([r])=>!e.models.includes(r)).map(([e,r])=>({model:e,priority:String(r)}))]:eE(e?.models??[],[])}),eO=(e,r)=>eA.has(e)?r:"",eL={"latency-based-routing":'Example: { "ttl": 3600, "lowest_latency_buffer": 0 }'},eT=({open:e,mode:l,initialValue:i,availableStrategies:o,strategyDescriptions:d,modelOptions:c,existingGroupNames:u,groupNameByModel:m,onClose:g,onSubmit:p,saving:f})=>{let x=(0,eS.d)(),h=("edit"===l&&i?Array.from(new Set([...o,i.routing_strategy])):o).map(e=>({label:"priority"===e?"Priority":e,value:e})),y=(0,t.useMemo)(()=>new Set(u.filter(e=>e!==i?.group_name).map(e=>e.toLowerCase())),[u,i]),j=(0,t.useMemo)(()=>{let e={group_name:e_.z.string().trim().min(1,"Group name is required").max(64,"Must be 64 characters or fewer").refine(e=>!y.has(e.toLowerCase()),"A group with this name already exists"),models:ev.z.array(e_.z.string()).min(1,"Select at least one model"),routing_strategy:e_.z.string().min(1,"Strategy is required"),routing_strategy_args:e_.z.string(),model_priorities:ev.z.array(ek.z.object({model:e_.z.string(),priority:e_.z.string()}))};return ek.z.object(e).superRefine((e,r)=>{if("priority"===e.routing_strategy)return;let t=((e,r)=>{let t=(e??[]).filter(e=>Object.hasOwn(r,e));if(0===t.length)return null;let a=t.map(e=>`${e} (in "${r[e]}")`).join(", ");return`Each model may belong to at most one non-priority group. Already claimed: ${a}`})(e.models,m);null!==t&&r.addIssue({code:"custom",message:t,path:["models"]})})},[y,m]),b=(0,eM.f)(j,{defaultValues:e$(i,o)});(0,t.useEffect)(()=>{b.reset(e$(i,o))},[e,i,o,b]);let _=(0,eb.h)({control:b.control,name:"routing_strategy"}),v=(0,eb.h)({control:b.control,name:"models"}),k=async e=>{let r=(e=>{let r={group_name:e.group_name.trim(),models:e.models,routing_strategy:e.routing_strategy},t=eO(e.routing_strategy,e.routing_strategy_args);if("priority"===e.routing_strategy){let t=e.model_priorities,a=new Set(t.map(({model:e})=>e)),l=a.size===e.models.length&&e.models.every(e=>a.has(e));return 0!==t.length&&t.length===e.models.length&&l?t.some(({priority:e})=>!/^\d+$/.test(e.trim())||!Number.isSafeInteger(Number(e))||1>Number(e))?{ok:!1,field:"model_priorities",message:"Priorities must be whole numbers from 1 to 9007199254740991"}:{ok:!0,group:{...r,routing_strategy_args:null,model_priorities:Object.fromEntries(t.map(({model:e,priority:r})=>[e,Number(r)]))}}:{ok:!1,field:"model_priorities",message:"Set a priority for each selected model and remove unused priorities"}}if(!t.trim())return{ok:!0,group:{...r,routing_strategy_args:null}};try{return{ok:!0,group:{...r,routing_strategy_args:JSON.parse(t)}}}catch{return{ok:!1,field:"routing_strategy_args",message:"Must be valid JSON"}}})(e);r.ok?await p(r.group):b.setError(r.field,{message:r.message})};return(0,r.jsx)(A.N,{open:e,onOpenChange:e=>!e&&g(),children:(0,r.jsxs)(A.d,{className:"max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[560px]",children:[(0,r.jsx)(A.s,{children:(0,r.jsx)(A.F,{children:"create"===l?"Create Routing Group":`Edit ${i?.group_name??""}`})}),(0,r.jsx)("form",{onSubmit:e=>e.preventDefault(),noValidate:!0,children:(0,r.jsxs)(eN.e,{children:[(0,r.jsx)(ew.f,{control:b.control,name:"group_name",label:"Group Name",description:"Use this name as the model in API calls. LiteLLM routes the request to one of the group's models.",children:({ref:e,...t})=>(0,r.jsx)(s.f,{...t,ref:e,placeholder:"fast-chat",disabled:"edit"===l})}),(0,r.jsx)(ew.f,{control:b.control,name:"models",label:"Models",description:"priority"===_?"Models from your model list that this group routes between. Models can belong to multiple priority groups.":"Models from your model list that this group routes between. A model can belong to one non-priority group.",children:({id:e,value:t,onChange:a,"aria-invalid":l,"aria-describedby":s})=>(0,r.jsxs)(eS.x,{multiple:!0,items:c,value:t,onValueChange:e=>{a(e),b.setValue("model_priorities",eE(e,b.getValues("model_priorities")))},children:[(0,r.jsx)(eS.V,{render:(0,r.jsx)("div",{ref:x}),children:(0,r.jsx)(eS.l,{children:t=>(0,r.jsxs)(r.Fragment,{children:[t.map(e=>(0,r.jsx)(eS.Q,{"aria-label":e,children:e},e)),(0,r.jsx)(eS.z,{id:e,"aria-invalid":l,"aria-describedby":s,placeholder:"Select models"})]})})}),(0,r.jsxs)(eS.R,{anchor:x,children:[(0,r.jsx)(eS.S,{children:"No models found"}),(0,r.jsx)(eS.k,{children:e=>(0,r.jsx)(eS.T,{value:e,children:e},e)})]})]})}),(0,r.jsx)(ew.f,{control:b.control,name:"routing_strategy",label:"Routing Strategy",description:"priority"===_?"Lower priorities are tried first. Models with the same priority share traffic. Applies only when calling this group.":d[_],children:({id:e,value:t,onChange:a,"aria-invalid":l,"aria-describedby":s})=>(0,r.jsxs)(n.d,{items:h,value:t,onValueChange:e=>{a(e??""),b.setValue("routing_strategy_args",eO(e??"",b.getValues("routing_strategy_args")))},children:[(0,r.jsx)(n.P,{id:e,"aria-invalid":l,"aria-describedby":s,children:(0,r.jsx)(n.U,{placeholder:"Select strategy"})}),(0,r.jsx)(n.r,{children:h.map(e=>(0,r.jsx)(n.T,{value:e.value,children:e.label},e.value))})]})}),"priority"===_&&(0,r.jsx)(ew.f,{control:b.control,name:"model_priorities",label:"Model Priorities",description:"Use 1 for your first choice, 2 for your next choice, and so on. Unavailable models are skipped.",children:({id:e,value:t,onChange:l,"aria-invalid":i,"aria-describedby":n})=>(0,r.jsxs)("div",{id:e,className:"space-y-2",role:"group","aria-label":"Model priorities",children:[0===t.length&&(0,r.jsx)("p",{className:"text-sm text-muted-foreground",children:"Select models to set priorities"}),t.map((o,d)=>(0,r.jsxs)("div",{className:"flex items-center justify-between gap-3",children:[(0,r.jsxs)("label",{htmlFor:`${e}-${d}`,className:"min-w-0 flex-1 break-words text-sm",children:[o.model,!v.includes(o.model)&&(0,r.jsx)("span",{className:"block text-xs text-destructive",children:"Model is not selected"})]}),(0,r.jsx)(s.f,{id:`${e}-${d}`,"aria-label":`Priority for ${o.model}`,"aria-invalid":i,"aria-describedby":n,inputMode:"numeric",className:"w-24",value:o.priority,onChange:e=>l(t.map((r,t)=>t===d?{...r,priority:e.target.value}:r))}),!v.includes(o.model)&&(0,r.jsx)(a.K,{type:"button",variant:"ghost",size:"sm","aria-label":`Remove priority for ${o.model}`,onClick:()=>l(t.filter((e,r)=>r!==d)),children:"Remove"})]},o.model))]})}),eA.has(_)&&(0,r.jsx)(ew.f,{control:b.control,name:"routing_strategy_args",label:"Strategy Arguments (JSON)",description:eL[_]??'Example: { "ttl": 60 }',children:({ref:e,...t})=>(0,r.jsx)(eC.f,{...t,ref:e,rows:4,placeholder:'{ "ttl": 3600 }',className:"font-mono text-xs"})}),(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"priority"===_?"Direct requests to a member model keep their existing routing behavior.":"Models outside non-priority groups use the proxy's top-level routing strategy."})]})}),(0,r.jsxs)(A.c,{children:[(0,r.jsx)(a.K,{variant:"outline",onClick:g,children:"Cancel"}),(0,r.jsx)(a.K,{onClick:()=>void b.handleSubmit(k)(),disabled:f,"aria-busy":f,children:"create"===l?"Create Group":"Save Changes"})]})]})})},eF=()=>{let{data:e,isLoading:s,refetch:n,isFetching:o}=(()=>{let{accessToken:e,userId:r,userRole:t}=(0,G.f)();return(0,L.f)({queryKey:Y.lists(),queryFn:()=>J(e),enabled:!!(e&&r&&t)})})(),{data:d}=(()=>{let{accessToken:e,userId:r,userRole:t}=(0,G.f)();return(0,L.f)({queryKey:W.detail("fields"),queryFn:async()=>await Q(e),enabled:!!(e&&r&&t)})})(),{data:c}=(0,Z.x)(),{accessToken:m}=(0,G.f)(),g=(0,ee.U)(m),f=(()=>{let{accessToken:e}=(0,G.f)(),r=(0,H.d)();return(0,B.f)({mutationFn:r=>(0,u.ir)(e,{router_settings:{routing_groups:r}}),onSuccess:()=>{r.invalidateQueries({queryKey:Y.lists()})}})})(),[x,h]=(0,t.useState)(""),[y,j]=(0,t.useState)(!1),[b,_]=(0,t.useState)("create"),[v,k]=(0,t.useState)(null),[N,w]=(0,t.useState)(null),S=(0,t.useMemo)(()=>e?.routingGroups??[],[e?.routingGroups]),C=(0,t.useMemo)(()=>{let e=x.trim().toLowerCase();return e?S.filter(r=>r.group_name.toLowerCase().includes(e)||r.routing_strategy.toLowerCase().includes(e)||r.models.some(r=>r.toLowerCase().includes(e))):S},[S,x]),M=(0,t.useMemo)(()=>e?.availableStrategies?.length?e.availableStrategies:d?.routing_group_strategies?.length?d.routing_group_strategies:d?.fields?.find(e=>"routing_strategy"===e.field_name)?.options??[],[e?.availableStrategies,d]),E=d?.routing_strategy_descriptions??{},$=(0,t.useMemo)(()=>{let e;return e="edit"===b?v?.group_name:void 0,Object.fromEntries(S.filter(r=>r.group_name!==e&&"priority"!==r.routing_strategy).flatMap(e=>e.models.map(r=>[r,e.group_name])))},[S,b,v]),O=(0,t.useMemo)(()=>Array.from(new Set((c?.data??[]).map(e=>e.model_group).filter(e=>!!e))),[c]),T=async e=>{let r="create"===b?[...S,e]:S.map(r=>r.group_name===v?.group_name?e:r);try{await f.mutateAsync(r),p.f.success("create"===b?`Created routing group "${e.group_name}"`:`Updated routing group "${e.group_name}"`),j(!1)}catch(e){p.f.error(e instanceof Error?e.message:"Failed to save routing group")}},F=async()=>{if(!N)return;let e=S.filter(e=>e.group_name!==N.group_name);try{await f.mutateAsync(e),p.f.success(`Deleted routing group "${N.group_name}"`),w(null)}catch(e){p.f.error(e instanceof Error?e.message:"Failed to delete routing group")}};return(0,r.jsxs)("div",{className:"flex w-full flex-col gap-4",children:[(0,r.jsx)(l.N,{size:"sm",children:(0,r.jsxs)(l.O,{children:[(0,r.jsxs)("div",{className:"mb-4 flex items-center justify-between gap-3",children:[(0,r.jsxs)(i.H,{className:"max-w-sm",children:[(0,r.jsx)(i.B,{children:(0,r.jsx)(U.f,{className:"size-4 text-muted-foreground"})}),(0,r.jsx)(i.M,{placeholder:"Search groups...",value:x,onChange:e=>h(e.target.value)}),x&&(0,r.jsx)(i.B,{align:"inline-end",children:(0,r.jsx)(i.w,{size:"icon-xs","aria-label":"Clear search",onClick:()=>h(""),children:(0,r.jsx)(V.X,{})})})]}),(0,r.jsxs)("div",{className:"flex items-center gap-3",children:[(0,r.jsxs)(a.K,{variant:"outline",onClick:()=>n(),disabled:o&&!s,"aria-busy":o&&!s,children:[(0,r.jsx)(R.f,{}),"Refresh"]}),(0,r.jsxs)(a.K,{onClick:()=>{_("create"),k(null),j(!0)},children:[(0,r.jsx)(D.f,{}),"Create Group"]}),(0,r.jsxs)("span",{className:"text-sm whitespace-nowrap text-muted-foreground",children:["Showing ",C.length," ",1===C.length?"result":"results"]})]})]}),(0,r.jsx)(ej,{groups:C,isLoading:s,onEdit:e=>{_("edit"),k(e),j(!0)},onDelete:e=>w(e),proxyBaseUrl:g.LITELLM_UI_API_DOC_BASE_URL?.trim()||g.PROXY_BASE_URL||""})]})}),(0,r.jsx)(eT,{open:y,mode:b,initialValue:v,availableStrategies:M,strategyDescriptions:E,modelOptions:O,existingGroupNames:S.map(e=>e.group_name),groupNameByModel:$,onClose:()=>j(!1),onSubmit:T,saving:f.isPending}),(0,r.jsx)(A.N,{open:!!N,onOpenChange:e=>!e&&w(null),children:(0,r.jsxs)(A.d,{className:"max-h-[calc(100dvh-2rem)] overflow-y-auto",children:[(0,r.jsx)(A.s,{children:(0,r.jsx)(A.F,{children:"Delete routing group?"})}),(0,r.jsx)("p",{className:"text-sm text-foreground",children:N?.routing_strategy==="priority"?(0,r.jsxs)(r.Fragment,{children:["Calls to ",(0,r.jsx)("span",{className:"font-medium",children:N.group_name})," will stop working. Direct requests to its member models keep their existing routing behavior. This cannot be undone."]}):(0,r.jsxs)(r.Fragment,{children:["Models in ",(0,r.jsx)("span",{className:"font-medium",children:N?.group_name})," will fall back to the proxy's top-level routing strategy. This cannot be undone."]})}),(0,r.jsxs)(A.c,{children:[(0,r.jsx)(a.K,{variant:"outline",onClick:()=>w(null),children:"Cancel"}),(0,r.jsx)(a.K,{onClick:F,variant:"destructive",disabled:f.isPending,"aria-busy":f.isPending,children:"Delete"})]})]})})]})},eK=e=>"object"==typeof e&&null!==e&&"key"in e&&"string"==typeof e.key,eq=[{value:"exact",label:"Exact value"},{value:"any",label:"Any value"}],ez=/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/,eI=e=>void 0!==e.value&&null!==e.value,eP=e=>JSON.stringify([e.key,eI(e)?e.value:null]),eD=(e,r)=>{var t;let a=eP(e[r]),l=e.slice(0,r).some(e=>eP(e)===a);return((t=e[r]).key?ez.test(t.key)?eI(t)?t.value?t.tag?null:"Enter a tag":"Enter a match value":t.tag_prefix?null:"Enter a tag prefix":"Use only letters, digits, and !#$%&'*+-.^_`|~ in the baggage key":"Enter a baggage key")??(l?"Another mapping already matches this key and value":null)};function eR({value:e,onChange:t,disabled:l=!1}){let i=Array.isArray(e)?e.filter(eK):[],d=(e,r)=>t(i.map((t,a)=>a===e?{...t,...r}:t));return(0,r.jsxs)("div",{className:"min-w-80 space-y-4 whitespace-normal",children:[(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"Exact value mappings take priority over any-value rules for the same key."}),i.map((e,c)=>{let u=eI(e),g=eD(i,c),p=`Mapping ${c+1}`;return(0,r.jsxs)("fieldset",{disabled:l,className:"space-y-3 rounded-lg border p-3 disabled:opacity-60",children:[(0,r.jsx)("legend",{className:"px-1 text-xs font-medium",children:p}),(0,r.jsxs)("div",{className:"flex items-center gap-2",children:[(0,r.jsxs)(n.d,{items:eq,value:u?"exact":"any",onValueChange:e=>d(c,"exact"===e?{value:"",tag:"",tag_prefix:void 0}:{value:void 0,tag:void 0,tag_prefix:""}),children:[(0,r.jsx)(n.P,{"aria-label":`${p} mode`,children:(0,r.jsx)(n.U,{})}),(0,r.jsx)(n.r,{children:eq.map(e=>(0,r.jsx)(n.T,{value:e.value,children:e.label},e.value))})]}),(0,r.jsx)(a.K,{type:"button",variant:"ghost",size:"icon","aria-label":`Delete ${p.toLowerCase()}`,onClick:()=>t(i.filter((e,r)=>r!==c)),children:(0,r.jsx)(m.f,{className:"h-4 w-4"})})]}),(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Baggage key",(0,r.jsx)(s.f,{"aria-label":`${p} baggage key`,placeholder:"entrypoint",value:e.key,onChange:e=>d(c,{key:e.target.value})})]}),u?(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Match value",(0,r.jsx)(s.f,{"aria-label":`${p} match value`,value:e.value??"",onChange:e=>d(c,{value:e.target.value})})]}),(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Tag",(0,r.jsx)(s.f,{"aria-label":`${p} tag`,value:e.tag??"",onChange:e=>d(c,{tag:e.target.value})})]}),(0,r.jsxs)("p",{className:"text-xs text-muted-foreground",children:[e.key,"=",e.value||"value"," → ",e.tag||"tag"]})]}):(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Tag prefix",(0,r.jsx)(s.f,{"aria-label":`${p} tag prefix`,placeholder:"usecase:",value:e.tag_prefix??"",onChange:e=>d(c,{tag_prefix:e.target.value})})]}),(0,r.jsxs)("p",{className:"text-xs text-muted-foreground",children:[e.key,"=x → ",e.tag_prefix,"x"]})]}),g?(0,r.jsx)("p",{className:"text-xs text-destructive",children:g}):null,(0,r.jsxs)("label",{className:"flex items-center justify-between gap-3 text-xs",children:["Require a mapped value",(0,r.jsx)(o.f,{"aria-label":`${p} required`,checked:e.required??!1,onCheckedChange:e=>d(c,{required:e})})]}),(0,r.jsxs)("label",{className:"flex items-center justify-between gap-3 text-xs",children:["Require an existing tag budget",(0,r.jsx)(o.f,{"aria-label":`${p} require budget`,checked:e.require_budgeted_tag??!1,onCheckedChange:e=>d(c,{require_budgeted_tag:e})})]})]},c)}),l?(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"Managed in config.yaml. Edit the YAML to change these rules."}):null,(0,r.jsx)(a.K,{disabled:l,type:"button",variant:"outline",onClick:()=>t([...i,{header:"baggage",key:"",value:"",tag:""}]),children:"Add mapping"})]})}let eU="enable_anthropic_prompt_caching",eV="anthropic_prompt_caching_ttl",eG="openai_system_messages_first",eB=e=>!0===e||"true"===e,eH="w-36",eX=e=>""===e?null:Number(e),eY=({setting:e,onChange:t})=>{if("request_tag_mappings"===e.field_name)return(0,r.jsx)(eR,{value:e.field_value,disabled:!1===e.editable,onChange:r=>t(e.field_name,r)});if("String"===e.field_type)return(0,r.jsx)(s.f,{"aria-label":e.field_name,value:e.field_value??"",onChange:r=>t(e.field_name,r.target.value)});if("Integer"===e.field_type)return(0,r.jsx)(s.f,{type:"number",step:1,className:eH,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))});if("Boolean"===e.field_type)return(0,r.jsx)(o.f,{checked:!0===e.field_value||"true"===e.field_value,onCheckedChange:r=>t(e.field_name,r)});if("Float"===e.field_type)return(0,r.jsx)(s.f,{type:"number",min:0,max:1,step:.05,className:eH,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))});if("Dollar"===e.field_type)return(0,r.jsxs)(i.H,{className:eH,children:[(0,r.jsx)(i.B,{children:"$"}),(0,r.jsx)(i.M,{type:"number",min:.01,step:.25,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))})]});if("List"===e.field_type){let a;return(0,r.jsx)(s.f,{"aria-label":e.field_name,placeholder:"Comma-separated values",defaultValue:Array.isArray(a=e.field_value)?a.join(", "):"",onChange:r=>{let a;return t(e.field_name,0===(a=r.target.value.split(",").map(e=>e.trim()).filter(e=>""!==e)).length?null:a)}},String(e.stored_in_db))}return"Select"===e.field_type?(0,r.jsxs)(n.d,{value:e.field_value??null,onValueChange:r=>t(e.field_name,r),children:[(0,r.jsx)(n.P,{className:"min-w-32",children:(0,r.jsx)(n.U,{placeholder:"Default"})}),(0,r.jsxs)(n.r,{children:[(0,r.jsx)(n.T,{value:null,children:"Default"}),(e.field_options??[]).map(e=>(0,r.jsx)(n.T,{value:e,children:e},e))]})]}):null},eJ=({accessToken:e,settings:t,onChange:a})=>{let s=t.find(e=>e.field_name===eU),i=t.find(e=>e.field_name===eV),d=t.find(e=>e.field_name===eG);if(!s)return null;let c=eB(s.field_value),m=(r,t)=>{a(r,t),""===t||null==t?(0,u.x5)(e,r):(0,u.Gy)(e,r,t)};return(0,r.jsx)(l.N,{children:(0,r.jsxs)(l.O,{children:[(0,r.jsx)(l.v,{children:"Prompt Caching"}),(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:"font-medium",children:"Automatic Anthropic prompt caching"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:s.field_description})]}),(0,r.jsx)(o.f,{checked:c,onCheckedChange:e=>m(eU,e)})]}),i&&(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:`font-medium ${c?"":"text-muted-foreground"}`,children:"Cache lifetime (TTL)"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:i.field_description})]}),(0,r.jsxs)(n.d,{disabled:!c,value:i.field_value??null,onValueChange:e=>m(eV,e),children:[(0,r.jsx)(n.P,{className:"min-w-40",children:(0,r.jsx)(n.U,{placeholder:"5m (default)"})}),(0,r.jsxs)(n.r,{children:[(0,r.jsx)(n.T,{value:null,children:"5m (default)"}),(i.field_options??[]).map(e=>(0,r.jsx)(n.T,{value:e,children:e},e))]})]})]}),d&&(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:"font-medium",children:"System messages first for OpenAI"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:d.field_description})]}),(0,r.jsx)(o.f,{"aria-label":"System messages first for OpenAI",checked:eB(d.field_value),onCheckedChange:e=>m(eG,e)})]})]})})};e.s(["T",0,eJ,"U",0,({accessToken:e,userRole:s,userID:i})=>{let[n,o]=(0,t.useState)([]);(0,t.useEffect)(()=>{e&&(0,u.T4)(e).then(e=>{o(e)})},[e]);let p=(e,r)=>{o(n.map(t=>t.field_name===e?{...t,field_value:r}:t))},f=async r=>{if(!e)return;let t=n.find(e=>e.field_name===r),a=t?.field_value;if(null==a){(t?.field_type==="Select"||t?.field_type==="List")&&x(r);return}try{await (0,u.Gy)(e,r,a);let t=n.map(e=>e.field_name===r?{...e,stored_in_db:!0}:e);o(t)}catch(e){}},x=r=>{if(e)try{(0,u.x5)(e,r);let t=n.map(e=>e.field_name===r?{...e,stored_in_db:null,field_value:e.field_default_value??null}:e);o(t)}catch(e){}};return e?(0,r.jsx)("div",{className:"w-full",children:(0,r.jsxs)(c.i,{defaultValue:"loadbalancing",className:"h-[75vh] w-full",children:[(0,r.jsxs)(c.U,{variant:"line",className:"mx-8 mt-4",children:[(0,r.jsx)(c.k,{value:"loadbalancing",children:"Loadbalancing"}),(0,r.jsx)(c.k,{value:"routing-groups",children:"Routing Groups"}),(0,r.jsx)(c.k,{value:"fallbacks",children:"Fallbacks"}),(0,r.jsx)(c.k,{value:"prompt-caching",children:"Prompt Caching"}),(0,r.jsx)(c.k,{value:"general",children:"General"})]}),(0,r.jsx)(c.S,{value:"loadbalancing",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(h,{accessToken:e,userRole:s,userID:i})}),(0,r.jsx)(c.S,{value:"routing-groups",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(eF,{})}),(0,r.jsx)(c.S,{value:"fallbacks",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(P,{accessToken:e,userRole:s,userID:i})}),(0,r.jsx)(c.S,{value:"prompt-caching",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(eJ,{accessToken:e,settings:n,onChange:p})}),(0,r.jsx)(c.S,{value:"general",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(l.N,{children:(0,r.jsx)(l.O,{children:(0,r.jsxs)(d.s,{children:[(0,r.jsx)(d.E,{children:(0,r.jsxs)(d.N,{children:[(0,r.jsx)(d.V,{children:"Setting"}),(0,r.jsx)(d.V,{children:"Value"}),(0,r.jsx)(d.V,{children:"Status"}),(0,r.jsx)(d.V,{children:"Action"})]})}),(0,r.jsx)(d.z,{children:n.filter(e=>"TypedDictionary"!==e.field_type&&"prompt_caching"!==e.field_tab).map((e,t)=>{let l;return(0,r.jsxs)(d.N,{children:[(0,r.jsxs)(d.d,{className:"whitespace-normal",children:[(0,r.jsx)("p",{className:"break-words",children:e.field_name}),(0,r.jsx)("p",{style:{fontSize:"0.65rem",color:"#808080",fontStyle:"italic"},className:"mt-1 break-words",children:e.field_description})]}),(0,r.jsx)(d.d,{children:(0,r.jsx)(eY,{setting:e,onChange:p})}),(0,r.jsx)(d.d,{children:!0==e.stored_in_db?(0,r.jsx)(g.f,{tone:"success",label:"In DB"}):!1==e.stored_in_db?(0,r.jsx)(g.f,{tone:"neutral",label:"In Config"}):(0,r.jsx)(g.f,{tone:"neutral",label:"Not Set"})}),(0,r.jsxs)(d.d,{children:[(0,r.jsx)(a.K,{disabled:"request_tag_mappings"===e.field_name&&(!1===e.editable||!(Array.isArray(l=e.field_value)&&l.every(eK)&&l.every((e,r)=>null===eD(l,r)))),onClick:()=>f(e.field_name),children:"Update"}),"request_tag_mappings"===e.field_name&&!1===e.editable?null:(0,r.jsx)("span",{onClick:()=>x(e.field_name),className:"inline-flex shrink-0 cursor-pointer items-center justify-center px-1.5 py-1.5 text-destructive",children:(0,r.jsx)(m.f,{className:"h-5 w-5 shrink-0"})})]})]},t)})})]})})})})]})}):null}],863679)}]); \ No newline at end of file +console.log(response);`}];function ed({group:e,baseUrl:t}){return(0,r.jsxs)("div",{className:"border-y bg-muted/40 px-4 py-4",children:[(0,r.jsxs)("div",{className:"mb-2 flex items-center gap-2",children:[(0,r.jsx)(ea.f,{className:"size-4 text-primary"}),(0,r.jsx)("span",{className:"text-sm font-medium text-foreground",children:"How routing works for this group"})]}),(0,r.jsx)("p",{className:"mb-3 text-sm text-muted-foreground",children:"priority"===e.routing_strategy?(0,r.jsxs)(r.Fragment,{children:["Request ",(0,r.jsx)("span",{className:"font-medium text-foreground",children:e.group_name})," to try eligible models in priority order. Direct requests to a member model keep their existing routing behavior."]}):(0,r.jsxs)(r.Fragment,{children:["Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the"," ",(0,r.jsx)("span",{className:"font-medium text-foreground",children:ei(e.routing_strategy)})," strategy."]})}),(0,r.jsxs)(c.i,{defaultValue:"curl",children:[(0,r.jsx)(c.U,{variant:"line",className:"h-auto w-full justify-start rounded-none border-b p-0",children:eo.map(e=>(0,r.jsx)(c.k,{value:e.value,className:"flex-none rounded-none px-4 py-2",children:e.label},e.value))}),eo.map(a=>(0,r.jsx)(c.S,{value:a.value,className:"pt-3",children:(0,r.jsx)(el.f,{language:a.language,code:a.build(e,t)})},a.value))]})]})}let ec=(0,e.i(475254).f)("git-branch",[["line",{x1:"6",x2:"6",y1:"3",y2:"15",key:"17qcm7"}],["circle",{cx:"18",cy:"6",r:"3",key:"1h7g24"}],["circle",{cx:"6",cy:"18",r:"3",key:"fqmcym"}],["path",{d:"M18 9a9 9 0 0 1-9 9",key:"n2h4wq"}]]);var eu=e.i(541071),em=e.i(494862),eg=e.i(997422),ep=e.i(547227),ef=e.i(755146),ex=e.i(196631);function eh({group:e,onEdit:t,onDelete:l}){return(0,r.jsxs)(ef.G,{children:[(0,r.jsx)(ef.X,{"aria-label":`Open actions for ${e.group_name}`,"data-testid":`routing-group-actions-${e.group_name}`,className:(0,ex.c)((0,a.i)({variant:"ghost",size:"icon-sm"}),"text-muted-foreground"),children:(0,r.jsx)(eu.f,{className:"size-4"})}),(0,r.jsxs)(ef.L,{align:"end",className:"w-44",children:[(0,r.jsxs)(ef.r,{"data-testid":"routing-group-action-edit",onClick:()=>t(e),children:[(0,r.jsx)(b.f,{}),"Edit"]}),(0,r.jsxs)(ef.r,{variant:"destructive","data-testid":"routing-group-action-delete",onClick:()=>l(e),children:[(0,r.jsx)(m.f,{}),"Delete"]})]})]})}function ey(){return(0,r.jsxs)("div",{className:"flex flex-col items-center gap-1 py-6",children:[(0,r.jsx)("div",{className:"mb-1 flex size-10 items-center justify-center rounded-lg bg-muted",children:(0,r.jsx)(er.f,{className:"size-5 text-muted-foreground"})}),(0,r.jsx)("div",{className:"text-sm font-medium text-foreground",children:"No routing groups yet"}),(0,r.jsx)("div",{className:"text-sm text-muted-foreground",children:"Create a group to load-balance a set of models behind one name."})]})}let ej=({groups:e,isLoading:a,onEdit:l,onDelete:s,proxyBaseUrl:i})=>{let[n,o]=(0,t.useState)([]),[d,c]=(0,t.useState)({}),u=i&&i.trim()?i:window.location?.origin?window.location.origin:"",m=(0,t.useCallback)(e=>{c(r=>{let t=!0===r?{}:r;return{...t,[e.group_name]:!0!==t[e.group_name]}})},[]),g=(0,t.useMemo)(()=>(({onEdit:e,onDelete:t,onToggleUsage:a})=>[{id:"group_name",accessorKey:"group_name",meta:{title:"Group Name",skeleton:"text"},header:({column:e})=>(0,r.jsx)(em.v,{column:e,title:"Group Name"}),size:240,enableSorting:!0,cell:({row:e})=>(0,r.jsx)(eg.f,{title:e.original.group_name,className:"max-w-60",onClick:()=>a(e.original)})},{id:"models",meta:{title:"Models",skeleton:"chips"},header:"Models",size:320,enableSorting:!1,cell:({row:e})=>(0,r.jsx)(ep.f,{models:e.original.models})},{id:"routing_strategy",accessorKey:"routing_strategy",meta:{title:"Strategy",skeleton:"text"},header:({column:e})=>(0,r.jsx)(em.v,{column:e,title:"Strategy"}),size:180,enableSorting:!0,cell:({row:e})=>(0,r.jsxs)("span",{className:"flex items-center gap-1.5 text-sm",children:[(0,r.jsx)(ec,{className:"size-4 shrink-0 text-muted-foreground"}),ei(e.original.routing_strategy)]})},{id:"actions",meta:{className:"text-right",headerClassName:"text-right"},header:()=>(0,r.jsx)("span",{className:"sr-only",children:"Actions"}),size:64,enableSorting:!1,enableHiding:!1,cell:({row:a})=>(0,r.jsx)("div",{className:"flex justify-end",children:(0,r.jsx)(eh,{group:a.original,onEdit:e,onDelete:t})})}])({onEdit:l,onDelete:s,onToggleUsage:m}),[l,s,m]);return(0,r.jsx)(et.f,{data:e,paginationMode:"client",columns:g,getRowId:e=>e.group_name,sortingMode:"client",sorting:n,onSortingChange:o,expanded:d,onExpandedChange:c,getRowCanExpand:()=>!0,renderSubComponent:({row:e})=>(0,r.jsx)(ed,{group:e.original,baseUrl:u}),isLoading:a,loadingMessage:"Loading routing groups…",noDataMessage:(0,r.jsx)(ey,{}),size:"compact"})};var eb=e.i(653145),e_=e.i(54105),ev=e.i(772325),ek=e.i(411382),eN=e.i(542450),ew=e.i(182668),eS=e.i(131792),eC=e.i(624687),eM=e.i(991326);let eA=new Set(["latency-based-routing","usage-based-routing"]),eE=(e,r)=>e.map((e,t)=>({model:e,priority:r.find(r=>r.model===e)?.priority??String(t+1)})),e$=(e,r)=>({group_name:e?.group_name??"",models:e?.models??[],routing_strategy:e?.routing_strategy??r[0]??"simple-shuffle",routing_strategy_args:e?.routing_strategy_args?JSON.stringify(e.routing_strategy_args,null,2):"",model_priorities:e?.routing_strategy==="priority"?[...e.models.map(r=>({model:r,priority:Object.hasOwn(e.model_priorities??{},r)?String(e.model_priorities?.[r]):""})),...Object.entries(e.model_priorities??{}).filter(([r])=>!e.models.includes(r)).map(([e,r])=>({model:e,priority:String(r)}))]:eE(e?.models??[],[])}),eO=(e,r)=>eA.has(e)?r:"",eL={"latency-based-routing":'Example: { "ttl": 3600, "lowest_latency_buffer": 0 }'},eT=({open:e,mode:l,initialValue:i,availableStrategies:o,strategyDescriptions:d,modelOptions:c,existingGroupNames:u,groupNameByModel:m,onClose:g,onSubmit:p,saving:f})=>{let x=(0,eS.d)(),h=("edit"===l&&i?Array.from(new Set([...o,i.routing_strategy])):o).map(e=>({label:"priority"===e?"Priority":e,value:e})),y=(0,t.useMemo)(()=>new Set(u.filter(e=>e!==i?.group_name).map(e=>e.toLowerCase())),[u,i]),j=(0,t.useMemo)(()=>{let e={group_name:e_.z.string().trim().min(1,"Group name is required").max(64,"Must be 64 characters or fewer").refine(e=>!y.has(e.toLowerCase()),"A group with this name already exists"),models:ev.z.array(e_.z.string()).min(1,"Select at least one model"),routing_strategy:e_.z.string().min(1,"Strategy is required"),routing_strategy_args:e_.z.string(),model_priorities:ev.z.array(ek.z.object({model:e_.z.string(),priority:e_.z.string()}))};return ek.z.object(e).superRefine((e,r)=>{if("priority"===e.routing_strategy)return;let t=((e,r)=>{let t=(e??[]).filter(e=>Object.hasOwn(r,e));if(0===t.length)return null;let a=t.map(e=>`${e} (in "${r[e]}")`).join(", ");return`Each model may belong to at most one non-priority group. Already claimed: ${a}`})(e.models,m);null!==t&&r.addIssue({code:"custom",message:t,path:["models"]})})},[y,m]),b=(0,eM.f)(j,{defaultValues:e$(i,o)});(0,t.useEffect)(()=>{b.reset(e$(i,o))},[e,i,o,b]);let _=(0,eb.h)({control:b.control,name:"routing_strategy"}),v=(0,eb.h)({control:b.control,name:"models"}),k=async e=>{let r=(e=>{let r={group_name:e.group_name.trim(),models:e.models,routing_strategy:e.routing_strategy},t=eO(e.routing_strategy,e.routing_strategy_args);if("priority"===e.routing_strategy){let t=e.model_priorities,a=new Set(t.map(({model:e})=>e)),l=a.size===e.models.length&&e.models.every(e=>a.has(e));return 0!==t.length&&t.length===e.models.length&&l?t.some(({priority:e})=>!/^\d+$/.test(e.trim())||!Number.isSafeInteger(Number(e))||1>Number(e))?{ok:!1,field:"model_priorities",message:"Priorities must be whole numbers from 1 to 9007199254740991"}:{ok:!0,group:{...r,routing_strategy_args:null,model_priorities:Object.fromEntries(t.map(({model:e,priority:r})=>[e,Number(r)]))}}:{ok:!1,field:"model_priorities",message:"Set a priority for each selected model and remove unused priorities"}}if(!t.trim())return{ok:!0,group:{...r,routing_strategy_args:null}};try{return{ok:!0,group:{...r,routing_strategy_args:JSON.parse(t)}}}catch{return{ok:!1,field:"routing_strategy_args",message:"Must be valid JSON"}}})(e);r.ok?await p(r.group):b.setError(r.field,{message:r.message})};return(0,r.jsx)(A.N,{open:e,onOpenChange:e=>!e&&g(),children:(0,r.jsxs)(A.d,{className:"max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[560px]",children:[(0,r.jsx)(A.s,{children:(0,r.jsx)(A.F,{children:"create"===l?"Create Routing Group":`Edit ${i?.group_name??""}`})}),(0,r.jsx)("form",{onSubmit:e=>e.preventDefault(),noValidate:!0,children:(0,r.jsxs)(eN.e,{children:[(0,r.jsx)(ew.f,{control:b.control,name:"group_name",label:"Group Name",description:"Use this name as the model in API calls. LiteLLM routes the request to one of the group's models.",children:({ref:e,...t})=>(0,r.jsx)(s.f,{...t,ref:e,placeholder:"fast-chat",disabled:"edit"===l})}),(0,r.jsx)(ew.f,{control:b.control,name:"models",label:"Models",description:"priority"===_?"Models from your model list that this group routes between. Models can belong to multiple priority groups.":"Models from your model list that this group routes between. A model can belong to one non-priority group.",children:({id:e,value:t,onChange:a,"aria-invalid":l,"aria-describedby":s})=>(0,r.jsxs)(eS.x,{multiple:!0,items:c,value:t,onValueChange:e=>{a(e),b.setValue("model_priorities",eE(e,b.getValues("model_priorities")))},children:[(0,r.jsx)(eS.V,{render:(0,r.jsx)("div",{ref:x}),children:(0,r.jsx)(eS.l,{children:t=>(0,r.jsxs)(r.Fragment,{children:[t.map(e=>(0,r.jsx)(eS.Q,{"aria-label":e,children:e},e)),(0,r.jsx)(eS.z,{id:e,"aria-invalid":l,"aria-describedby":s,placeholder:"Select models"})]})})}),(0,r.jsxs)(eS.R,{anchor:x,children:[(0,r.jsx)(eS.S,{children:"No models found"}),(0,r.jsx)(eS.k,{children:e=>(0,r.jsx)(eS.T,{value:e,children:e},e)})]})]})}),(0,r.jsx)(ew.f,{control:b.control,name:"routing_strategy",label:"Routing Strategy",description:"priority"===_?"Lower priorities are tried first. Models with the same priority share traffic. Applies only when calling this group.":d[_],children:({id:e,value:t,onChange:a,"aria-invalid":l,"aria-describedby":s})=>(0,r.jsxs)(n.d,{items:h,value:t,onValueChange:e=>{a(e??""),b.setValue("routing_strategy_args",eO(e??"",b.getValues("routing_strategy_args")))},children:[(0,r.jsx)(n.P,{id:e,"aria-invalid":l,"aria-describedby":s,children:(0,r.jsx)(n.U,{placeholder:"Select strategy"})}),(0,r.jsx)(n.r,{children:h.map(e=>(0,r.jsx)(n.T,{value:e.value,children:e.label},e.value))})]})}),"priority"===_&&(0,r.jsx)(ew.f,{control:b.control,name:"model_priorities",label:"Model Priorities",description:"Use 1 for your first choice, 2 for your next choice, and so on. Unavailable models are skipped.",children:({id:e,value:t,onChange:l,"aria-invalid":i,"aria-describedby":n})=>(0,r.jsxs)("div",{id:e,className:"space-y-2",role:"group","aria-label":"Model priorities",children:[0===t.length&&(0,r.jsx)("p",{className:"text-sm text-muted-foreground",children:"Select models to set priorities"}),t.map((o,d)=>(0,r.jsxs)("div",{className:"flex items-center justify-between gap-3",children:[(0,r.jsxs)("label",{htmlFor:`${e}-${d}`,className:"min-w-0 flex-1 break-words text-sm",children:[o.model,!v.includes(o.model)&&(0,r.jsx)("span",{className:"block text-xs text-destructive",children:"Model is not selected"})]}),(0,r.jsx)(s.f,{id:`${e}-${d}`,"aria-label":`Priority for ${o.model}`,"aria-invalid":i,"aria-describedby":n,inputMode:"numeric",className:"w-24",value:o.priority,onChange:e=>l(t.map((r,t)=>t===d?{...r,priority:e.target.value}:r))}),!v.includes(o.model)&&(0,r.jsx)(a.K,{type:"button",variant:"ghost",size:"sm","aria-label":`Remove priority for ${o.model}`,onClick:()=>l(t.filter((e,r)=>r!==d)),children:"Remove"})]},o.model))]})}),eA.has(_)&&(0,r.jsx)(ew.f,{control:b.control,name:"routing_strategy_args",label:"Strategy Arguments (JSON)",description:eL[_]??'Example: { "ttl": 60 }',children:({ref:e,...t})=>(0,r.jsx)(eC.f,{...t,ref:e,rows:4,placeholder:'{ "ttl": 3600 }',className:"font-mono text-xs"})}),(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"priority"===_?"Direct requests to a member model keep their existing routing behavior.":"Models outside non-priority groups use the proxy's top-level routing strategy."})]})}),(0,r.jsxs)(A.c,{children:[(0,r.jsx)(a.K,{variant:"outline",onClick:g,children:"Cancel"}),(0,r.jsx)(a.K,{onClick:()=>void b.handleSubmit(k)(),disabled:f,"aria-busy":f,children:"create"===l?"Create Group":"Save Changes"})]})]})})},eF=()=>{let{data:e,isLoading:s,refetch:n,isFetching:o}=(()=>{let{accessToken:e,userId:r,userRole:t}=(0,G.f)();return(0,L.f)({queryKey:Y.lists(),queryFn:()=>J(e),enabled:!!(e&&r&&t)})})(),{data:d}=(()=>{let{accessToken:e,userId:r,userRole:t}=(0,G.f)();return(0,L.f)({queryKey:W.detail("fields"),queryFn:async()=>await Q(e),enabled:!!(e&&r&&t)})})(),{data:c}=(0,Z.x)(),{accessToken:m}=(0,G.f)(),g=(0,ee.U)(m),f=(()=>{let{accessToken:e}=(0,G.f)(),r=(0,H.d)();return(0,B.f)({mutationFn:r=>(0,u.ir)(e,{router_settings:{routing_groups:r}}),onSuccess:()=>{r.invalidateQueries({queryKey:Y.lists()})}})})(),[x,h]=(0,t.useState)(""),[y,j]=(0,t.useState)(!1),[b,_]=(0,t.useState)("create"),[v,k]=(0,t.useState)(null),[N,w]=(0,t.useState)(null),S=(0,t.useMemo)(()=>e?.routingGroups??[],[e?.routingGroups]),C=(0,t.useMemo)(()=>{let e=x.trim().toLowerCase();return e?S.filter(r=>r.group_name.toLowerCase().includes(e)||r.routing_strategy.toLowerCase().includes(e)||r.models.some(r=>r.toLowerCase().includes(e))):S},[S,x]),M=(0,t.useMemo)(()=>e?.availableStrategies?.length?e.availableStrategies:d?.routing_group_strategies?.length?d.routing_group_strategies:d?.fields?.find(e=>"routing_strategy"===e.field_name)?.options??[],[e?.availableStrategies,d]),E=d?.routing_strategy_descriptions??{},$=(0,t.useMemo)(()=>{let e;return e="edit"===b?v?.group_name:void 0,Object.fromEntries(S.filter(r=>r.group_name!==e&&"priority"!==r.routing_strategy).flatMap(e=>e.models.map(r=>[r,e.group_name])))},[S,b,v]),O=(0,t.useMemo)(()=>Array.from(new Set((c?.data??[]).map(e=>e.model_group).filter(e=>!!e))),[c]),T=async e=>{let r="create"===b?[...S,e]:S.map(r=>r.group_name===v?.group_name?e:r);try{await f.mutateAsync(r),p.f.success("create"===b?`Created routing group "${e.group_name}"`:`Updated routing group "${e.group_name}"`),j(!1)}catch(e){p.f.error(e instanceof Error?e.message:"Failed to save routing group")}},F=async()=>{if(!N)return;let e=S.filter(e=>e.group_name!==N.group_name);try{await f.mutateAsync(e),p.f.success(`Deleted routing group "${N.group_name}"`),w(null)}catch(e){p.f.error(e instanceof Error?e.message:"Failed to delete routing group")}};return(0,r.jsxs)("div",{className:"flex w-full flex-col gap-4",children:[(0,r.jsx)(l.N,{size:"sm",children:(0,r.jsxs)(l.O,{children:[(0,r.jsxs)("div",{className:"mb-4 flex items-center justify-between gap-3",children:[(0,r.jsxs)(i.H,{className:"max-w-sm",children:[(0,r.jsx)(i.B,{children:(0,r.jsx)(U.f,{className:"size-4 text-muted-foreground"})}),(0,r.jsx)(i.M,{placeholder:"Search groups...",value:x,onChange:e=>h(e.target.value)}),x&&(0,r.jsx)(i.B,{align:"inline-end",children:(0,r.jsx)(i.w,{size:"icon-xs","aria-label":"Clear search",onClick:()=>h(""),children:(0,r.jsx)(V.X,{})})})]}),(0,r.jsxs)("div",{className:"flex items-center gap-3",children:[(0,r.jsxs)(a.K,{variant:"outline",onClick:()=>n(),disabled:o&&!s,"aria-busy":o&&!s,children:[(0,r.jsx)(R.f,{}),"Refresh"]}),(0,r.jsxs)(a.K,{onClick:()=>{_("create"),k(null),j(!0)},children:[(0,r.jsx)(D.f,{}),"Create Group"]}),(0,r.jsxs)("span",{className:"text-sm whitespace-nowrap text-muted-foreground",children:["Showing ",C.length," ",1===C.length?"result":"results"]})]})]}),(0,r.jsx)(ej,{groups:C,isLoading:s,onEdit:e=>{_("edit"),k(e),j(!0)},onDelete:e=>w(e),proxyBaseUrl:g.LITELLM_UI_API_DOC_BASE_URL?.trim()||g.PROXY_BASE_URL||""})]})}),(0,r.jsx)(eT,{open:y,mode:b,initialValue:v,availableStrategies:M,strategyDescriptions:E,modelOptions:O,existingGroupNames:S.map(e=>e.group_name),groupNameByModel:$,onClose:()=>j(!1),onSubmit:T,saving:f.isPending}),(0,r.jsx)(A.N,{open:!!N,onOpenChange:e=>!e&&w(null),children:(0,r.jsxs)(A.d,{className:"max-h-[calc(100dvh-2rem)] overflow-y-auto",children:[(0,r.jsx)(A.s,{children:(0,r.jsx)(A.F,{children:"Delete routing group?"})}),(0,r.jsx)("p",{className:"text-sm text-foreground",children:N?.routing_strategy==="priority"?(0,r.jsxs)(r.Fragment,{children:["Calls to ",(0,r.jsx)("span",{className:"font-medium",children:N.group_name})," will stop working. Direct requests to its member models keep their existing routing behavior. This cannot be undone."]}):(0,r.jsxs)(r.Fragment,{children:["Models in ",(0,r.jsx)("span",{className:"font-medium",children:N?.group_name})," will fall back to the proxy's top-level routing strategy. This cannot be undone."]})}),(0,r.jsxs)(A.c,{children:[(0,r.jsx)(a.K,{variant:"outline",onClick:()=>w(null),children:"Cancel"}),(0,r.jsx)(a.K,{onClick:F,variant:"destructive",disabled:f.isPending,"aria-busy":f.isPending,children:"Delete"})]})]})})]})},eK=e=>"object"==typeof e&&null!==e&&"key"in e&&"string"==typeof e.key,eq=[{value:"exact",label:"Exact value"},{value:"any",label:"Any value"}],ez=/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/,eI=e=>void 0!==e.value&&null!==e.value,eP=e=>JSON.stringify([e.key,eI(e)?e.value:null]),eD=(e,r)=>{var t;let a=eP(e[r]),l=e.slice(0,r).some(e=>eP(e)===a);return((t=e[r]).key?ez.test(t.key)?eI(t)?t.value?t.tag?null:"Enter a tag":"Enter a match value":t.tag_prefix?null:"Enter a tag prefix":"Use only letters, digits, and !#$%&'*+-.^_`|~ in the baggage key":"Enter a baggage key")??(l?"Another mapping already matches this key and value":null)};function eR({value:e,onChange:t,disabled:l=!1}){let i=Array.isArray(e)?e.filter(eK):[],d=(e,r)=>t(i.map((t,a)=>a===e?{...t,...r}:t));return(0,r.jsxs)("div",{className:"min-w-80 space-y-4 whitespace-normal",children:[(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"Exact value mappings take priority over any-value rules for the same key."}),i.map((e,c)=>{let u=eI(e),g=eD(i,c),p=`Mapping ${c+1}`;return(0,r.jsxs)("fieldset",{disabled:l,className:"space-y-3 rounded-lg border p-3 disabled:opacity-60",children:[(0,r.jsx)("legend",{className:"px-1 text-xs font-medium",children:p}),(0,r.jsxs)("div",{className:"flex items-center gap-2",children:[(0,r.jsxs)(n.d,{items:eq,value:u?"exact":"any",onValueChange:e=>d(c,"exact"===e?{value:"",tag:"",tag_prefix:void 0}:{value:void 0,tag:void 0,tag_prefix:""}),children:[(0,r.jsx)(n.P,{"aria-label":`${p} mode`,children:(0,r.jsx)(n.U,{})}),(0,r.jsx)(n.r,{children:eq.map(e=>(0,r.jsx)(n.T,{value:e.value,children:e.label},e.value))})]}),(0,r.jsx)(a.K,{type:"button",variant:"ghost",size:"icon","aria-label":`Delete ${p.toLowerCase()}`,onClick:()=>t(i.filter((e,r)=>r!==c)),children:(0,r.jsx)(m.f,{className:"h-4 w-4"})})]}),(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Baggage key",(0,r.jsx)(s.f,{"aria-label":`${p} baggage key`,placeholder:"service",value:e.key,onChange:e=>d(c,{key:e.target.value})})]}),u?(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Match value",(0,r.jsx)(s.f,{"aria-label":`${p} match value`,value:e.value??"",onChange:e=>d(c,{value:e.target.value})})]}),(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Tag",(0,r.jsx)(s.f,{"aria-label":`${p} tag`,value:e.tag??"",onChange:e=>d(c,{tag:e.target.value})})]}),(0,r.jsxs)("p",{className:"text-xs text-muted-foreground",children:[e.key,"=",e.value||"value"," → ",e.tag||"tag"]})]}):(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)("label",{className:"flex flex-col gap-1 text-xs",children:["Tag prefix",(0,r.jsx)(s.f,{"aria-label":`${p} tag prefix`,placeholder:"app:",value:e.tag_prefix??"",onChange:e=>d(c,{tag_prefix:e.target.value})})]}),(0,r.jsxs)("p",{className:"text-xs text-muted-foreground",children:[e.key,"=x → ",e.tag_prefix,"x"]})]}),g?(0,r.jsx)("p",{className:"text-xs text-destructive",children:g}):null,(0,r.jsxs)("label",{className:"flex items-center justify-between gap-3 text-xs",children:["Require a mapped value",(0,r.jsx)(o.f,{"aria-label":`${p} required`,checked:e.required??!1,onCheckedChange:e=>d(c,{required:e})})]}),(0,r.jsxs)("label",{className:"flex items-center justify-between gap-3 text-xs",children:["Require an existing tag budget",(0,r.jsx)(o.f,{"aria-label":`${p} require budget`,checked:e.require_budgeted_tag??!1,onCheckedChange:e=>d(c,{require_budgeted_tag:e})})]})]},c)}),l?(0,r.jsx)("p",{className:"text-xs text-muted-foreground",children:"Managed in config.yaml. Edit the YAML to change these rules."}):null,(0,r.jsx)(a.K,{disabled:l,type:"button",variant:"outline",onClick:()=>t([...i,{header:"baggage",key:"",value:"",tag:""}]),children:"Add mapping"})]})}let eU="enable_anthropic_prompt_caching",eV="anthropic_prompt_caching_ttl",eG="openai_system_messages_first",eB=e=>!0===e||"true"===e,eH="w-36",eX=e=>""===e?null:Number(e),eY=({setting:e,onChange:t})=>{if("request_tag_mappings"===e.field_name)return(0,r.jsx)(eR,{value:e.field_value,disabled:!1===e.editable,onChange:r=>t(e.field_name,r)});if("String"===e.field_type)return(0,r.jsx)(s.f,{"aria-label":e.field_name,value:e.field_value??"",onChange:r=>t(e.field_name,r.target.value)});if("Integer"===e.field_type)return(0,r.jsx)(s.f,{type:"number",step:1,className:eH,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))});if("Boolean"===e.field_type)return(0,r.jsx)(o.f,{checked:!0===e.field_value||"true"===e.field_value,onCheckedChange:r=>t(e.field_name,r)});if("Float"===e.field_type)return(0,r.jsx)(s.f,{type:"number",min:0,max:1,step:.05,className:eH,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))});if("Dollar"===e.field_type)return(0,r.jsxs)(i.H,{className:eH,children:[(0,r.jsx)(i.B,{children:"$"}),(0,r.jsx)(i.M,{type:"number",min:.01,step:.25,value:e.field_value??"",onChange:r=>t(e.field_name,eX(r.target.value))})]});if("List"===e.field_type){let a;return(0,r.jsx)(s.f,{"aria-label":e.field_name,placeholder:"Comma-separated values",defaultValue:Array.isArray(a=e.field_value)?a.join(", "):"",onChange:r=>{let a;return t(e.field_name,0===(a=r.target.value.split(",").map(e=>e.trim()).filter(e=>""!==e)).length?null:a)}},String(e.stored_in_db))}return"Select"===e.field_type?(0,r.jsxs)(n.d,{value:e.field_value??null,onValueChange:r=>t(e.field_name,r),children:[(0,r.jsx)(n.P,{className:"min-w-32",children:(0,r.jsx)(n.U,{placeholder:"Default"})}),(0,r.jsxs)(n.r,{children:[(0,r.jsx)(n.T,{value:null,children:"Default"}),(e.field_options??[]).map(e=>(0,r.jsx)(n.T,{value:e,children:e},e))]})]}):null},eJ=({accessToken:e,settings:t,onChange:a})=>{let s=t.find(e=>e.field_name===eU),i=t.find(e=>e.field_name===eV),d=t.find(e=>e.field_name===eG);if(!s)return null;let c=eB(s.field_value),m=(r,t)=>{a(r,t),""===t||null==t?(0,u.x5)(e,r):(0,u.Gy)(e,r,t)};return(0,r.jsx)(l.N,{children:(0,r.jsxs)(l.O,{children:[(0,r.jsx)(l.v,{children:"Prompt Caching"}),(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:"font-medium",children:"Automatic Anthropic prompt caching"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:s.field_description})]}),(0,r.jsx)(o.f,{checked:c,onCheckedChange:e=>m(eU,e)})]}),i&&(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:`font-medium ${c?"":"text-muted-foreground"}`,children:"Cache lifetime (TTL)"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:i.field_description})]}),(0,r.jsxs)(n.d,{disabled:!c,value:i.field_value??null,onValueChange:e=>m(eV,e),children:[(0,r.jsx)(n.P,{className:"min-w-40",children:(0,r.jsx)(n.U,{placeholder:"5m (default)"})}),(0,r.jsxs)(n.r,{children:[(0,r.jsx)(n.T,{value:null,children:"5m (default)"}),(i.field_options??[]).map(e=>(0,r.jsx)(n.T,{value:e,children:e},e))]})]})]}),d&&(0,r.jsxs)("div",{className:"mt-6 flex items-start justify-between gap-8",children:[(0,r.jsxs)("div",{className:"min-w-0 max-w-2xl",children:[(0,r.jsx)("p",{className:"font-medium",children:"System messages first for OpenAI"}),(0,r.jsx)("p",{className:"mt-1 break-words text-xs text-muted-foreground",children:d.field_description})]}),(0,r.jsx)(o.f,{"aria-label":"System messages first for OpenAI",checked:eB(d.field_value),onCheckedChange:e=>m(eG,e)})]})]})})};e.s(["T",0,eJ,"U",0,({accessToken:e,userRole:s,userID:i})=>{let[n,o]=(0,t.useState)([]);(0,t.useEffect)(()=>{e&&(0,u.T4)(e).then(e=>{o(e)})},[e]);let p=(e,r)=>{o(n.map(t=>t.field_name===e?{...t,field_value:r}:t))},f=async r=>{if(!e)return;let t=n.find(e=>e.field_name===r),a=t?.field_value;if(null==a){(t?.field_type==="Select"||t?.field_type==="List")&&x(r);return}try{await (0,u.Gy)(e,r,a);let t=n.map(e=>e.field_name===r?{...e,stored_in_db:!0}:e);o(t)}catch(e){}},x=r=>{if(e)try{(0,u.x5)(e,r);let t=n.map(e=>e.field_name===r?{...e,stored_in_db:null,field_value:e.field_default_value??null}:e);o(t)}catch(e){}};return e?(0,r.jsx)("div",{className:"w-full",children:(0,r.jsxs)(c.i,{defaultValue:"loadbalancing",className:"h-[75vh] w-full",children:[(0,r.jsxs)(c.U,{variant:"line",className:"mx-8 mt-4",children:[(0,r.jsx)(c.k,{value:"loadbalancing",children:"Loadbalancing"}),(0,r.jsx)(c.k,{value:"routing-groups",children:"Routing Groups"}),(0,r.jsx)(c.k,{value:"fallbacks",children:"Fallbacks"}),(0,r.jsx)(c.k,{value:"prompt-caching",children:"Prompt Caching"}),(0,r.jsx)(c.k,{value:"general",children:"General"})]}),(0,r.jsx)(c.S,{value:"loadbalancing",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(h,{accessToken:e,userRole:s,userID:i})}),(0,r.jsx)(c.S,{value:"routing-groups",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(eF,{})}),(0,r.jsx)(c.S,{value:"fallbacks",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(P,{accessToken:e,userRole:s,userID:i})}),(0,r.jsx)(c.S,{value:"prompt-caching",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(eJ,{accessToken:e,settings:n,onChange:p})}),(0,r.jsx)(c.S,{value:"general",className:"px-8 py-6",keepMounted:!0,children:(0,r.jsx)(l.N,{children:(0,r.jsx)(l.O,{children:(0,r.jsxs)(d.s,{children:[(0,r.jsx)(d.E,{children:(0,r.jsxs)(d.N,{children:[(0,r.jsx)(d.V,{children:"Setting"}),(0,r.jsx)(d.V,{children:"Value"}),(0,r.jsx)(d.V,{children:"Status"}),(0,r.jsx)(d.V,{children:"Action"})]})}),(0,r.jsx)(d.z,{children:n.filter(e=>"TypedDictionary"!==e.field_type&&"prompt_caching"!==e.field_tab).map((e,t)=>{let l;return(0,r.jsxs)(d.N,{children:[(0,r.jsxs)(d.d,{className:"whitespace-normal",children:[(0,r.jsx)("p",{className:"break-words",children:e.field_name}),(0,r.jsx)("p",{style:{fontSize:"0.65rem",color:"#808080",fontStyle:"italic"},className:"mt-1 break-words",children:e.field_description})]}),(0,r.jsx)(d.d,{children:(0,r.jsx)(eY,{setting:e,onChange:p})}),(0,r.jsx)(d.d,{children:!0==e.stored_in_db?(0,r.jsx)(g.f,{tone:"success",label:"In DB"}):!1==e.stored_in_db?(0,r.jsx)(g.f,{tone:"neutral",label:"In Config"}):(0,r.jsx)(g.f,{tone:"neutral",label:"Not Set"})}),(0,r.jsxs)(d.d,{children:[(0,r.jsx)(a.K,{disabled:"request_tag_mappings"===e.field_name&&(!1===e.editable||!(Array.isArray(l=e.field_value)&&l.every(eK)&&l.every((e,r)=>null===eD(l,r)))),onClick:()=>f(e.field_name),children:"Update"}),"request_tag_mappings"===e.field_name&&!1===e.editable?null:(0,r.jsx)("span",{onClick:()=>x(e.field_name),className:"inline-flex shrink-0 cursor-pointer items-center justify-center px-1.5 py-1.5 text-destructive",children:(0,r.jsx)(m.f,{className:"h-5 w-5 shrink-0"})})]})]},t)})})]})})})})]})}):null}],863679)}]); \ No newline at end of file diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index b14f43635b2..2f613396413 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -504,10 +504,13 @@ def _lazy_lock(app: "FastAPI", module_path: str) -> asyncio.Lock: def _register_feature(app: "FastAPI", feat: LazyFeature, module: object, features: tuple[LazyFeature, ...]) -> None: - before: Final = len(app.router.routes) + routes_before: Final = frozenset(id(route) for route in app.router.routes) feat.register_fn(app, module) lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType( - {**_lazy_routes(app), feat.module_path: tuple(app.router.routes[before:])} + { + **_lazy_routes(app), + feat.module_path: tuple(route for route in app.router.routes if id(route) not in routes_before), + } ) app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added app.router.routes[:] = _route_table( # rebind-ok: the app owns its route table diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5a80583b207..ab85b6f8842 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7454,6 +7454,31 @@ "claude_code_marketplace": { "components": { "schemas": { + "Body_upload_plugin_claude_code_plugins_upload_post": { + "properties": { + "file": { + "contentMediaType": "application/octet-stream", + "description": "Skill ZIP containing SKILL.md and supporting files, up to 10 MiB", + "title": "File", + "type": "string" + }, + "metadata": { + "contentMediaType": "application/json", + "contentSchema": { + "$ref": "#/components/schemas/UploadPluginRequest" + }, + "description": "JSON skill metadata without a source URL", + "title": "Metadata", + "type": "string" + } + }, + "required": [ + "file", + "metadata" + ], + "title": "Body_upload_plugin_claude_code_plugins_upload_post", + "type": "object" + }, "HTTPValidationError": { "properties": { "detail": { @@ -8057,6 +8082,137 @@ "title": "UpdatePluginRequest", "type": "object" }, + "UploadPluginRequest": { + "properties": { + "author": { + "anyOf": [ + { + "$ref": "#/components/schemas/PluginAuthor" + }, + { + "type": "null" + } + ], + "description": "Plugin author" + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin category", + "title": "Category" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin description", + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill domain (e.g., 'Productivity')", + "title": "Domain" + }, + "homepage": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Plugin homepage URL", + "title": "Homepage" + }, + "installation_preference": { + "anyOf": [ + { + "enum": [ + "available", + "auto_install", + "required" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Emitted as installationPreference on this plugin's marketplace.json entry. Needs an archive source, the only plugins Claude Desktop lists from a marketplace.json URL, and 'auto_install' or 'required' also need its sha256. Claude Desktop honors the mark only when the archive and marketplace.json share its inference gateway origin", + "title": "Installation Preference" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Search keywords", + "title": "Keywords" + }, + "name": { + "description": "Skill name (kebab-case)", + "pattern": "^[a-z0-9-]+$", + "title": "Name", + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Skill namespace within domain (e.g., 'workflows')", + "title": "Namespace" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "1.0.0", + "description": "Semantic version", + "title": "Version" + } + }, + "required": [ + "name" + ], + "title": "UploadPluginRequest", + "type": "object" + }, "ValidationError": { "properties": { "ctx": { @@ -8242,6 +8398,53 @@ ] } }, + "/claude-code/plugins/upload": { + "post": { + "description": "Create a skill and its uploaded ZIP atomically. Duplicate names return 409.", + "operationId": "upload_plugin_claude_code_plugins_upload_post", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_upload_plugin_claude_code_plugins_upload_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterPluginResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Upload Plugin", + "tags": [ + "claude_code_marketplace" + ] + } + }, "/claude-code/plugins/{plugin_name}": { "delete": { "description": "Delete a plugin from the marketplace.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: The name of the plugin to delete", @@ -8388,6 +8591,59 @@ ] } }, + "/claude-code/plugins/{plugin_name}/archive/{digest}.zip": { + "get": { + "description": "Download an uploaded ZIP under the same visibility policy as its skill.", + "operationId": "download_plugin_archive_claude_code_plugins__plugin_name__archive__digest__zip_get", + "parameters": [ + { + "in": "path", + "name": "plugin_name", + "required": true, + "schema": { + "title": "Plugin Name", + "type": "string" + } + }, + { + "in": "path", + "name": "digest", + "required": true, + "schema": { + "title": "Digest", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Download Plugin Archive", + "tags": [ + "claude_code_marketplace" + ] + } + }, "/claude-code/plugins/{plugin_name}/disable": { "post": { "description": "Disable a plugin without deleting it.\n\nRequires a proxy admin API key.\n\nParameters:\n - plugin_name: The name of the plugin to disable", @@ -10237,8 +10493,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -10256,7 +10519,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -10687,8 +10949,15 @@ "title": "Choice" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10718,8 +10987,7 @@ "required": [ "name", "choice", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIChoiceAnswer", "type": "object" @@ -10924,8 +11192,15 @@ "OpenAIScoreAnswer": { "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10959,8 +11234,7 @@ "required": [ "name", "score", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIScoreAnswer", "type": "object" @@ -10992,8 +11266,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -11034,7 +11315,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], @@ -19953,7 +20233,7 @@ "type": "null" } ], - "description": "Regex pattern applied to the tool's function name", + "description": "Regex pattern applied to the tool's name, or to its type when the tool has no name", "title": "Tool Name" }, "tool_type": { @@ -23542,8 +23822,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -23561,7 +23848,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -29400,8 +29686,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -29442,7 +29735,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], @@ -34331,6 +34623,7 @@ }, "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post", "responses": { "200": { @@ -34350,6 +34643,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post", "responses": { "200": { @@ -54536,6 +54830,7 @@ "paths": { "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post_2", "responses": { "200": { @@ -54555,6 +54850,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post_2", "responses": { "200": { @@ -54608,6 +54904,7 @@ }, "/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_realtime_calls_post", "responses": { "200": { @@ -54627,6 +54924,7 @@ }, "/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_realtime_client_secrets_post", "responses": { "200": { @@ -54678,8 +54976,489 @@ ] } }, + "/realtime/translations/calls": { + "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", + "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "WebRTC SDP answer" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/translations/client_secrets": { + "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", + "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_v1_realtime_calls_post", "responses": { "200": { @@ -54699,6 +55478,7 @@ }, "/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_v1_realtime_client_secrets_post", "responses": { "200": { @@ -54749,6 +55529,486 @@ "realtime" ] } + }, + "/v1/realtime/translations/calls": { + "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", + "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "WebRTC SDP answer" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/translations/client_secrets": { + "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", + "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c9960d794c6..1864c3ae82e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -38,7 +38,6 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_no_callback_env_reference, ) from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled -from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings from litellm.types.agents import ( AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export ) @@ -332,6 +331,8 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + "/realtime/translations", + "/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -342,6 +343,10 @@ class LiteLLMRoutes(enum.Enum): "/realtime/transcription_sessions", "/v1/realtime/transcription_sessions", "/openai/v1/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", @@ -461,6 +466,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/skills/{skill_id}", "/claude-code/marketplace.json", "/claude-code/plugins", + "/claude-code/plugins/upload", + "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", "/claude-code/plugins/{plugin_name}", ] @@ -2473,20 +2480,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ - accounting_capture: UsageCaptureSettings | None = Field( - None, - description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.", - ) - accounting_sync_ack: bool = Field( + use_spend_logs_ledger: bool = Field( False, - description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. " - "Other requests remain supported with asynchronous usage capture.", - ) - accounting_sync_ack_timeout_seconds: float = Field( - 20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval" - ) - accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field( - "fail_closed", description="Return 503 or log the persistence failure after the response wait expires" + description="Apply request accounting from stored SpendLogs deliveries. Enable only after all accounting writers are upgraded and ledger activation gates pass.", ) request_tag_mappings: Sequence[RequestTagMapping] | None = Field( @@ -3809,6 +3805,9 @@ class SpendLogsMetadata(TypedDict): class SpendLogsPayload(TypedDict): + accounting_payload: NotRequired[Mapping[str, object]] # writable-ok: capture before buffering + accounting_log_disabled: NotRequired[bool] # writable-ok: capture logging mode before buffering + accounting_delivery_id: NotRequired[str] # writable-ok: assigned once before buffering request_id: str call_type: str api_key: str @@ -4790,11 +4789,14 @@ class UnregisteredJWTClientBehavior(str, enum.Enum): - auto_register: Automatically create a new virtual key and mapping on first encounter. The new key has no budget/model restrictions; admins can tighten it later via /jwt_client/update. + - link_only: Map an eligible existing user key or return HTTP 403, without + creating a key. Proxy admins authenticate without a mapped key. """ FALLBACK_TEAM_MAPPING = "fallback_team_mapping" REJECT = "reject" AUTO_REGISTER = "auto_register" + LINK_ONLY = "link_only" class JWTIssuerConfig(LiteLLMBaseModel): @@ -5032,7 +5034,12 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): "What to do when virtual_key_claim_field is set but the JWT claim value " "has no registered mapping. 'fallback_team_mapping' (default): fall through " "to team-based JWT auth. 'reject': return HTTP 403. " - "'auto_register': auto-create a virtual key and mapping on first encounter." + "'auto_register': auto-create a virtual key and mapping on first encounter. " + "'link_only': map an eligible existing user key or return HTTP 403, without creating a key. " + "link_only ignores auto_register_map_existing_key; user provisioning still follows user_id_upsert. " + "Existing-key selection ignores allowed_routes; JWTs inherit the selected key's permissions, " + "including management routes, with request-time key restrictions still enforced. " + "Unmapped proxy admins authenticate directly in auto_register and link_only modes." ), ) auto_register_map_existing_key: bool = Field( @@ -5042,8 +5049,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): "field is the user_id_jwt_field or user_email_jwt_field, the JWT claim is mapped to a virtual key the " "JWT-resolved user already owns instead of minting a new one. If the user owns several, the most recently created key in the " "JWT-resolved team (or with no team when the JWT resolves none) is chosen among keys that never " - "expire, are not blocked, are not Admin UI session keys, were not minted by auto_register, and " - "have no allowed_routes or include llm_api_routes. Otherwise a new key is minted as usual." + "expire, are not blocked, are not Admin UI session keys, and were not minted by auto_register. " + "allowed_routes does not affect key selection. JWTs inherit the selected key's permissions, " + "including management routes; request-time key restrictions still apply. " + "Otherwise a new key is minted as usual." ), ) routing_overrides: list[JWTRoutingOverride] | None = Field( diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 3dfaa7f213e..f0a4d47f1e4 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -26,6 +26,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header +from litellm.llms.azure.common_utils import caused_by_entra_rejection from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.a2a.agent_card import point_agent_card_at_proxy @@ -1202,6 +1203,8 @@ async def invoke_agent_a2a( pass if isinstance(e, litellm.BadRequestError): return _jsonrpc_error(body.get("id"), -32602, e.message, 400) + if isinstance(e, litellm.AuthenticationError) and caused_by_entra_rejection(e): + return _jsonrpc_error(body.get("id"), -32000, e.message, 401) return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500) diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index cdbbd6029fa..57e99edecba 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -11,7 +11,15 @@ from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failur from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext -_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime")) +_MANAGED_REALTIME_ROUTES: Final = frozenset( + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime/translations", + "/v1/realtime/translations", + ) +) _MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions")) _MANAGED_MODEL_ROUTES: Final = frozenset( f"{prefix}/{operation}" @@ -112,7 +120,11 @@ def managed_inference_request( if route in _MANAGED_LIVE_ROUTES: return dict(body) if route in _MANAGED_REALTIME_ROUTES: - model: Final = query_model or body.get("model") + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + model: Final = resolve_realtime_route_model( + query_model if isinstance(query_model, str) else None, None, route.endswith("/realtime/translations") + ) if not isinstance(model, str) or not model: raise_identity_failure( AgentIdentityFailure(message="Managed inference requires an explicit or configured model") diff --git a/litellm/proxy/alerting_endpoints/__init__.py b/litellm/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..6a7a204b435 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/__init__.py @@ -0,0 +1 @@ +"""HTTP endpoints for gateway alerts.""" diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py new file mode 100644 index 00000000000..a69dd6ee025 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -0,0 +1,109 @@ +"""Shared acknowledgement state for model migration alerts.""" + +import hashlib +import json +from datetime import date, datetime, timezone +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import TypeAdapter + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.table_repositories import DailyTagSpendRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.model_deprecation import DeprecationAlertState, DeprecationAlertUpdate, DeprecationTagUsage + +router: Final = APIRouter() + + +def alert_state_repository(database: DatabaseDep) -> ConfigRepository: + if database is None: + raise HTTPException(status_code=503, detail="Alert state requires a database connection") + return ConfigRepository(database.prisma, use_writer=True) + + +def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository: + if database is None: + raise HTTPException(status_code=503, detail="Model usage tags require a database connection") + return DailyTagSpendRepository(database.prisma) + + +def _state_key(model: str, deprecation_date: date) -> str: + identity: Final = json.dumps([model, deprecation_date.isoformat()]) + digest: Final = hashlib.sha256(identity.encode()).hexdigest() + return f"model_deprecation_alert:{digest}" + + +def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if not write and user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + return + raise HTTPException(status_code=403, detail="Only gateway admins can manage model deprecation alerts") + + +@router.get("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def get_deprecation_alert_state( + model: Annotated[str, Query(min_length=1, max_length=1024)], + deprecation_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=False) + record: Final = await repository.get_param(_state_key(model, deprecation_date)) + if record is None: + return DeprecationAlertState(model=model, deprecation_date=deprecation_date) + return DeprecationAlertState.model_validate(record.param_value) + + +@router.put("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def update_deprecation_alert_state( + data: DeprecationAlertUpdate, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=True) + state: Final = DeprecationAlertState( + **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id + ) + await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) + return state + + +@router.get("/alerting/model-deprecations/tags", tags=["alerting"], response_model=tuple[DeprecationTagUsage, ...]) +async def get_deprecation_usage_tags( + model: Annotated[str, Query(min_length=1, max_length=1024)], + api_keys: Annotated[tuple[str, ...], Query(min_length=1, max_length=100)], + start_date: date, + end_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[DailyTagSpendRepository, Depends(alert_tag_repository)], +) -> tuple[DeprecationTagUsage, ...]: + """Distinct tags observed with this model for a page of affected reporting key IDs.""" + _require_admin(user, write=False) + if not 0 <= (end_date - start_date).days < 30: + raise HTTPException(status_code=422, detail="Select up to 30 consecutive calendar dates") + rows: Final = await repository.table.group_by( + by=["api_key", "tag"], + where={ + "api_key": { + # bounded-ok: query validation limits api_keys to 100 IDs. + "in": [api_key_from_bucket(key) for key in api_keys] + }, + "model": model, + "date": {"gte": start_date.isoformat(), "lte": end_date.isoformat()}, + "api_requests": {"gt": 0}, + "tag": {"not": None}, + }, + ) + tags: Final = TypeAdapter(tuple[DeprecationTagUsage, ...]).validate_python(rows) + return tuple( + sorted( + (DeprecationTagUsage(api_key=api_key_bucket(usage.api_key), tag=usage.tag) for usage in tags), + key=lambda usage: (usage.api_key, usage.tag), + ) + ) diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py index 8a3f71e656e..17e250a2dbe 100644 --- a/litellm/proxy/analytics_endpoints/cache_activity.py +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -2,31 +2,22 @@ import asyncio import json from collections.abc import Sequence from datetime import datetime -from typing import Final, Protocol +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter from litellm.proxy._types import LiteLLMRoutes from litellm.proxy.auth.authorization import ReadScope, read_scope_bind_params +from litellm.proxy.utils import query_raw_within_statement_timeout from litellm.types.llms.base import LiteLLMBaseModel +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + UNKNOWN_CALL_TYPE: Final = "Unknown" INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value) -class _SupportsQueryRaw(Protocol): - """The single database operation the cache-activity queries issue.""" - - async def query_raw(self, query: str, *args: object) -> Sequence[object]: ... - - -class _SupportsRawQueryDb(Protocol): - """A prisma client handle, narrowed to the raw-query surface used here.""" - - @property - def db(self) -> _SupportsQueryRaw: ... - - class CacheActivityGroup(LiteLLMBaseModel): call_type: str api_requests: int @@ -174,7 +165,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: async def get_cache_activity( - prisma_client: _SupportsRawQueryDb, + prisma_client: "PrismaClient", read_scope: ReadScope, start_date: datetime, end_date: datetime, @@ -185,14 +176,46 @@ async def get_cache_activity( models_json: Final = json.dumps(list(models)) scope_params: Final = read_scope_bind_params(read_scope) group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather( - prisma_client.db.query_raw( - GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params + query_raw_within_statement_timeout( + prisma_client, + GROUPS_SQL, + start_date, + end_date, + key_aliases_json, + models_json, + INFO_ROUTES_JSON, + *scope_params, + what="Cache activity", ), - prisma_client.db.query_raw( - ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params + query_raw_within_statement_timeout( + prisma_client, + ERROR_BREAKDOWN_SQL, + start_date, + end_date, + key_aliases_json, + models_json, + INFO_ROUTES_JSON, + *scope_params, + what="Cache activity", + ), + query_raw_within_statement_timeout( + prisma_client, + KEY_ALIAS_OPTIONS_SQL, + start_date, + end_date, + INFO_ROUTES_JSON, + *scope_params, + what="Cache activity", + ), + query_raw_within_statement_timeout( + prisma_client, + MODEL_OPTIONS_SQL, + start_date, + end_date, + INFO_ROUTES_JSON, + *scope_params, + what="Cache activity", ), - prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params), - prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params), ) groups: Final = _groups_adapter.validate_python(group_rows or []) return CacheActivityResponse( diff --git a/litellm/proxy/anthropic_endpoints/answered_status.py b/litellm/proxy/anthropic_endpoints/answered_status.py new file mode 100644 index 00000000000..26eeb08c8c9 --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/answered_status.py @@ -0,0 +1,53 @@ +from typing import Final, TypeVar + +from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError + +_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529 +_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503 + +_Raised = TypeVar("_Raised", bound=Exception) + + +def is_provider_overload(exc: Exception) -> bool: + return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE + + +def with_openai_overload_status(exc: _Raised) -> _Raised: + return ( + _copy_carrying( + _copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE), + "status_code", + _OPENAI_OVERLOAD_ANSWERED_STATUS_CODE, + ) + if is_provider_overload(exc) + else exc + ) + + +def _provider_status_code(exc: Exception) -> int | None: + unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal + while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None: + unwrapped = unwrapped.original_exception + match unwrapped: + case InternalServerError(provider_status_code=int() as provider_status): + return provider_status + case APIError(provider_status_code=int() as provider_status): + return provider_status + case APIError(status_code=int() as provider_status): + return provider_status + case _: + return None + + +def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised: + answered: Final = type(exc).__new__(type(exc), *exc.args) + answered.__dict__.update({**exc.__dict__, attribute: value}) + answered.__cause__ = exc.__cause__ + answered.__context__ = exc.__context__ + answered.__suppress_context__ = exc.__suppress_context__ + return answered.with_traceback(exc.__traceback__) + + +def with_answered_status(exc: Exception) -> Exception: + provider_status_code: Final = _provider_status_code(exc) + return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code) diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index fc82686947d..b8c90fe7e93 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -18,29 +18,40 @@ Endpoints: /claude-code/plugins/{name} - DELETE - Delete a plugin (proxy admin only) """ +import asyncio +import hashlib import json import re from collections.abc import Mapping, Sequence from datetime import datetime, timezone from types import MappingProxyType from typing import Annotated, Final, Protocol, TypedDict, get_args -from urllib.parse import urlsplit -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import JSONResponse -from pydantic import TypeAdapter +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import JSONResponse, Response +from pydantic import Json, TypeAdapter from typing_extensions import ReadOnly from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.url_utils import is_https_url_with_host +from litellm.models.plugin_upload import PluginUpload from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import ( SkillVisibility, resolve_skill_visibility, ) +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.skill_upload_archive import ( + MAX_SKILL_UPLOAD_BYTES, + build_uploaded_skill_archive, +) from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required +from litellm.proxy.common_utils.claude_code_marketplace_auth import ( + marketplace_auth_required, + marketplace_key_header_present, +) from litellm.proxy.common_utils.resource_ownership import is_proxy_admin +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( @@ -52,6 +63,7 @@ from litellm.types.proxy.claude_code_endpoints import ( RegisterPluginRequest, RegisterPluginResponse, UpdatePluginRequest, + UploadPluginRequest, ) router: Final = APIRouter() @@ -148,6 +160,117 @@ async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAu return None +@router.post( + "/claude-code/plugins/upload", + tags=["Claude Code Marketplace"], + response_model=RegisterPluginResponse, +) +async def upload_plugin( + request: Request, + uploads: PluginUploadRepositoryDep, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + file: Annotated[UploadFile, File(description="Skill ZIP containing SKILL.md and supporting files, up to 10 MiB")], + metadata: Annotated[Json[UploadPluginRequest], Form(description="JSON skill metadata without a source URL")], +) -> RegisterPluginResponse: + """Create a skill and its uploaded ZIP atomically. Duplicate names return 409.""" + from litellm.proxy.utils import get_custom_url + + _require_proxy_admin(user_api_key_dict) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) + content: Final = await file.read(MAX_SKILL_UPLOAD_BYTES + 1) + if len(content) > MAX_SKILL_UPLOAD_BYTES: + raise _error_response(413, "Skill ZIP must be 10 MiB or smaller") + archive: Final = await asyncio.to_thread(build_uploaded_skill_archive, content, metadata.name) + if archive is None: + raise _error_response( + 400, "Upload a valid ZIP with SKILL.md at the root or inside one skill folder, and safe supporting files" + ) + digest: Final = hashlib.sha256(archive).hexdigest() + source: Final = { + "source": "archive", + "url": get_custom_url( + request_base_url=str(request.base_url), + route=f"claude-code/plugins/{metadata.name}/archive/{digest}.zip", + ), + "sha256": digest, + } + spec: Final = RegisterPluginRequest.model_validate({**metadata.model_dump(), "source": source}) + _validate_installation_preference(spec) + plugin_id: Final = await uploads.create( + PluginUpload( + name=spec.name, + version=spec.version, + description=spec.description, + manifest=_build_plugin_manifest(spec.name, spec), + content=archive, + created_by=user_api_key_dict.user_id, + ) + ) + if plugin_id is None: + raise _name_conflict_error(spec.name) + return RegisterPluginResponse( + status="success", + action="created", + plugin=PluginResponse( + id=plugin_id, + name=spec.name, + version=spec.version, + description=spec.description, + source=source, + installation_preference=spec.installation_preference, + enabled=True, + ), + ) + + +@router.get( + "/claude-code/plugins/{plugin_name}/archive/{digest}.zip", + tags=["Claude Code Marketplace"], + response_class=Response, + responses={200: {"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}}}}, +) +async def download_plugin_archive( + request: Request, + plugin_name: str, + digest: str, + uploads: PluginUploadRepositoryDep, +) -> Response: + """Download an uploaded ZIP under the same visibility policy as its skill.""" + from litellm.proxy.utils import get_custom_url + + caller: Final = ( + await _header_key_caller(request) + if marketplace_key_header_present(request) + else await _marketplace_caller(request, None) + ) + visibility: Final = await resolve_skill_visibility(caller) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) + archive: Final = await uploads.get_metadata(plugin_name) + expected_url: Final = get_custom_url( + request_base_url=str(request.base_url), route=f"claude-code/plugins/{plugin_name}/archive/{digest}.zip" + ) + if ( + archive is None + or not visibility.allows(archive) + or archive.source != {"source": "archive", "url": expected_url, "sha256": digest} + ): + raise _error_response(404, "Skill archive not found") + content: Final = await uploads.get_content(archive) + if content is None: + raise _error_response(404, "Skill archive not found") + return Response( + content=content, + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{archive.name}.zip"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) + + @router.get( "/claude-code/marketplace.json", tags=["Claude Code Marketplace"], @@ -254,14 +377,6 @@ _VALID_GIT_SUBDIR_PATH_RE: Final = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a- _VALID_SHA256_RE: Final = re.compile(r"^[0-9a-fA-F]{64}$") -def _is_https_url_with_host(url: str) -> bool: - try: - parts: Final = urlsplit(url) - except ValueError: - return False - return parts.scheme == "https" and bool(parts.hostname) - - def _validate_plugin_source(source: Mapping[str, str]) -> None: """Validate plugin source format, raising HTTPException on invalid input.""" source_type: Final = source.get("source") @@ -298,7 +413,7 @@ def _validate_plugin_source(source: Mapping[str, str]) -> None: }, ) elif source_type == "archive": - if not _is_https_url_with_host(source.get("url", "")): + if not is_https_url_with_host(source.get("url", "")): raise HTTPException( status_code=400, detail={ @@ -680,18 +795,19 @@ async def update_plugin( prisma_client: Final = await get_prisma_client() - _validate_plugin_source(request.source) - _validate_installation_preference(request) - - existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( - where={"name": plugin_name} - ) + repository: Final = ClaudeCodePluginRepository(prisma_client, use_writer=True) + existing: Final[_PluginRecord | None] = await repository.table.find_unique(where={"name": plugin_name}) if not existing: raise _error_response(404, f"Plugin '{plugin_name}' not found") + current_manifest: Final = MANIFEST_FIELDS.validate_json(existing.manifest_json or "{}") + if request.source != current_manifest.get("source"): + _validate_plugin_source(request.source) + _validate_installation_preference(request) + manifest: Final[Mapping[str, object]] = _build_plugin_manifest(plugin_name, request) - plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.update( + plugin: Final[_PluginRecord | None] = await repository.table.update( where={"name": plugin_name}, data={ "version": request.version, diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py index 9e5b55e7e36..4775498dec3 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py @@ -19,8 +19,11 @@ if TYPE_CHECKING: class _SkillRecord(Protocol): - name: str - enabled: bool + @property + def name(self) -> str: ... + + @property + def enabled(self) -> bool: ... def _skills_of(permission: LiteLLM_ObjectPermissionTable | None) -> frozenset[str]: diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/skill_upload_archive.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/skill_upload_archive.py new file mode 100644 index 00000000000..48af819b673 --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/skill_upload_archive.py @@ -0,0 +1,30 @@ +import json +import re +import struct +import zipfile +from typing import Final + +from litellm.proxy.discovery_endpoints.agent_skills_archive import pack_skill_archive, read_skill_archive_members + +MAX_SKILL_UPLOAD_BYTES: Final = 10 * 1024 * 1024 +MAX_SKILL_ARCHIVE_COMPRESSION_RATIO: Final = 50 + + +def build_uploaded_skill_archive(content: bytes, name: str) -> bytes | None: + if len(content) > MAX_SKILL_UPLOAD_BYTES or re.fullmatch(r"[a-z0-9-]+", name) is None: + return None + try: + skill: Final = read_skill_archive_members(content, for_upload=True) + if skill is None: + return None + members: Final = ((".claude-plugin/plugin.json", json.dumps({"name": name}).encode(), 0o644),) + tuple( + (f"skills/{name}/{path}", data, mode) for path, data, mode in skill + ) + compressed: Final = pack_skill_archive(members) + return ( + pack_skill_archive(members, zipfile.ZIP_STORED) + if sum(len(data) for _, data, _ in members) > len(compressed) * MAX_SKILL_ARCHIVE_COMPRESSION_RATIO + else compressed + ) + except (struct.error, UnicodeEncodeError): + return None diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index baa6602b042..7e990e3c71b 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -16,12 +16,14 @@ from litellm.anthropic_interface.exceptions import ( AnthropicExceptionMapping, ) from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) from litellm.proxy._types import * +from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -48,6 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.common_utils.token_count_log import rejected_token_count from litellm.proxy.common_utils.token_count_settings import count_source_fields +from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.litellm_pre_call_utils import ( forwarded_anthropic_oauth_token, should_auto_drop_params_for_agentic_cli, @@ -225,15 +228,18 @@ async def anthropic_response( request_id=request.headers.get("x-request-id"), ) return JSONResponse(status_code=e.status_code, content=body) - except Exception as e: + except Exception as raised: + answered: Final = with_answered_status(raised) await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data + user_api_key_dict=user_api_key_dict, + original_exception=answered, + request_data=base_llm_response_processor.data, ) - log_llm_api_exception(e, base_llm_response_processor.litellm_call_id) + log_llm_api_exception(answered, base_llm_response_processor.litellm_call_id) - if isinstance(e, ProxyException): + if isinstance(answered, ProxyException): return _anthropic_error_json_response( - with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request + with_litellm_call_id(answered, base_llm_response_processor.litellm_call_id), request ) # Extract model_id from request metadata (same as success path) @@ -247,23 +253,27 @@ async def anthropic_response( call_id=base_llm_response_processor.litellm_call_id, model_id=model_id, version=version, - response_cost=0, + response_cost=( + failure_row_response_cost(base_llm_response_processor.data) + if is_agentic_followup_failure(answered) + else 0 + ), model_region=getattr(user_api_key_dict, "allowed_model_region", ""), request_data=data, - timeout=getattr(e, "timeout", None), + timeout=getattr(answered, "timeout", None), litellm_logging_obj=None, ) - if isinstance(e, HTTPException): - return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request) + if isinstance(answered, HTTPException): + return _anthropic_error_json_response(proxy_exception_from_http_exception(answered, headers), request) - error_msg: Final = f"{e}" + error_msg: Final = f"{answered}" return _anthropic_error_json_response( ProxyException( - message=getattr(e, "message", error_msg), - type=openai_error_type(e, error_status_code(e, 500)), - param=openai_error_param(e), - code=error_status_code(e, 500), + message=getattr(answered, "message", error_msg), + type=openai_error_type(answered, error_status_code(answered, 500)), + param=openai_error_param(answered), + code=error_status_code(answered, 500), headers=headers, ), request, @@ -383,8 +393,9 @@ async def count_tokens( except ProxyException as e: return _anthropic_error_json_response(e, request) except Exception as e: - log_llm_api_exception(e, litellm_call_id) - return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) + answered: Final = with_answered_status(e) + log_llm_api_exception(answered, litellm_call_id) + return _anthropic_error_json_response(proxy_exception_from_route_error(answered), request) else: return { "input_tokens": token_response.total_tokens, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fdc8aa82268..af60a315c60 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -43,7 +43,9 @@ from litellm.constants import ( REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, decisions_request_price from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.tool_fields import DropPolicy @@ -572,7 +574,6 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader return False continue try: - # Use the router registry's group info directly for better reliability model_group_info = llm_router.registry().group(model_name, include_hidden=True) if model_group_info is None: @@ -730,7 +731,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) - def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool: - """Whether any of the deployments carries an explicit positive per-token price. + """Whether any of the deployments carries an explicit positive per-token or per-query price. The group's price is read from the deployments named after the routed group and from that group's own alias target, while an alias chain is served from the deployments named @@ -741,14 +742,21 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") if model_id is None: continue - raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) - if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost( - raw_entry.get("output_cost_per_token") + raw_entry = as_str_mapping(litellm.model_cost.get(model_id)) or _EMPTY_COST_ENTRY + if ( + _is_positive_cost(raw_entry.get("input_cost_per_token")) + or _is_positive_cost(raw_entry.get("output_cost_per_token")) + or _is_positive_cost(_deployment_query_price(deployment["litellm_params"], raw_entry)) ): return True return False +def _deployment_query_price(litellm_params: Mapping[str, object], cost_entry: Mapping[str, object]) -> float | None: + provider: Final = litellm_params.get("custom_llm_provider") or str(litellm_params.get("model")).partition("/")[0] + return decisions_request_price({**cost_entry, "litellm_provider": provider}, decisions_pricing_fields(cost_entry)) + + _EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({}) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9350c9c935c..161ff2fdfcd 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1971,6 +1971,7 @@ _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",) # can_key_call_model() validates the real target model. _MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS: Final = ( "/realtime/client_secrets", + "/realtime/translations/client_secrets", "/realtime/calls", ) _MODEL_ROUTING_ID_FIELDS: Final = ( diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 4bcf020a9c4..73edcd66bd7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -6,7 +6,7 @@ import copy import functools import itertools import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from types import MappingProxyType from typing import Final, TypeVar @@ -23,7 +23,12 @@ from litellm.router import Router from litellm.router_api import fallback_chain from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order +from litellm.utils import ( + AZURE_MODEL_PLACEHOLDER, + ProviderConfigManager, + get_valid_models, + potential_model_names_in_lookup_order, +) LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value}) @@ -35,7 +40,9 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing") -_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value}) +_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset( + {LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} +) _Listed: Final = TypeVar("_Listed") @@ -373,6 +380,10 @@ def _ids_routing_to_catalog_models( catalog_ids: Sequence[str], routes_to_model: Callable[[str, str], bool], ) -> tuple[str, ...]: + if name_pattern.count("*") > deployment_pattern.count("*"): + return _catalog_ids_a_wider_name_matches( + name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments( name_pattern, deployment_pattern ): @@ -417,6 +428,23 @@ def _ids_routing_to_catalog_models( ) +def _catalog_ids_a_wider_name_matches( + name_pattern: str, + deployment_pattern: str, + provider: str, + catalog_ids: Sequence[str], + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: + name_regex: Final = _wildcard_name_regex(name_pattern) + return tuple( + catalog_id + for catalog_id in _ids_routing_to_catalog_models( + deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) + if name_regex.fullmatch(catalog_id) + ) + + def _model_info_field(deployment: Mapping[str, object], field: str) -> object: model_info: Final = deployment.get("model_info") return model_info.get(field) if is_str_keyed_mapping(model_info) else None @@ -454,11 +482,12 @@ def _routes_to_model( return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id) -def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None: +def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model): return None + name_suffix: Final = wildcard_model.split("/", 1)[-1] deployment_suffix: Final = litellm_params.model.partition("/")[2] - return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None + return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None def get_known_models_from_wildcard( @@ -466,6 +495,43 @@ def get_known_models_from_wildcard( litellm_params: LiteLLM_Params | None = None, routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, ) -> list[str]: + provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params) + if litellm_params is None or provider_id_prefix is None: + return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model)) + return [ + listed_id.removeprefix(provider_id_prefix) + for listed_id in _known_models_from_wildcard( + wildcard_model, + litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}), + lambda listed_id, model: routes_to_model( + listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix) + ), + ) + ] + + +def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: + if ( + litellm_params is None + or not litellm_params.custom_llm_provider + or "/" in wildcard_model + or wildcard_model == "*" + or "/" in litellm_params.model + or not _check_wildcard_routing(litellm_params.model) + ): + return None + return f"{litellm_params.custom_llm_provider}/" + + +def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]: + return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER) + + +def _known_models_from_wildcard( + wildcard_model: str, + litellm_params: LiteLLM_Params | None, + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: wildcard_model_to_expand: Final = ( litellm_params.model if wildcard_model == "*" @@ -474,15 +540,14 @@ def get_known_models_from_wildcard( and "/" in litellm_params.model else wildcard_model ) - try: - wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) - except ValueError: # safely fail - return [] deployment_pattern_naming_listed_ids: Final = ( wildcard_model_to_expand if wildcard_model_to_expand != wildcard_model - else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params) ) + wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/") + if not has_provider_prefix and deployment_pattern_naming_listed_ids is None: + return () # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -499,16 +564,14 @@ def get_known_models_from_wildcard( wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) if wildcard_models is None: - return [] + return () if deployment_pattern_naming_listed_ids is not None: - return list( - _ids_routing_to_catalog_models( - name_pattern=wildcard_model, - deployment_pattern=deployment_pattern_naming_listed_ids, - provider=provider, - catalog_ids=wildcard_models, - routes_to_model=routes_to_model, - ) + return _ids_routing_to_catalog_models( + name_pattern=wildcard_model, + deployment_pattern=deployment_pattern_naming_listed_ids, + provider=provider, + catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models), + routes_to_model=routes_to_model, ) if wildcard_suffix != "*": model_prefix: Final = wildcard_suffix.replace("*", "") @@ -524,7 +587,7 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - return list( + return tuple( dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) ) @@ -562,9 +625,17 @@ def _names_no_deployment_serves_by_name( return tuple(name for name in names if not deployed_by_name(name, team_id)) +def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool: + if allowed_model_names is None: + return True + pattern: Final = _wildcard_name_regex(wildcard_pattern) + return any(pattern.match(name) for name in allowed_model_names) + + def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], llm_router: Router | None = None, + allowed_model_names: Collection[str] | None = None, ) -> list[dict[str, object]]: """Expand wildcard deployments into one row per known provider model. @@ -581,19 +652,11 @@ def expand_wildcard_deployments_for_model_info( raw_params = deployment.get("litellm_params") litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} litellm_model = str(litellm_params_dict.get("model") or "") - - # Determine the wildcard pattern to expand. - # Branch order matters: only fall to litellm_model when model_name is - # also a wildcard, so a concrete model_name is never overwritten. - if _check_wildcard_routing(model_name) and ("/" in model_name or model_name == "*"): - wildcard_pattern = model_name - elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model): - wildcard_pattern = litellm_model - elif _check_wildcard_routing(model_name): - wildcard_pattern = model_name - else: + if not _check_wildcard_routing(model_name): expanded.append(deployment) continue + if not _can_expand_to_an_allowed_name(model_name, allowed_model_names): + continue try: litellm_params = LiteLLM_Params.model_validate(litellm_params_dict) if litellm_params_dict else None @@ -601,7 +664,7 @@ def expand_wildcard_deployments_for_model_info( expanded.append(deployment) continue expanded_names = get_known_models_from_wildcard( - wildcard_model=wildcard_pattern, + wildcard_model=model_name, litellm_params=litellm_params, routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a863de8b0f8..82135cb1b74 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -13,6 +13,7 @@ import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from enum import Enum from types import MappingProxyType from typing import Final, NamedTuple, Protocol, Union, cast @@ -148,6 +149,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg populate_request_with_path_params, read_raw_json_body, read_request_body, + resolve_realtime_route_model, rewrite_request_model, safe_get_request_headers, safe_get_request_query_params, @@ -1246,11 +1248,21 @@ def _apply_budget_limits_to_end_user_params( async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model")) + return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model", "")) + + +async def user_api_key_auth_realtime_websocket(websocket: WebSocket) -> UserAPIKeyAuth: + route_path: Final[object] = websocket.scope.get("path") + model: Final = resolve_realtime_route_model( + model=websocket.query_params.get("model"), + intent=websocket.query_params.get("intent"), + is_translation=isinstance(route_path, str) and route_path.endswith("/realtime/translations"), + ) + return await user_api_key_auth_websocket_for_model(websocket, model=model or "") async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=None) + return await user_api_key_auth_websocket_for_model(websocket, model="") def _has_required_baggage_tag_mapping() -> bool: @@ -1266,7 +1278,7 @@ def _has_required_baggage_tag_mapping() -> bool: async def user_api_key_auth_websocket_for_model( - websocket: WebSocket, model: str | None, *, close_on_auth_error: bool = True + websocket: WebSocket, model: str, *, close_on_auth_error: bool = True ) -> UserAPIKeyAuth: ws_scope: Final = websocket.scope or {} scope_headers: Final = list(ws_scope.get("headers") or []) @@ -1565,10 +1577,14 @@ _JWT_AUTH_DISABLED_HINT = ( ) +class _JWTKeyMappingFailure(Enum): + NO_ELIGIBLE_KEY = "JWT Key Mapping: No eligible existing virtual key for this identity. Access denied." + + class _PendingAutoRegister(NamedTuple): """ Signal returned by ``_resolve_jwt_to_virtual_key`` when the JWT's claim is - unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER. + unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER or LINK_ONLY. The caller MUST run standard ``JWTAuthManager.auth_builder`` to apply RBAC, scope mappings, ``custom_validate``, and ``user_allowed_email_domain`` @@ -1582,23 +1598,22 @@ class _PendingAutoRegister(NamedTuple): claim_value: str cache_key: str jwt_issuer: str | None = None + behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER def _claim_identifies_user(jwt_handler: JWTHandler, claim_field: str, jwt_issuer: str | None) -> bool: - if not jwt_handler.litellm_jwtauth.auto_register_map_existing_key: - return False if jwt_handler.litellm_jwtauth.is_user_identity_claim(claim_field, jwt_issuer): return True verbose_proxy_logger.warning( - "JWT Key Mapping (auto_register_map_existing_key): claim '%s' is not the user_id or user_email JWT field " - "and may be shared by several users, so a new key is minted instead of reusing one the user owns.", + "JWT Key Mapping: claim '%s' is not the user_id or user_email JWT field " + "and may be shared by several users, so existing user keys cannot be linked automatically.", claim_field, ) return False async def _reusable_key_hash_for_user(prisma_client: PrismaClient, user_id: str, team_id: str | None) -> str | None: - key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_llm_api_key(user_id, team_id) + key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_key(user_id, team_id) return None if key is None else key.token @@ -1617,11 +1632,13 @@ async def _auto_register_jwt_mapping( org_id: str | None = None, end_user_id: str | None = None, agent_id: str | None = None, -) -> UserAPIKeyAuth | None: + behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER, +) -> UserAPIKeyAuth | _JWTKeyMappingFailure | None: """ Auto-register: create a new virtual key + mapping for an unrecognised JWT claim value, or point the mapping at a key the resolved user already owns - when ``auto_register_map_existing_key`` is set. ``team_id`` and ``user_id`` + when ``auto_register_map_existing_key`` is set. LINK_ONLY always searches + for an existing key and rejects when none is eligible. ``team_id`` and ``user_id`` must come from a successful ``JWTAuthManager.auth_builder`` run — they encode the JWT identity AFTER RBAC/scope/custom_validate/email-domain policy has been enforced. The key @@ -1639,11 +1656,21 @@ async def _auto_register_jwt_mapping( generate_key_helper_fn, ) + link_only: Final = behavior == UnregisteredJWTClientBehavior.LINK_ONLY + reuse_existing: Final = link_only or jwt_handler.litellm_jwtauth.auto_register_map_existing_key existing_token_hash: Final = ( - await _reusable_key_hash_for_user(prisma_client, user_id, team_id) - if user_id is not None and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer) + await _reusable_key_hash_for_user( + prisma_client, + user_id, + team_id, + ) + if reuse_existing + and user_id is not None + and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer) else None ) + if link_only and existing_token_hash is None: + return _JWTKeyMappingFailure.NO_ELIGIBLE_KEY minted: Final = existing_token_hash is None if existing_token_hash is not None: token_hash = existing_token_hash @@ -1680,8 +1707,8 @@ async def _auto_register_jwt_mapping( "jwt_claim_name": virtual_key_claim_field, "jwt_claim_value": claim_value, "token": token_hash, - "created_by": "auto_register", - "updated_by": "auto_register", + "created_by": behavior.value, + "updated_by": behavior.value, } ) except Exception as e: @@ -1693,8 +1720,8 @@ async def _auto_register_jwt_mapping( # the DB and the cleartext is in memory on this request. Delete it # so orphans don't accumulate under sustained concurrency. verbose_proxy_logger.debug( - "JWT Key Mapping (auto_register): unique conflict on create — " - "deleting orphaned virtual key and fetching winner's mapping for %s='%s'.", + "JWT Key Mapping (%s): unique conflict on create, fetching winner's mapping for %s='%s'.", + behavior.value, virtual_key_claim_field, claim_value, ) @@ -1724,7 +1751,7 @@ async def _auto_register_jwt_mapping( raise HTTPException( status_code=503, detail=( - "JWT Key Mapping: AUTO_REGISTER race resolution failed — " + f"JWT Key Mapping: {behavior.value.upper()} race resolution failed — " "winner's mapping was concurrently removed. Retry the request." ), ) @@ -1738,7 +1765,8 @@ async def _auto_register_jwt_mapping( ) verbose_proxy_logger.info( - "JWT Key Mapping (auto_register): %s virtual key for %s='%s'.", + "JWT Key Mapping (%s): %s virtual key for %s='%s'.", + behavior.value, "created new" if minted else "mapped existing", virtual_key_claim_field, claim_value, @@ -1810,7 +1838,7 @@ async def _resolve_jwt_to_virtual_key( - ``UserAPIKeyAuth``: a resolved virtual key (cache hit or DB hit). The caller may use this directly; JWT policy has been enforced previously (at key-creation time or, for cached results, before caching). - - ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER. + - ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER or LINK_ONLY. The caller MUST run ``JWTAuthManager.auth_builder`` to enforce JWT policy (RBAC, scope, custom_validate, email-domain), then invoke ``_auto_register_jwt_mapping`` with the validated team_id/user_id. @@ -1818,7 +1846,7 @@ async def _resolve_jwt_to_virtual_key( The caller falls through to standard team-based JWT auth (which itself enforces full JWT policy via auth_builder). - Raises HTTPException: REJECT policy hit, missing claim under - REJECT/AUTO_REGISTER, or other policy violations. + REJECT/AUTO_REGISTER/LINK_ONLY, or other policy violations. """ raw_issuer: Final = jwt_claims.get(JWTHandler.LITELLM_JWT_ISSUER_CLAIM) normalized_issuer: Final = raw_issuer if isinstance(raw_issuer, str) else None @@ -1845,6 +1873,7 @@ async def _resolve_jwt_to_virtual_key( if behavior in ( UnregisteredJWTClientBehavior.REJECT, UnregisteredJWTClientBehavior.AUTO_REGISTER, + UnregisteredJWTClientBehavior.LINK_ONLY, ): raise HTTPException( status_code=403, @@ -1876,7 +1905,7 @@ async def _resolve_jwt_to_virtual_key( status_code=403, detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.", ) - if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER: + if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY): # Stale sentinel written under a prior fallback_team_mapping config — # evict it and defer auto-register to after auth_builder runs. Raise # the same 500 as the fresh-path AUTO_REGISTER branch when there is @@ -1886,7 +1915,7 @@ async def _resolve_jwt_to_virtual_key( raise HTTPException( status_code=500, detail=( - "JWT Key Mapping: AUTO_REGISTER requires a database connection. " + f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. " "Configure a database or change unregistered_jwt_client_behavior." ), ) @@ -1896,6 +1925,7 @@ async def _resolve_jwt_to_virtual_key( claim_value=str(claim_value), cache_key=cache_key, jwt_issuer=normalized_issuer, + behavior=behavior, ) return None elif cached_mapping is not None: @@ -1956,12 +1986,12 @@ async def _resolve_jwt_to_virtual_key( detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.", ) - if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER: + if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY): if prisma_client is None: raise HTTPException( status_code=500, detail=( - "JWT Key Mapping: AUTO_REGISTER requires a database connection. " + f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. " "Configure a database or change unregistered_jwt_client_behavior." ), ) @@ -1974,6 +2004,7 @@ async def _resolve_jwt_to_virtual_key( claim_value=str(claim_value), cache_key=cache_key, jwt_issuer=normalized_issuer, + behavior=behavior, ) # FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the @@ -2466,10 +2497,13 @@ async def user_api_key_auth_builder( org_id=org_id, end_user_id=end_user_id, agent_id=agent_id, + behavior=pending_auto_register.behavior, ) if pending_auto_register is not None and prisma_client is not None else None ) + if isinstance(auto_registered, _JWTKeyMappingFailure): + raise HTTPException(status_code=403, detail=auto_registered.value) if auto_registered is not None: auto_registered.jwt_claims = jwt_claims auto_registered.user_email = user_email @@ -2485,7 +2519,13 @@ async def user_api_key_auth_builder( falls_through_to_key_checks: Final = ( auto_registered is not None - and jwt_handler.litellm_jwtauth.auto_register_map_existing_key + and ( + jwt_handler.litellm_jwtauth.auto_register_map_existing_key + or ( + pending_auto_register is not None + and pending_auto_register.behavior == UnregisteredJWTClientBehavior.LINK_ONLY + ) + ) and master_key is not None ) if not falls_through_to_key_checks: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 994b3a96799..e812b03385d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -55,6 +55,7 @@ from litellm.constants import ( from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.bug_report import ( allowlisted, bug_report_notice, @@ -94,6 +95,7 @@ from litellm.litellm_core_utils.streaming_handler import ( ) from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload, with_openai_overload_status from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, is_model_cost_zero, @@ -114,6 +116,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + OverloadedProxyException, attribute_of, error_status_code, openai_error_param, @@ -132,13 +135,13 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash +from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( endpoint_unsupported_by_model, route_request, ) from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost -from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ProxyLogging, @@ -163,6 +166,7 @@ _ChunkT: Final = TypeVar("_ChunkT") KNOWN_PROXY_ROUTES: Final = frozenset( route for member in LiteLLMRoutes for route in member.value if route.startswith("/") ) +_OPENAI_OVERLOAD_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion", "aresponses"}) ProxyRouteType: TypeAlias = Literal[ "acompletion", @@ -171,7 +175,9 @@ ProxyRouteType: TypeAlias = Literal[ "_arealtime", "_aresponses_websocket", "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "aget_responses", "adelete_responses", "acancel_responses", @@ -270,6 +276,7 @@ if TYPE_CHECKING: ProxyConfig = _ProxyConfig else: ProxyConfig = Any +from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) @@ -1270,6 +1277,21 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]: return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n" +def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]: + _, error_obj = sse_error_payload(exc) + return _sse_error_frames(error_obj) + + +def _anthropic_late_failure_frames(exc: Exception) -> tuple[str, ...]: + answered: Final = proxy_exception_from_route_error(with_answered_status(exc)) + return ( + anthropic_error_sse_frame( + status_code=error_status_code(answered, status.HTTP_500_INTERNAL_SERVER_ERROR), + raw_message=answered.message, + ), + ) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -1518,6 +1540,7 @@ async def open_sse_before_first_byte( ping_interval_seconds: float | str | None, media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, + late_failure_frames: Callable[[Exception], tuple[str, ...]] = _openai_late_failure_frames, ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -1563,8 +1586,7 @@ async def open_sse_before_first_byte( # would never fire and the failure would go unaudited. The hook # also gets to sanitize what reaches the client, by returning or # raising a replacement, so its answer decides the frame. - _, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure)) - for frame in _sse_error_frames(error_obj): + for frame in late_failure_frames(await _sanitized_late_failure(exc, on_late_failure)): yield frame.encode() return async for chunk in _relay_late_response(produced): @@ -1885,6 +1907,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | def cost_headers( response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None ) -> Mapping[str, str | None]: + if ( + response_cost == 0 + and litellm_logging_obj is not None + and litellm_logging_obj.prepared_cost is not None + and not litellm_logging_obj.prepared_cost.price_known + ): + return {} cost_breakdown: Final = _get_cost_breakdown_from_logging_obj( litellm_logging_obj=litellm_logging_obj, response_cost=response_cost ) @@ -2973,13 +3002,14 @@ class ProxyBaseLLMRequestProcessing: idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets ``litellm_settings.sse_keepalive_ping_interval_seconds``. """ + answers_as_anthropic: Final = route_type == "anthropic_messages" async def _audit_late_failure(exc: Exception) -> HTTPException | None: # Once a keepalive is on the wire this can no longer raise, so the # caller's `except` never runs its own post_call_failure_hook. return await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=exc, + original_exception=with_answered_status(exc) if answers_as_anthropic else exc, request_data=self.data, dispatched_call_type=self.route_type, ) @@ -3008,6 +3038,7 @@ class ProxyBaseLLMRequestProcessing: ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, + late_failure_frames=_anthropic_late_failure_frames if answers_as_anthropic else _openai_late_failure_frames, ) async def _process_llm_request( @@ -3368,9 +3399,6 @@ class ProxyBaseLLMRequestProcessing: ) ), ) - await capture_proxy_response( - logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True - ) except (Exception, asyncio.CancelledError) as exc: _exception_raised = True if logging_obj.prepared_cost is not None: @@ -4373,27 +4401,28 @@ class ProxyBaseLLMRequestProcessing: version: str | None = None, ) -> Never: """Raises ProxyException (OpenAI API compatible) if an exception is raised""" - log_llm_api_exception(e, self.litellm_call_id) + answered: Final = with_openai_overload_status(e) if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES else e + log_llm_api_exception(answered, self.litellm_call_id) billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj") + failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=e, + original_exception=answered, request_data=self.data, dispatched_call_type=self.route_type, ) # Use transformed exception if callback returned one, otherwise use original - if transformed_exception is not None: - e = transformed_exception - litellm_debug_info: Final = getattr(e, "litellm_debug_info", "") + failure: Final = transformed_exception if transformed_exception is not None else answered + litellm_debug_info: Final = getattr(failure, "litellm_debug_info", "") verbose_proxy_logger.debug( "An error occurred: %s %s", - e, + failure, litellm_debug_info, ) timeout: Final = getattr( - e, "timeout", None + failure, "timeout", None ) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly _litellm_logging_obj: Final = ( billing_logging_obj if isinstance(billing_logging_obj, LiteLLMLoggingObj) else None @@ -4411,7 +4440,9 @@ class ProxyBaseLLMRequestProcessing: model_id=model_id, version=version, response_cost=( - _litellm_logging_obj.finalized_cost.amount + failure_row_response_cost(self.data) + if failed_in_agentic_follow_up + else _litellm_logging_obj.finalized_cost.amount if isinstance(_litellm_logging_obj, LiteLLMLoggingObj) and _litellm_logging_obj.finalized_cost is not None else 0 @@ -4423,11 +4454,11 @@ class ProxyBaseLLMRequestProcessing: # a failed request reports no timing, matching /v1/chat/completions read_timing_from_logging_obj=False, ) - headers = getattr(e, "headers", None) or {} + headers = getattr(failure, "headers", None) or {} if not headers: - _response: Final = attribute_of(e, "response") + _response: Final = attribute_of(failure, "response") _response_headers: Final = getattr(_response, "headers", None) if _response is not None else None - _provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None) + _provider_headers: Final = _response_headers or getattr(failure, "litellm_response_headers", None) if _provider_headers: headers = get_response_headers(dict(_provider_headers)) headers.update(custom_headers) @@ -4447,21 +4478,21 @@ class ProxyBaseLLMRequestProcessing: safe_headers: Final = {k: v for k, v in headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS} - self._apply_router_cooldown_retry_after(safe_headers, e) + self._apply_router_cooldown_retry_after(safe_headers, failure) - if isinstance(e, ProxyException): - e.headers = { - **{k: v for k, v in e.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, + if isinstance(failure, ProxyException): + failure.headers = { + **{k: v for k, v in failure.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}, **{k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()}, } - raise e + raise failure - if isinstance(e, HTTPException): - raise proxy_exception_from_http_exception(e, safe_headers) - elif isinstance(e, httpx.HTTPStatusError): + if isinstance(failure, HTTPException): + raise proxy_exception_from_http_exception(failure, safe_headers) + elif isinstance(failure, httpx.HTTPStatusError): # Handle httpx.HTTPStatusError - extract actual error from response # This matches the original behavior before the refactor in commit 511d435f6f - http_status_error: Final[httpx.HTTPStatusError] = e + http_status_error: Final[httpx.HTTPStatusError] = failure error_body: Final = await http_status_error.response.aread() error_text: Final = error_body.decode("utf-8") @@ -4471,12 +4502,12 @@ class ProxyBaseLLMRequestProcessing: detail={"error": error_text}, headers=error_headers, ) - error_msg: Final = f"{e}" + error_msg: Final = f"{failure}" # Check for AttributeError in the exception chain. # The AttributeError may be wrapped in multiple layers # (e.g. AttributeError -> OpenAIException -> APIConnectionError), # so walk __cause__, __context__, and original_exception recursively. - has_attribute_error: Final = _has_attribute_error_in_chain(e) + has_attribute_error: Final = _has_attribute_error_in_chain(failure) if has_attribute_error: raise ProxyException( @@ -4491,12 +4522,12 @@ class ProxyBaseLLMRequestProcessing: # VertexAIError, etc.) all have a status_code attribute reflecting # the upstream API response. Use it to return the correct HTTP code # instead of defaulting to 500. - _exc_status_code: Final = getattr(e, "status_code", None) + _exc_status_code: Final = getattr(failure, "status_code", None) if _exc_status_code is not None and isinstance(_exc_status_code, int) and 400 <= _exc_status_code <= 599: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR - if should_report_bug(e): + if should_report_bug(failure): proxy_server_request: Final = self.data.get("proxy_server_request") request_url: Final = ( proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None @@ -4505,24 +4536,33 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.error( bug_report_notice( build_proxy_bug_report( - e, + failure, call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), custom_llm_provider=self.data.get("custom_llm_provider"), stream=self.data.get("stream"), ) ) ) - client_message: Final = getattr(e, "message", error_msg) + client_message: Final = getattr(failure, "message", error_msg) + redacted_client_message: Final = redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ) + provider_specific_fields: Final = getattr( + failure, "provider_specific_fields", None + ) or relayed_provider_specific_fields(failure) + if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES and is_provider_overload(failure): + raise OverloadedProxyException( + message=redacted_client_message, + headers=cast(dict[str, str], safe_headers), # cast-ok: ProxyException coerces header values to strings + provider_specific_fields=provider_specific_fields, + ) raise ProxyException( - message=redact_internal_details_from_client_message( - strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg - ), - type=openai_error_type(e, _code), - param=openai_error_param(e), - openai_code=getattr(e, "code", None), + message=redacted_client_message, + type=openai_error_type(failure, _code), + param=openai_error_param(failure), + openai_code=getattr(failure, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None) - or relayed_provider_specific_fields(e), + provider_specific_fields=provider_specific_fields, headers=safe_headers, ) @@ -4632,6 +4672,7 @@ class ProxyBaseLLMRequestProcessing: request: Request | None = None, flush_tail: Callable[[], bytes] | None = None, seal_open_frame: Callable[[bytes], str] | None = None, + answer_status: Callable[[Exception], Exception] | None = None, ) -> AsyncGenerator[str, None]: """ Shared streaming data generator: runs proxy iterator hook, per-chunk hook, @@ -4647,6 +4688,9 @@ class ProxyBaseLLMRequestProcessing: relays raw upstream bytes, so an upstream that hangs up mid-frame leaves the client inside an open frame, where an error frame would be swallowed or misparsed instead of raised. + + ``answer_status`` gives a failure the status the route answers it with + before the failure hook records it. """ verbose_proxy_logger.debug("inside generator") # Resolve per-stream (not per-chunk) whether the heavy per-chunk path @@ -4743,6 +4787,7 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e ) + answered: Final = e if answer_status is None else answer_status(e) already_reported: Final = ( isinstance(stream_logging_obj, LiteLLMLoggingObj) and not stream_logging_obj.claim_stream_failure_report() @@ -4752,24 +4797,25 @@ class ProxyBaseLLMRequestProcessing: if already_reported else await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=e, + original_exception=answered, request_data=request_data, ) ) - if transformed_exception is not None: - e = transformed_exception + failure: Final = answered if transformed_exception is None else transformed_exception verbose_proxy_logger.debug( "An error occurred: %s", - e, + failure, ) - if isinstance(e, HTTPException): - raise e - stream_error_status: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) + if isinstance(failure, HTTPException): + if failure is e: + raise + raise failure from e + stream_error_status: Final = error_status_code(failure, status.HTTP_500_INTERNAL_SERVER_ERROR) proxy_exception: Final = ProxyException( - message=redact_internal_details_from_client_message(getattr(e, "message", str(e))), - type=openai_error_type(e, stream_error_status), - param=openai_error_param(e), + message=redact_internal_details_from_client_message(getattr(failure, "message", str(failure))), + type=openai_error_type(failure, stream_error_status), + param=openai_error_param(failure), code=stream_error_status, ) stream_completed = True @@ -4825,6 +4871,7 @@ class ProxyBaseLLMRequestProcessing: request=request, flush_tail=None if restamper is None else restamper.flush, seal_open_frame=seal_open_sse_frame, + answer_status=with_answered_status, ) @overload diff --git a/litellm/proxy/common_utils/claude_code_marketplace_auth.py b/litellm/proxy/common_utils/claude_code_marketplace_auth.py index 0cfb3061744..2cf9ae03327 100644 --- a/litellm/proxy/common_utils/claude_code_marketplace_auth.py +++ b/litellm/proxy/common_utils/claude_code_marketplace_auth.py @@ -1,10 +1,25 @@ from typing import Final +from fastapi import Request from pydantic import TypeAdapter, ValidationError +from litellm.types.proxy.auth.special_headers import SpecialHeaders + _AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) +def marketplace_key_header_present(request: Request) -> bool: + """Recognize header credentials before an optional-auth archive chooses anonymity.""" + from litellm.proxy.proxy_server import general_settings_view + + configured_header: Final = general_settings_view().get("litellm_key_header_name") + return ( + SpecialHeaders.openai_authorization.value in request.headers + or SpecialHeaders.custom_litellm_api_key.value in request.headers + or (isinstance(configured_header, str) and configured_header in request.headers) + ) + + def marketplace_auth_required() -> bool: from litellm.proxy.proxy_server import general_settings_view diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..1f8404fc37c --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,89 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_SETTING: Final = "fips_mode" +_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" + + +class FipsModeError(Exception): + pass + + +def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: + """general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including + a key with no value.""" + if FIPS_MODE_SETTING not in general_settings: + return False + value: Final = general_settings[FIPS_MODE_SETTING] + if isinstance(value, bool): + return value + return str_to_bool(value) if isinstance(value, str) else None + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" + try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from cryptography.hazmat.primitives import hashes + except ImportError: + return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True + + +def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: + """Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the + cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" + setting: Final = _fips_mode_setting(general_settings) + if setting is None: + value: Final = general_settings.get(FIPS_MODE_SETTING) + if value is None: + return ( + f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. " + "Set it to true or false, or remove it." + ) + return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it." + if not setting: + return None + if not openssl_enforces_fips(): + return ( + f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS " + f"provider active and default_properties = fips=yes, or turn {_SETTING} off." + ) + if not cryptography_enforces_fips(): + return ( + f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn " + f"{_SETTING} off." + ) + return None + + +def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: + refusal: Final = fips_mode_refusal(general_settings) + if refusal is not None: + raise FipsModeError(refusal) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 81ff460613c..e210c337922 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -37,6 +37,16 @@ _DECODED_JSON: Final = TypeAdapter(object) _JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) +def resolve_realtime_route_model(model: str | None, intent: str | None, is_translation: bool) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + def resolve_inference_model( body_model: object, settings: Mapping[str, object], @@ -197,10 +207,10 @@ def _parse_form_metadata(raw_metadata: str) -> object: """The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a 400 because no provider or database can encode it. ``json.loads`` accepts the escape, so the multipart ``metadata`` field gets the same check.""" - parsed: Final[object] = json.loads(raw_metadata) try: + parsed: Final[object] = json.loads(raw_metadata) json.dumps(parsed, ensure_ascii=False).encode("utf-8") - except UnicodeEncodeError as e: + except (json.JSONDecodeError, UnicodeEncodeError) as e: raise ProxyException( message=f"Invalid form payload: metadata {e}", type="invalid_request_error", @@ -210,6 +220,11 @@ def _parse_form_metadata(raw_metadata: str) -> object: return parsed +def _reraise_request_size_error(error: Exception) -> None: + if isinstance(error, HTTPException) and error.status_code == status.HTTP_413_CONTENT_TOO_LARGE: + raise error + + async def _read_form_data(request: Request) -> FormData: try: form_data: Final = await request.form() @@ -217,6 +232,7 @@ async def _read_form_data(request: Request) -> FormData: except ClientDisconnect: raise except Exception as e: + _reraise_request_size_error(e) # ``request.form()`` raises on malformed multipart (missing # boundary, malformed chunk encoding, …). Surface as 400 so # the auth-time pre-read does not silently cache ``{}`` while @@ -367,6 +383,7 @@ async def read_request_body(request: Request | None) -> dict: raise disconnected from e except Exception as e: # Catch unexpected errors to avoid crashes + _reraise_request_size_error(e) verbose_proxy_logger.exception("Unexpected error reading request body - %s", e) return {} @@ -572,14 +589,16 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat - if key.endswith("[]"): - clean_key = key[:-2] - parsed_form_data.setdefault(clean_key, []).append(value) - else: - parsed_form_data[key] = value - return parsed_form_data + form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) + array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) + normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) + normalized_keys: Final = tuple(dict.fromkeys(key for key, _ in normalized_items)) + return { + key: [value for item_key, value in normalized_items if item_key == key] + if key in array_keys + else next(value for item_key, value in reversed(normalized_items) if item_key == key) + for key in normalized_keys + } async def convert_upload_files_to_file_data( @@ -802,10 +821,6 @@ def populate_request_with_path_params(request_data: dict, request: Request) -> d request_data["vector_store_ids"] = [value] continue request_data.setdefault(key, value) - verbose_proxy_logger.debug( - "populate_request_with_path_params: Found path_params, vector_store_ids=%s", - request_data.get("vector_store_ids"), - ) return request_data # Fallback: parse the URL path directly to extract vector_store_id @@ -841,7 +856,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None existing_ids.append(vector_store_id) else: request_data["vector_store_ids"] = [vector_store_id] - verbose_proxy_logger.debug( - "populate_request_with_path_params: Updated request_data with vector_store_ids=%s", - request_data.get("vector_store_ids"), - ) diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 5c764d1a33e..8a91c7b0a0c 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, timezone @@ -15,6 +16,7 @@ from litellm.types.proxy.model_deprecation import ( DeprecationStatus, ModelDeprecationInfo, ModelDeprecationResponse, + SuccessorModelSource, ) if TYPE_CHECKING: @@ -32,6 +34,8 @@ class _ResolvedDeprecation: litellm_model: str | None litellm_provider: str | None copied_from_another_row: bool = False + successor_model: str | None = None + deprecation_source: str | None = None def _parse_deprecation_date(raw_value: object) -> date | None: @@ -47,8 +51,8 @@ def _parse_deprecation_date(raw_value: object) -> date | None: return None -def _successor_model(raw_value: object) -> str | None: - """A non-empty successor name, or None when the field is missing, blank, or not a string""" +def _non_empty_string(raw_value: object) -> str | None: + """A non-empty trimmed string, or None for missing, blank, or non-string fields""" return raw_value.strip() or None if isinstance(raw_value, str) else None @@ -58,10 +62,11 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R entry: Final = litellm.model_cost.get(model_key) if not isinstance(entry, Mapping): return None - parsed: Final = _parse_deprecation_date(entry.get("deprecation_date")) + fields: Final[Mapping[str, object]] = entry + parsed: Final = _parse_deprecation_date(fields.get("deprecation_date")) if parsed is None: return None - provider: Final = entry.get("litellm_provider") + provider: Final = fields.get("litellm_provider") return _ResolvedDeprecation( deprecation_date=parsed, litellm_model=model_key, @@ -69,6 +74,8 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R copied_from_another_row=isinstance(entry, AliasedCostEntry) and "deprecation_date" not in entry.registered_fields and model_key not in catalog, + successor_model=_non_empty_string(fields.get("successor_model")), + deprecation_source=_non_empty_string(fields.get("deprecation_source")), ) @@ -127,15 +134,6 @@ def _resolve_deployment_deprecation( litellm_params: Final = _mapping_field(deployment, "litellm_params") model_key: Final = _as_model_key(litellm_params.get("model")) - override: Final = _parse_deprecation_date(model_info.get("deprecation_date")) - if override is not None: - override_provider: Final = model_info.get("litellm_provider") - return _ResolvedDeprecation( - deprecation_date=override, - litellm_model=model_key, - litellm_provider=override_provider if isinstance(override_provider, str) else None, - ) - provider: Final = _deployment_provider(litellm_params) base_model: Final = _as_model_key(model_info.get("base_model")) model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key @@ -150,7 +148,19 @@ def _resolve_deployment_deprecation( if pick is not None ) own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row) - return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None) + from_catalog: Final = min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None) + + override: Final = _parse_deprecation_date(model_info.get("deprecation_date")) + if override is None: + return from_catalog + override_provider: Final = model_info.get("litellm_provider") + return _ResolvedDeprecation( + deprecation_date=override, + litellm_model=model_key, + litellm_provider=override_provider if isinstance(override_provider, str) else None, + successor_model=from_catalog.successor_model if from_catalog is not None else None, + deprecation_source=from_catalog.deprecation_source if from_catalog is not None else None, + ) def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus: @@ -161,14 +171,20 @@ def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus: return "upcoming" -def _checked_successor(model_name: str, model_info: Mapping[str, object], llm_router: Router) -> str | None: +def _checked_successor( + model_name: str, + model_info: Mapping[str, object], + llm_router: Router, + successor_log_level: int, +) -> str | None: """The deployment's successor_model as set, with a warning when no model in its team scope matches it""" - successor: Final = _successor_model(model_info.get("successor_model")) + successor: Final = _non_empty_string(model_info.get("successor_model")) if successor is None: return None team_id: Final = model_info.get("team_id") if not llm_router.registry().deployments(successor, team_id=team_id if isinstance(team_id, str) else None): - verbose_logger.warning( + verbose_logger.log( + successor_log_level, "model_deprecation: %s names successor %s, which matches no model group, alias, or wildcard route here", model_name, successor, @@ -182,7 +198,8 @@ def _build_info( today: date, warn_within_days: int, catalog: Mapping[str, object], -) -> ModelDeprecationInfo | None: + successor_log_level: int, +) -> _DeploymentInfo | None: model_name: Final = deployment.get("model_name") if not isinstance(model_name, str) or not model_name: return None @@ -192,41 +209,86 @@ def _build_info( return None days_until: Final = (resolved.deprecation_date - today).days - return ModelDeprecationInfo( - model_name=model_name, - litellm_model=resolved.litellm_model, - deprecation_date=resolved.deprecation_date, - days_until_deprecation=days_until, - status=_classify(days_until, warn_within_days), - litellm_provider=resolved.litellm_provider, - successor_model=_checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router), + model_info: Final = _mapping_field(deployment, "model_info") + configured: Final = _checked_successor(model_name, model_info, llm_router, successor_log_level) + team_public_model_name_value: Final = model_info.get("team_public_model_name") + team_public_model_name: Final = ( + team_public_model_name_value + if isinstance(team_public_model_name_value, str) and team_public_model_name_value.strip() + else None + ) + return _DeploymentInfo( + info=ModelDeprecationInfo( + model_name=model_name, + team_public_model_name=team_public_model_name, + litellm_model=resolved.litellm_model, + deprecation_date=resolved.deprecation_date, + days_until_deprecation=days_until, + status=_classify(days_until, warn_within_days), + litellm_provider=resolved.litellm_provider, + successor_model=configured or resolved.successor_model, + successor_model_source=( + SuccessorModelSource.DEPLOYMENT_CONFIG + if configured is not None + else SuccessorModelSource.MODEL_COST_MAP + if resolved.successor_model + else None + ), + deprecation_source=resolved.deprecation_source, + ), + configured_successor=configured is not None, ) -def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo: - """Keep the group's first deployment in config order, naming the first successor any of its deployments names""" - first: Final = group[0] - successors: Final = tuple(model.successor_model for model in group if model.successor_model is not None) +@dataclass(frozen=True, slots=True) +class _DeploymentInfo: + info: ModelDeprecationInfo + configured_successor: bool + + +def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) -> ModelDeprecationInfo: + """Keep the group's first deployment in config order, naming the first successor any of its deployments names, + a successor configured in model_info ahead of one the cost map supplies""" + first: Final = group[0].info + tiers: Final = tuple( + tuple( + deployment.info + for deployment in group + if deployment.configured_successor is configured and deployment.info.successor_model is not None + ) + for configured in (True, False) + ) + successors: Final = next((tier for tier in tiers if tier), ()) if not successors: return first - if len(frozenset(successors)) > 1: - verbose_logger.warning( + selected: Final = successors[0] + names: Final = frozenset(info.successor_model for info in successors if info.successor_model is not None) + if len(names) > 1: + verbose_logger.log( + successor_log_level, "model_deprecation: %s names several successors %s, reporting %s", first.model_name, - sorted(frozenset(successors)), - successors[0], + sorted(names), + selected.successor_model, ) - return first.model_copy(update={"successor_model": successors[0]}) + return first.model_copy( + update={ + "successor_model": selected.successor_model, + "successor_model_source": selected.successor_model_source, + "deprecation_source": selected.deprecation_source, + } + ) def _dedupe( - models: Sequence[ModelDeprecationInfo], + models: Sequence[_DeploymentInfo], + successor_log_level: int, ) -> tuple[ModelDeprecationInfo, ...]: """Report a model group carrying the same date on several deployments once""" - ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date)) + ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date)) return tuple( - _merge_group(tuple(group)) - for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date)) + _merge_group(tuple(group), successor_log_level) + for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date)) ) @@ -240,11 +302,18 @@ def _bucket(models: Sequence[ModelDeprecationInfo], status: DeprecationStatus) - def _collect_from_router( - llm_router: Router, today: date, warn_within_days: int, catalog: Mapping[str, object] + llm_router: Router, + today: date, + warn_within_days: int, + catalog: Mapping[str, object], + successor_log_level: int, ) -> tuple[ModelDeprecationInfo, ...]: deployments: Final = llm_router.registry().deployments() or () - infos: Final = (_build_info(deployment, llm_router, today, warn_within_days, catalog) for deployment in deployments) - deduped: Final = _dedupe(tuple(info for info in infos if info is not None)) + infos: Final = ( + _build_info(deployment, llm_router, today, warn_within_days, catalog, successor_log_level) + for deployment in deployments + ) + deduped: Final = _dedupe(tuple(info for info in infos if info is not None), successor_log_level) verbose_logger.debug( "model_deprecation: %d/%d deployments carry a deprecation date", @@ -259,12 +328,13 @@ def collect_model_deprecations( warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, today: date | None = None, loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, + successor_log_level: int = logging.WARNING, ) -> ModelDeprecationResponse: """Bucket every deployment carrying a deprecation date by how urgent it is""" snapshot_time: Final = datetime.now(timezone.utc) effective_today: Final = today or snapshot_time.date() deduped: Final = ( - _collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog()) + _collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog(), successor_log_level) if llm_router is not None else () ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index a948c390375..26369bf3f87 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -17,6 +17,7 @@ from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id" +_SERVER_IS_OVERLOADED_CODE: Final = "server_is_overloaded" _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( { @@ -44,6 +45,31 @@ _FAILOVER_REFUSALS: Final = ( ) +class OverloadedProxyException(ProxyException): + def __init__( + self, + message: str, + headers: dict[str, str] | None, # mutable-ok: matches ProxyException's constructor + provider_specific_fields: dict[str, object] | None, # mutable-ok: matches ProxyException's constructor + ) -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + message=message, + type="service_unavailable_error", + param=None, + code=status.HTTP_503_SERVICE_UNAVAILABLE, + headers=None if headers is None else dict(headers), + provider_specific_fields=provider_specific_fields, + ) + + def to_dict(self) -> dict[str, object]: # mutable-ok: matches ProxyException.to_dict + return { + **_ERROR_BODY.validate_python( + super().to_dict() # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments + ), + "code": _SERVER_IS_OVERLOADED_CODE, + } + + def attribute_of(value: object, name: str, default: object = None) -> object: return getattr(value, name, default) diff --git a/litellm/proxy/common_utils/realtime_utils.py b/litellm/proxy/common_utils/realtime_utils.py index e96685bdeae..d4fd4556ed4 100644 --- a/litellm/proxy/common_utils/realtime_utils.py +++ b/litellm/proxy/common_utils/realtime_utils.py @@ -11,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes: Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated string formatting work while keeping memory usage bounded. """ - return json.dumps({"model": model or ""}).encode() + return json.dumps({"model": model}).encode() _realtime_request_body: Final = realtime_request_body diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 5b3507f9b6d..86e716f19e3 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -920,8 +920,6 @@ def update_db_credential( # update model info if encrypted_credential.credential_info: """Update credential info""" - if "credential_info" not in merged_credential.credential_info: - merged_credential.credential_info = {} merged_credential.credential_info.update(encrypted_credential.credential_info) return merged_credential diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py deleted file mode 100644 index 0a9c5db1619..00000000000 --- a/litellm/proxy/db/accounting_ledger.py +++ /dev/null @@ -1,352 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -from collections.abc import AsyncGenerator, Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Final, Literal, TypeAlias - -from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter -from typing_extensions import assert_never - -from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent -from litellm.proxy.db.db_span import db_span -from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper - -if TYPE_CHECKING: - from prisma import Prisma - - from litellm.proxy.utils import PrismaClient - -AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent -_JSON: Final = TypeAdapter(dict[str, JsonValue]) -_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime) -_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...]) -_TIMEOUT: Final = timedelta(seconds=10) - - -class AccountingConflictError(ValueError): - pass - - -@dataclass(frozen=True, slots=True) -class UsageWrite: - event: InboundUsageEvent | OutboundUsageEvent - envelope: str - targets: tuple[DeliveryTarget, ...] = () - - -_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"] - - -@dataclass(frozen=True, slots=True) -class _AcceptedEvent: - event_id: str - kind: EventKind - source_event_id: str | None - occurred_at: datetime - payload: str - payload_hash: str - targets: tuple[str, ...] - schema_version: int - version: int - - -def _json(value: object) -> str: - return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) - - -def _prepare( - event: AccountingEvent, - targets: tuple[DeliveryTarget, ...], - envelope: str, - schema_version: int, - version: int, -) -> _AcceptedEvent: - if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1: - raise ValueError("Accounting versions must be positive integers") - validated: Final = type(event).model_validate_json(event.model_dump_json()) - payload: Final = _JSON.validate_json(envelope) - if payload.get("event") != validated.model_dump(mode="json"): - raise ValueError("Accounting envelope must contain the matching public event") - destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets))) - serialized: Final = _json(payload) - identity: Final = _json((schema_version, version, destinations, payload)) - kind: Final = ( - EventKind.COST - if isinstance(validated, CostEvent) - else EventKind.INBOUND_USAGE - if isinstance(validated, InboundUsageEvent) - else EventKind.OUTBOUND_USAGE - ) - return _AcceptedEvent( - event_id=validated.event_id, - kind=kind, - source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None, - occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at, - payload=serialized, - payload_hash=hashlib.sha256(identity.encode()).hexdigest(), - targets=destinations, - schema_version=schema_version, - version=version, - ) - - -_INSERT_EVENT: Final = """ -INSERT INTO "LiteLLM_AccountingEvent" - (event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload) -VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb) -ON CONFLICT (event_id) DO NOTHING RETURNING event_id -""" -_INSERT_DELIVERIES: Final = """ -INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) -SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb) -""" -_LOCK_SOURCE: Final = """ -SELECT event_id FROM "LiteLLM_AccountingEvent" -WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE -""" -_EXISTING_COST: Final = """ -SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1 -""" -_LOCK_EVENT: Final = """ -SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE -""" -_INSERT_USAGE_BATCH: Final = """ -INSERT INTO "LiteLLM_AccountingEvent" - (event_id, kind, schema_version, payload_hash, occurred_at, payload) -SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload -FROM jsonb_to_recordset($1::jsonb) AS records( - event_id text, kind text, payload_hash text, occurred_at text, payload jsonb) -ORDER BY event_id -ON CONFLICT (event_id) DO NOTHING RETURNING event_id -""" -_LOCK_USAGE_BATCH: Final = """ -SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" -WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) -ORDER BY event_id FOR UPDATE -""" -_LOAD_USAGE_BATCH: Final = """ -SELECT event.payload, ARRAY( - SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery - WHERE delivery.event_id = event.event_id ORDER BY target::text -) AS targets -FROM "LiteLLM_AccountingEvent" event -WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) - AND event.kind = 'outbound_usage' AND event.schema_version = 1 -""" -_INSERT_USAGE_DELIVERIES: Final = """ -INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) -SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1 -FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb) -CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination -""" -_COMPLETED_FAMILY: Final = """ -root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') -AND root.recorded_at < $1::timestamp -AND (root.kind = 'inbound_usage' OR EXISTS ( - SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost' -)) -AND NOT EXISTS ( - SELECT 1 FROM "LiteLLM_AccountingEvent" AS member - WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id)) - AND (member.recorded_at >= $1::timestamp - OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery - WHERE delivery.event_id = member.event_id AND delivery.state != 'applied')) -) -""" -_RETENTION_CANDIDATES: Final = ( - 'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE ' - + _COMPLETED_FAMILY - + " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED" -) -_ELIGIBLE_GROUP: Final = ( - 'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY -) -_DELETE_DELIVERIES: Final = """ -DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN ( - SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1) -) -""" -_DELETE_EVENTS: Final = """ -DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1) -""" - - -_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"] - - -async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict: - if record.source_event_id is not None: - source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id)) - if not source: - return "missing_source" - costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id))) - if any(cost["event_id"] != record.event_id for cost in costs): - return "competing_cost" - inserted: Final = _STRINGS.validate_python( - await db.query_raw( - _INSERT_EVENT, - record.event_id, - record.kind.value, - record.source_event_id, - record.schema_version, - record.payload_hash, - record.occurred_at.astimezone(timezone.utc).isoformat(), - record.payload, - ) - ) - if not inserted: - existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id))) - if not existing or existing[0]["payload_hash"] != record.payload_hash: - return "identity_conflict" - return "duplicate" - await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version) - return "inserted" - - -async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int: - if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)): - return 0 - await db.execute_raw(_DELETE_DELIVERIES, event_id) - return await db.execute_raw(_DELETE_EVENTS, event_id) - - -class AccountingLedger: - def __init__(self, transaction: Callable[[], _Transaction]) -> None: - self._transaction: Final = transaction - - @classmethod - def for_client(cls, client: PrismaClient) -> AccountingLedger: - from prisma.client import TransactionManager - - managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True)) - - @asynccontextmanager - async def transaction() -> AsyncGenerator[Prisma]: - primary: Final = writer_wrapper(client.db) - manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT)) - async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db: - await db.execute_raw("SET LOCAL statement_timeout = 5000") - await db.execute_raw("SET LOCAL lock_timeout = 1000") - yield db - - return cls(transaction) - - async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None: - records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes) - if not records: - return - unique: Final = {record.event_id: record for record in records} - if any(record.payload_hash != unique[record.event_id].payload_hash for record in records): - raise AccountingConflictError("Batch contains conflicting copies of a usage identity") - batch: Final = _json( - tuple( - { - "event_id": record.event_id, - "kind": record.kind.value, - "payload_hash": record.payload_hash, - "occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(), - "payload": _JSON.validate_json(record.payload), - } - for record in unique.values() - ) - ) - async with self._transaction() as db: - inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch)) - accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique)))) - if len(accepted) != len(unique) or any( - row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted - ): - raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set") - deliveries: Final = _json( - tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted) - ) - await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries) - - async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]: - if not event_ids: - return () - async with self._transaction() as db: - rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( - await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids)) - ) - return tuple( - UsageWrite( - OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])), - _json(row["payload"]), - TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]), - ) - for row in rows - ) - - async def insert_event( - self, - event: AccountingEvent, - targets: tuple[DeliveryTarget, ...], - *, - envelope: str, - schema_version: int = 1, - version: int = 1, - ) -> Literal["inserted", "duplicate"]: - """Accept T5/T6's serialized envelope, containing its public event under `event`. - - The immutable acceptance hash includes the target set and delivery version. - Changing either for an accepted event conflicts. T7 must provide a separate - add-target or new-version workflow that preserves receipts and prevents - duplicate contributions; redelivery cannot silently create or reset work. - - The Prisma adapter opens one interactive transaction per event, including - two SET LOCAL statements and a source-row lock for costs. I/O is async, - but awaiting this method inline adds commit latency per event, potentially - several times per request. T5 owns bounded asynchronous capture and an - explicit batching strategy; looping over this method does not batch commits. - Success acknowledges commit, independently of HTTP response acknowledgement. - """ - record: Final = _prepare(event, targets, envelope, schema_version, version) - async with self._transaction() as db: - result: Final = await _accept(db, record) - match result: - case "inserted" | "duplicate": - return result - case "missing_source": - raise AccountingConflictError("Cost source must be a durably accepted outbound usage event") - case "competing_cost": - raise AccountingConflictError("Usage event already has an authoritative cost") - case "identity_conflict": - raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set") - case _: - assert_never(result) - - async def purge_completed( - self, - *, - redelivery_before: datetime | None = None, - recovery_before: datetime | None = None, - limit: int = 100, - ) -> int: - """Delete at most `limit` completed families once both retention owners permit it. - - T8 must supply a coherent recovery watermark including outstanding holds. - Omitting either watermark disables deletion. Each usage row and its cost - remain together so a retained charge never loses its source or vice versa. - Applied receipts are deleted with the family, ending its retained dedup - protection. The caller must guarantee events recorded before - redelivery_before cannot be redelivered through supported paths. - T8's recovery_before must also protect checkpoints, - outstanding holds and unfinished settlements; age alone is insufficient. - """ - if type(limit) is not int or not 1 <= limit <= 1000: - raise ValueError("Accounting retention batch size must be between 1 and 1000") - if redelivery_before is None or recovery_before is None: - return 0 - cutoff: Final = min( - _INSTANT.validate_python(redelivery_before).astimezone(timezone.utc), - _INSTANT.validate_python(recovery_before).astimezone(timezone.utc), - ).isoformat() - async with self._transaction() as db: - roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit))) - deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots]) - return sum(deleted) diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index 99b49331932..d2b69f24397 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -14,11 +14,13 @@ interval of increments queued elsewhere: the same lag the SpendLogs aggregate it replaces (and every other spend column) already has. """ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Final, Protocol +from pydantic import TypeAdapter from typing_extensions import LiteralString, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -74,30 +76,44 @@ _OPEN_WINDOW_SPEND_SQL: Final = ( "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')" ) +_HAS_DELIVERY: Final = 'EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id)' + +_SETTLED_SPEND_LOG_FILTER: Final = ( + ' AND NOT EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery ' + 'WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id ' + "AND delivery.state IN ('pending', 'parked'))" +) + _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, " - "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + "COALESCE(SUM(spend) FILTER (WHERE " + + _HAS_DELIVERY + + " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) " + "AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, " - "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + "COALESCE(SUM(spend) FILTER (WHERE " + + _HAS_DELIVERY + + " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) " + "AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " 'FROM "LiteLLM_SpendLogs" ' - "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER ) _UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60) @@ -114,6 +130,10 @@ class _ExistingKeyRow(TypedDict): window_duration: ReadOnly[str] +_SEED_TOTALS_ROWS: Final = TypeAdapter(tuple[_SeedTotalsRow, ...]) +_EXISTING_KEY_ROWS: Final = TypeAdapter(tuple[_ExistingKeyRow, ...]) + + class _SeedTotalsQuery(Protocol): async def __call__(self, query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: ... @@ -135,6 +155,8 @@ class _UpsertBatchManager(Protocol): class _UpsertTransaction(Protocol): def batch_(self) -> _UpsertBatchManager: ... + async def query_raw(self, query: LiteralString, *args: object) -> Sequence[Mapping[str, object]]: ... + class _UpsertTransactionManager(Protocol): async def __aenter__(self) -> _UpsertTransaction: ... @@ -178,6 +200,7 @@ class WindowSpendLogsAggregate(Protocol): entity_id: str, window_start: datetime, batch_started_at: datetime | None, + query: _SeedTotalsQuery | None = None, ) -> WindowSeedTotals | None: ... @@ -187,6 +210,7 @@ async def spend_logs_seed_totals( entity_id: str, window_start: datetime, batch_started_at: datetime | None, + query: _SeedTotalsQuery | None = None, ) -> WindowSeedTotals | None: """LiteLLM_SpendLogs spend for one entity since window_start, both in full and up to the start of the batch being flushed, in one scan. @@ -210,11 +234,12 @@ async def spend_logs_seed_totals( bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_TEAM_SQL, _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL else: return None + read: Final = query or _seed_totals_query(prisma_client) async with db_span("seed_window_spend", "LiteLLM_SpendLogs"): rows: Final = ( - await _seed_totals_query(prisma_client)(unbounded_sql, entity_id, window_start) + await read(unbounded_sql, entity_id, window_start) if batch_started_at is None - else await _seed_totals_query(prisma_client)( + else await read( bounded_sql, entity_id, window_start, @@ -246,9 +271,10 @@ def _primary_key(transaction: WindowSpendTransaction) -> tuple[str, str, str]: async def _existing_primary_keys( prisma_client: "PrismaClient", transactions: tuple[WindowSpendTransaction, ...], + query: _ExistingKeysQuery | None = None, ) -> frozenset[tuple[str, str, str]]: async with db_span("select_window_spend_rows", "LiteLLM_BudgetWindowSpend"): - rows: Final = await _existing_keys_query(prisma_client)( + rows: Final = await (query or _existing_keys_query(prisma_client))( _SELECT_EXISTING_ROWS_SQL, tuple(transaction["entity_type"] for transaction in transactions), tuple(transaction["entity_id"] for transaction in transactions), @@ -262,6 +288,7 @@ async def _seed_base_for_missing_row( transaction: WindowSpendTransaction, existing_primary_keys: frozenset[tuple[str, str, str]], spend_logs_aggregate: WindowSpendLogsAggregate, + query: _SeedTotalsQuery | None = None, ) -> float: """Spend already recorded for a window that has no row yet. @@ -272,12 +299,24 @@ async def _seed_base_for_missing_row( """ if _primary_key(transaction) in existing_primary_keys: return 0.0 - totals: Final = await spend_logs_aggregate( - prisma_client=prisma_client, - entity_type=transaction["entity_type"], - entity_id=transaction["entity_id"], - window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc), - batch_started_at=_transaction_started_at(transaction), + window_start: Final = datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc) + totals: Final = ( + await spend_logs_aggregate( + prisma_client=prisma_client, + entity_type=transaction["entity_type"], + entity_id=transaction["entity_id"], + window_start=window_start, + batch_started_at=_transaction_started_at(transaction), + ) + if query is None + else await spend_logs_aggregate( + prisma_client=prisma_client, + entity_type=transaction["entity_type"], + entity_id=transaction["entity_id"], + window_start=window_start, + batch_started_at=_transaction_started_at(transaction), + query=query, + ) ) if totals is None: return 0.0 @@ -324,10 +363,30 @@ def _upsert_params( ) +def _existing_keys_in(transaction: _UpsertTransaction) -> _ExistingKeysQuery: + async def query_raw(query: LiteralString, *args: object) -> Sequence[_ExistingKeyRow]: + return _EXISTING_KEY_ROWS.validate_python(await transaction.query_raw(query, *args)) + + return query_raw + + +def _seed_totals_in(transaction: _UpsertTransaction) -> _SeedTotalsQuery: + async def query_raw(query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: + return _SEED_TOTALS_ROWS.validate_python(await transaction.query_raw(query, *args)) + + return query_raw + + +def _transaction_reads(transaction: _UpsertTransaction) -> tuple[_ExistingKeysQuery, _SeedTotalsQuery]: + """Reads made inside an enclosing transaction share its connection, so they cannot wait on it.""" + return _existing_keys_in(transaction), _seed_totals_in(transaction) + + async def commit_window_spend_updates( prisma_client: "PrismaClient", transactions: Sequence[WindowSpendTransaction], spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_seed_totals, + outer_transaction: _UpsertTransaction | None = None, ) -> None: """Apply aggregated window increments to LiteLLM_BudgetWindowSpend. @@ -343,9 +402,11 @@ async def commit_window_spend_updates( return ordered: Final = tuple(sorted(transactions, key=window_spend_group_key)) + reads: Final = _transaction_reads(outer_transaction) if outer_transaction is not None else (None, None) existing_primary_keys: Final = await _existing_primary_keys( prisma_client=prisma_client, transactions=ordered, + query=reads[0], ) seed_bases: Final = tuple( [ @@ -354,6 +415,7 @@ async def commit_window_spend_updates( transaction=transaction, existing_primary_keys=existing_primary_keys, spend_logs_aggregate=spend_logs_aggregate, + query=reads[1], ) for transaction in ordered ] @@ -367,7 +429,9 @@ async def commit_window_spend_updates( ) async with ( db_span("commit_window_spend_updates", "LiteLLM_BudgetWindowSpend"), - _upsert_transaction(prisma_client) as db_transaction, + ( + nullcontext(outer_transaction) if outer_transaction is not None else _upsert_transaction(prisma_client) + ) as db_transaction, db_transaction.batch_() as batcher, ): for transaction, seed_base in zip(ordered, seed_bases): diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..2d1770f4c1d 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -55,7 +55,7 @@ _ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType( "tag": "request_tags", } ) -_TAGS: Final = TypeAdapter(tuple[str, ...]) +_TAGS: Final = TypeAdapter(tuple[object, ...]) def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]: @@ -67,7 +67,7 @@ def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEnti if value is None: return () tags: Final = _TAGS.validate_json(value) if isinstance(value, str) else _TAGS.validate_python(value) - return tuple(dict.fromkeys(tags)) + return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str))) if value is None: return (None,) if entity == "user" else () if not isinstance(value, str) or (entity == "end_user" and not value): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index d9fb236632b..80f09df6c86 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,6 +12,7 @@ import json import os import random import time +import uuid from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar @@ -34,6 +35,7 @@ from litellm.constants import ( DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, ) +from litellm.litellm_core_utils.batch_utils import BATCH_COST_REQUEST_ID_SUFFIX from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -92,8 +94,10 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable from litellm.responses.streaming_iterator import spawn_logging_task from litellm.types.utils import CallTypes +from litellm.types.videos.utils import VIDEO_COST_REQUEST_ID_SUFFIX, VIDEO_STATUS_CALL_TYPES if TYPE_CHECKING: + from litellm.accounting.projection import Contribution from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -158,23 +162,73 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str: return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}" -def _is_batch_cost_row(payload: SpendLogsPayload) -> bool: - return payload.get("call_type") == CallTypes.aretrieve_batch.value and payload.get("status") == "success" +ACCOUNTING_LEDGER_KWARG: Final = "accounting_ledger" -_BATCH_COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"}) +def _persistable_api_key(token: str | None, logged_api_key: str) -> str | None: + from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash + from litellm.proxy.spend_tracking.spend_tracking_utils import redact_logged_api_key + + if token is None or token == logged_api_key or is_valid_sha256_hash(token): + return token + return redact_logged_api_key(token) -def _batch_cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]: - """Reduce a batch's cost row to what tells the retrieves apart when logging is off. +async def _track_guardrail_usage_of_cost_row( + payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool +) -> None: + """A job's cost row skips the spend log queue, so its guardrail runs are counted here instead of at the flush.""" + if disable_spend_logs: + return + from litellm.proxy.guardrails.usage_tracking import process_spend_logs_guardrail_usage - A proxy run with spend logs disabled still needs one row per batch to charge it once, + try: + await process_spend_logs_guardrail_usage(prisma_client=prisma_client, logs_to_process=(payload,)) + except Exception as e: # noqa: BLE001 # guardrail usage is best effort, as at the spend log flush + verbose_proxy_logger.warning( + "Guardrail usage tracking failed for spend row %s (non-fatal): %s", payload["request_id"], e + ) + + +def _is_settle_once_cost_row(payload: SpendLogsPayload) -> bool: + """A poll of a job whose cost is charged once, by the first poll that sees it, however often it is polled. + + A batch's cost is known only when a retrieve sees it finished (LIT-7048). A video's cost is + charged at create by the built-in providers, but a custom provider may report it on the + status poll that sees the video completed. Only a row carrying the job's cost suffix settles + the job, so a poll that pays only its guardrails, under an id that client call ids cannot + turn into the job's, is charged like any other request. + """ + if payload.get("status") != "success": + return False + call_type: Final = payload.get("call_type") + request_id: Final = payload["request_id"] + if call_type == CallTypes.aretrieve_batch.value: + return request_id.endswith(BATCH_COST_REQUEST_ID_SUFFIX) + return call_type in VIDEO_STATUS_CALL_TYPES and request_id.endswith(VIDEO_COST_REQUEST_ID_SUFFIX) + + +_COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "startTime", "endTime", "status"}) + + +def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]: + """Reduce a job's cost row to what tells its polls apart when logging is off. + + A legacy proxy run with spend logs disabled still needs one row per job to charge it once, so the row is written either way, but it carries no request of its own: no metadata, - no requester IP, no key, model, or token counts (LIT-7048). + no requester IP, no key, model, or token counts (LIT-7048). A ledger batch is deduplicated + by its delivery row instead, so it writes no local row when logging is off. """ if disable_spend_logs is False: return payload - return MappingProxyType({field: value for field, value in payload.items() if field in _BATCH_COST_CLAIM_FIELDS}) + if "accounting_payload" in payload: + return { + "request_id": payload["request_id"], + "call_type": payload["call_type"], + "accounting_payload": payload["accounting_payload"], + "accounting_log_disabled": True, + } + return MappingProxyType({field: value for field, value in payload.items() if field in _COST_CLAIM_FIELDS}) class _SpendIncrement(TypedDict): @@ -187,8 +241,17 @@ class _MemberSpendRow(TypedDict): cost: ReadOnly[float] +class SpendUpsertTable(Protocol): + def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + class _SpendBatch(Protocol): litellm_usertable: BatchTable + litellm_endusertable: SpendUpsertTable + litellm_dailymodelusage: SpendUpsertTable + + def execute_raw(self, query: LiteralString, *args: object) -> None: ... + litellm_verificationtoken: BatchTable litellm_teamtable: BatchTable litellm_teammembership: BatchTable @@ -219,14 +282,14 @@ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) return _ENTITY_SPEND_TABLES[table_accessor](batcher) -class _SpendBatchManager(Protocol): +class SpendBatchManager(Protocol): async def __aenter__(self) -> _SpendBatch: ... async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... class _SpendTransaction(Protocol): - def batch_(self) -> _SpendBatchManager: ... + def batch_(self) -> SpendBatchManager: ... async def execute_raw(self, query: LiteralString, *args: object) -> int: ... @@ -381,8 +444,14 @@ _ENTITY_SPEND_MODELS: Final[Mapping[_EntitySpendTable, str]] = MappingProxyType( @asynccontextmanager async def _spend_update_tx( - prisma_client: PrismaClient, table: str, call_type: str = "commit_spend_updates" + prisma_client: PrismaClient, + table: str, + call_type: str = "commit_spend_updates", + outer_transaction: _SpendTransaction | None = None, ) -> AsyncGenerator[_SpendTransaction]: + if outer_transaction is not None: + yield outer_transaction + return tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60)) async with db_span(call_type, table), tx as transaction: await apply_rollup_lock_timeout(transaction) @@ -569,6 +638,21 @@ def debitable_model_access_groups( return tuple(group for group in ordered if group in served) +def _billable_cost(response_cost: float | None, payload: SpendLogsPayload) -> float | None: + billable_cost: Final = finite_cost(response_cost) + if response_cost is not None and billable_cost is None: + verbose_proxy_logger.warning( + "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", + response_cost, + single_log_line(payload["request_id"]), + ) + return billable_cost + + +def _iso_timestamp(value: datetime | str) -> str: + return value.isoformat() if isinstance(value, datetime) else value + + class DBSpendUpdateWriter: """ Module responsible for @@ -613,10 +697,11 @@ class DBSpendUpdateWriter: ) -> bool: """Record the request's spend, answering whether its cost still needs charging. - False only for a batch retrieve whose cost row another retrieve already wrote, + False only for a batch or video poll whose cost row another poll already wrote, so the caller leaves the key, team, and user counters alone (LIT-7048). """ from litellm.proxy.proxy_server import ( + accounting_window_keys, disable_spend_logs, general_settings, # pyright: ignore[reportUnknownVariableType] # legacy proxy configuration is narrowed below litellm_proxy_budget_name, @@ -624,6 +709,13 @@ class DBSpendUpdateWriter: ) from litellm.proxy.utils import ProxyUpdateSpend, hash_token + proxy_settings: Final = cast( # cast-ok: proxy settings use string keys + Mapping[str, object], general_settings + ) + use_ledger: Final = proxy_settings.get("use_spend_logs_ledger") is True + request_fields: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs or {}) + if kwargs is not None: + kwargs[ACCOUNTING_LEDGER_KWARG] = use_ledger # rebind-ok: spend counters leave ledger windows to the worker try: verbose_proxy_logger.debug( "Enters prisma db call, response_cost: %s, token: %s; user_id: %s; team_id: %s", @@ -634,10 +726,9 @@ class DBSpendUpdateWriter: ) if ProxyUpdateSpend.disable_spend_updates() is True: return True - if token is not None and isinstance(token, str) and token.startswith("sk-"): - hashed_token = hash_token(token=token) - else: - hashed_token = token + hashed_token: Final = ( + hash_token(token=token) if isinstance(token, str) and token.startswith("sk-") else token + ) ## CREATE SPEND LOG PAYLOAD ## from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -653,32 +744,42 @@ class DBSpendUpdateWriter: end_time=end_time, llm_router=llm_router, ) - billable_cost: Final = finite_cost(response_cost) - if response_cost is not None and billable_cost is None: - verbose_proxy_logger.warning( - "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", - response_cost, - single_log_line(payload["request_id"]), - ) + billable_cost: Final = _billable_cost(response_cost, payload) payload["spend"] = billable_cost or 0.0 - if isinstance(payload["startTime"], datetime): - payload["startTime"] = payload["startTime"].isoformat() - if isinstance(payload["endTime"], datetime): - payload["endTime"] = payload["endTime"].isoformat() - - if org_id is not None and org_id != "": + payload["startTime"] = _iso_timestamp(payload["startTime"]) + payload["endTime"] = _iso_timestamp(payload["endTime"]) + if org_id: payload["organization_id"] = org_id - - if team_id is not None and team_id != "": + if team_id: payload["team_id"] = team_id - error_log_settings: Final = cast( # cast-ok: proxy settings use string keys - Mapping[str, object], general_settings - ) + if use_ledger: + from litellm.accounting.snapshot import build_accounting_payload + + if not _is_settle_once_cost_row(payload): + payload["accounting_delivery_id"] = uuid.uuid4().hex + payload["accounting_payload"] = build_accounting_payload( + payload, + entities={ + "api_key": _persistable_api_key(hashed_token, payload.get("api_key") or ""), + "user": user_id, + "end_user": end_user_id, + "team_id": team_id, + "organization_id": org_id, + "project_id": project_id, + "agent_id": payload["agent_id"], + "proxy_budget_name": litellm_proxy_budget_name if litellm.max_budget > 0 else None, + }, + model_access_groups=debitable_model_access_groups( + get_request_model_access_groups(request_fields), payload["model_id"], llm_router + ), + windows=await accounting_window_keys(hashed_token, team_id) if billable_cost else (), + router=get_llm_router, + ) keep_spend_logs: Final = disable_spend_logs is False and not ( "status" in payload and payload["status"] == "failure" - and error_log_settings.get("disable_error_logs") is True + and proxy_settings.get("disable_error_logs") is True ) if not await self._record_spend_log( payload=payload, @@ -707,24 +808,26 @@ class DBSpendUpdateWriter: "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur." ) - spawn_logging_task( - asyncio.get_running_loop(), - self._batch_database_updates( - response_cost=billable_cost, - user_id=user_id, - hashed_token=hashed_token, - team_id=team_id, - org_id=org_id, - project_id=project_id, - end_user_id=end_user_id, - prisma_client=prisma_client, - litellm_proxy_budget_name=litellm_proxy_budget_name, - payload=payload, - request_model_access_groups=get_request_model_access_groups(kwargs), - ), - task_name="spend rollups", - ) - + if not use_ledger: + spawn_logging_task( + asyncio.get_running_loop(), + self._batch_database_updates( + response_cost=billable_cost, + user_id=user_id, + hashed_token=hashed_token, + team_id=team_id, + org_id=org_id, + project_id=project_id, + end_user_id=end_user_id, + prisma_client=prisma_client, + litellm_proxy_budget_name=litellm_proxy_budget_name, + payload=payload, + request_model_access_groups=get_request_model_access_groups(request_fields), + ), + task_name="spend rollups", + ) + else: + self._record_request_error_safely(payload=payload, prisma_client=prisma_client) self._enqueue_tool_registry_upsert( kwargs=kwargs, completion_response=completion_response, @@ -755,24 +858,32 @@ class DBSpendUpdateWriter: disable_spend_logs: bool, llm_router: "Router | None" = None, ) -> bool: - if prisma_client is not None and _is_batch_cost_row(payload): - return await self._claim_batch_cost_spend_log( + if prisma_client is not None and _is_settle_once_cost_row(payload): + return await self._claim_settle_once_cost_row( payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs ) - if disable_spend_logs is False: + if "accounting_payload" in payload: + await self._insert_spend_log_to_db( + payload={**payload, "accounting_log_disabled": disable_spend_logs}, + prisma_client=prisma_client, + llm_router=llm_router, + ) + elif disable_spend_logs is False: await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router) return True - async def _claim_batch_cost_spend_log( + async def _claim_settle_once_cost_row( self, payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool ) -> bool: - """Write the batch's cost row now, or learn that another retrieve already did. + """Write the job's cost row now, or learn that another poll already did. - Every retrieve of one batch shares this row, so the insert that lands first owns + Every poll of one batch or video shares this row, so the insert that lands first owns the charge and every later one finds the row and charges nothing (LIT-7048). Only - a row that recorded a charge counts: a failed retrieve, a request whose client - picked the batch id as its call id, and the $0 row an older proxy left behind - while the batch was still running all leave the charge to be made. + a row that recorded a charge counts: a failed poll, a request whose client + picked the job id as its call id, and the $0 row a poll left behind + while the job was still running all leave the charge to be made. A ledger job + is owned by its delivery row, keyed by its cost row id, in every logging mode, and a + $0 poll records no delivery so it cannot block the poll that carries the cost. """ from litellm.repositories.table_repositories import SpendLogsRepository @@ -783,43 +894,59 @@ class DBSpendUpdateWriter: ) row: Final = spend_log_row_with_retained_metadata( - _batch_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() + _cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() ) - spend_logs: Final = SpendLogsRepository(prisma_client).table + repository: Final = SpendLogsRepository(prisma_client) try: - claimed: Final = await spend_logs.create_many( + if "accounting_payload" in payload: + if payload["spend"] <= 0: + return True + accepted: Final = await repository.insert_with_deliveries([prisma_client.jsonify_object(row)]) == 1 + if accepted: + await _track_guardrail_usage_of_cost_row(payload, prisma_client, disable_spend_logs) + return accepted + if await repository.has_delivery(request_id): + return False + claimed: Final = await repository.table.create_many( data=[prisma_client.jsonify_object(row)], skip_duplicates=True, ) - if claimed == 1: - return True - existing: Final = await spend_logs.find_unique(where={"request_id": request_id}) + existing: Final = ( + None if claimed == 1 else await repository.table.find_unique(where={"request_id": request_id}) + ) except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log verbose_proxy_logger.warning( - "Could not claim spend row %s for a batch's cost, queueing it: %s", request_id, e + "Could not claim spend row %s for a job's cost, queueing it: %s", request_id, e ) await self._insert_spend_log_to_db(payload=prisma_client.jsonify_object(row), prisma_client=prisma_client) return True - if existing is None or existing.call_type != CallTypes.aretrieve_batch.value or existing.status != "success": + if claimed == 1: + await _track_guardrail_usage_of_cost_row(payload, prisma_client, disable_spend_logs) + return True + if existing is None or existing.call_type != payload["call_type"] or existing.status != "success": verbose_proxy_logger.warning( - "Spend row %s belongs to a %s request, so this batch's cost is charged without a row of its own", + "Spend row %s belongs to a %s request, so this job's cost is charged without a row of its own", request_id, getattr(existing, "call_type", None), ) return True if existing.spend > 0: - verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id) + verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this job", request_id) return False - return await self._take_over_uncharged_batch_cost_row(payload=payload, prisma_client=prisma_client, row=row) + taken_over: Final = await self._take_over_uncharged_cost_row( + payload=payload, prisma_client=prisma_client, row=row + ) + if taken_over: + await _track_guardrail_usage_of_cost_row(payload, prisma_client, disable_spend_logs) + return taken_over - async def _take_over_uncharged_batch_cost_row( + async def _take_over_uncharged_cost_row( self, payload: SpendLogsPayload, prisma_client: "PrismaClient", row: Mapping[str, object] ) -> bool: - """Take the batch's cost row over from the poll that left it charging nothing. + """Take the job's cost row over from the poll that left it charging nothing. - A pre-upgrade proxy wrote that row every time it polled the batch while it was still - running, so the charge is still to be made and the row still has to end up carrying - it. The row stops matching the moment it carries a charge, so it is one retrieve that + A poll that saw the job still running wrote that row at $0, so the charge is still to be + made and the row still has to end up carrying it. The row stops matching the moment it carries a charge, so it is one poll that takes it over and charges, and every later one reads the charge and charges nothing. """ from litellm.repositories.table_repositories import SpendLogsRepository @@ -827,7 +954,7 @@ class DBSpendUpdateWriter: request_id: Final = payload["request_id"] if payload["spend"] <= 0: verbose_proxy_logger.debug( - "Cost tracking skipped: this batch costs nothing and spend row %s says so", request_id + "Cost tracking skipped: this job costs nothing and spend row %s says so", request_id ) return False try: @@ -837,18 +964,18 @@ class DBSpendUpdateWriter: ), where={ "request_id": request_id, - "call_type": CallTypes.aretrieve_batch.value, + "call_type": payload["call_type"], "status": "success", "spend": 0.0, }, ) except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; the next retrieve takes the row over verbose_proxy_logger.warning( - "Could not take over spend row %s, leaving this batch's cost to the next retrieve: %s", request_id, e + "Could not take over spend row %s, leaving this job's cost to the next poll: %s", request_id, e ) return False if taken_over == 0: - verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this batch", request_id) + verbose_proxy_logger.debug("Cost tracking skipped: spend row %s already charged this job", request_id) return False return True @@ -1231,10 +1358,7 @@ class DBSpendUpdateWriter: self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client), payload=payload_copy, ) - try: - self._record_request_error(payload=payload_copy, prisma_client=prisma_client) - except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes - _report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e) + self._record_request_error_safely(payload=payload_copy, prisma_client=prisma_client) await self._run_spend_update_step( "add_spend_log_transaction_to_daily_org_transaction", self.add_spend_log_transaction_to_daily_org_transaction( @@ -1564,7 +1688,7 @@ class DBSpendUpdateWriter: async def _insert_spend_log_to_db( self, - payload: dict | SpendLogsPayload, + payload: Mapping[str, object], prisma_client: PrismaClient | None = None, spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"), llm_router: "Router | None" = None, @@ -1608,6 +1732,13 @@ class DBSpendUpdateWriter: else: - Regular flow of this method """ + from litellm.accounting.worker import flush_accounting_deliveries + + try: + await flush_accounting_deliveries(self, prisma_client, proxy_logging_obj) + except Exception: # noqa: BLE001 # pending deliveries survive outages while legacy queues must still drain + verbose_proxy_logger.exception("Accounting delivery flush failed; pending work will retry") + if RedisUpdateBuffer.should_commit_spend_updates_to_redis(): await self._commit_spend_updates_to_db_with_redis( prisma_client=prisma_client, @@ -2134,6 +2265,39 @@ class DBSpendUpdateWriter: _handle_spend_update_failure = handle_spend_update_failure + @staticmethod + async def invalidate_accounting_entities( + contributions: tuple["Contribution", ...], proxy_logging_obj: ProxyLogging + ) -> None: + await DBSpendUpdateWriter._invalidate_project_caches( + tuple(dict(item.key)["project_id"] for item in contributions if item.table == "LiteLLM_ProjectTable"), + proxy_logging_obj, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + details: Final = _SPEND_METADATA_ADAPTER.validate_python( + proxy_logging_obj.call_details # pyright: ignore[reportUnknownMemberType] # legacy logging metadata is validated at this boundary + ) + user_api_key_cache: Final = details.get("user_api_key_cache") + if isinstance(user_api_key_cache, UserApiKeyCache): + with service_target(AUTH_OBJECTS_TARGET): + for item in contributions: + if item.table == "LiteLLM_TeamMembership": + await user_api_key_cache.async_delete_cache( + key=f"team_membership:{dict(item.key)['user_id']}:{dict(item.key)['team_id']}" + ) + + async def apply_accounting_entities( + self, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + transaction: _SpendTransaction, + updates: DBSpendUpdateTransactions, + ) -> None: + await self._commit_spend_updates_to_db( + prisma_client, 0, proxy_logging_obj, updates, outer_transaction=transaction + ) + async def _commit_spend_updates_to_db( self, prisma_client: PrismaClient, @@ -2141,6 +2305,7 @@ class DBSpendUpdateWriter: proxy_logging_obj: ProxyLogging, db_spend_update_transactions: DBSpendUpdateTransactions, on_table_committed: Callable[[_SpendTableName], None] | None = None, + outer_transaction: _SpendTransaction | None = None, ): """ Commits all the spend `UPDATE` transactions to the Database @@ -2155,7 +2320,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_UserTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_UserTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by ID for consistent lock ordering across pods to prevent deadlocks. # batch_() issues statements sequentially within the tx, so iteration @@ -2167,6 +2334,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2186,6 +2355,7 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, end_user_list_transactions=end_user_list_transactions, + outer_transaction=outer_transaction, ) if on_table_committed is not None: on_table_committed("end_user_list_transactions") @@ -2196,7 +2366,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_VerificationToken") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_VerificationToken", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by token for consistent lock ordering across pods to prevent deadlocks. for token, response_cost in sorted(key_list_transactions.items()): @@ -2211,6 +2383,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2228,7 +2402,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_TeamTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_TeamTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. for team_id, response_cost in sorted(team_list_transactions.items()): @@ -2241,6 +2417,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2266,11 +2444,15 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_TeamMembership") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_TeamMembership", outer_transaction=outer_transaction + ) as transaction: await _write_team_member_spend(transaction, team_member_list_transactions) # Transaction succeeded, break out of retry loop break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2283,7 +2465,7 @@ class DBSpendUpdateWriter: # Invalidate cache for updated team memberships # This ensures budget checks read fresh spend data from the database - if team_memberships_to_invalidate and proxy_logging_obj is not None: + if outer_transaction is None and team_memberships_to_invalidate and proxy_logging_obj is not None: user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache") if user_api_key_cache is not None: with service_target(AUTH_OBJECTS_TARGET): @@ -2303,7 +2485,9 @@ class DBSpendUpdateWriter: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, "LiteLLM_OrganizationTable") as transaction: + async with _spend_update_tx( + prisma_client, "LiteLLM_OrganizationTable", outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by org_id for consistent lock ordering across pods to prevent deadlocks. for org_id, response_cost in sorted(org_list_transactions.items()): @@ -2313,6 +2497,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2330,7 +2516,9 @@ class DBSpendUpdateWriter: start_time = time.time() try: async with ( - _spend_update_tx(prisma_client, "LiteLLM_OrganizationMembership") as transaction, + _spend_update_tx( + prisma_client, "LiteLLM_OrganizationMembership", outer_transaction=outer_transaction + ) as transaction, transaction.batch_() as batcher, ): for key, response_cost in sorted(org_member_list_transactions.items()): @@ -2341,6 +2529,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await self.handle_spend_update_failure( e=e, attempt=i, @@ -2356,6 +2546,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Project", transactions=project_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_projecttable", where_field="project_id", n_retry_times=n_retry_times, @@ -2364,16 +2555,18 @@ class DBSpendUpdateWriter: ) if on_table_committed is not None: on_table_committed("project_list_transactions") - await DBSpendUpdateWriter._invalidate_project_caches( - project_ids=tuple(project_list_transactions or ()), - proxy_logging_obj=proxy_logging_obj, - ) + if outer_transaction is None: + await DBSpendUpdateWriter._invalidate_project_caches( + project_ids=tuple(project_list_transactions or ()), + proxy_logging_obj=proxy_logging_obj, + ) ### UPDATE TAG TABLE ### tag_list_transactions: Final = db_spend_update_transactions["tag_list_transactions"] await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Tag", transactions=tag_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_tagtable", where_field="tag_name", n_retry_times=n_retry_times, @@ -2390,6 +2583,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Model access group", transactions=model_access_group_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_modelaccessgroupbudgettable", where_field="access_group_name", n_retry_times=n_retry_times, @@ -2404,6 +2598,7 @@ class DBSpendUpdateWriter: await DBSpendUpdateWriter._update_entity_spend_in_db( entity_name="Agent", transactions=agent_list_transactions, + outer_transaction=outer_transaction, table_accessor="litellm_agentstable", where_field="agent_id", n_retry_times=n_retry_times, @@ -2433,6 +2628,7 @@ class DBSpendUpdateWriter: n_retry_times: int, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, + outer_transaction: _SpendTransaction | None = None, ): """ Helper function to update spend for any entity type (team, org, tag, etc). @@ -2445,13 +2641,16 @@ class DBSpendUpdateWriter: n_retry_times: Number of retries on failure prisma_client: Prisma client instance proxy_logging_obj: Proxy logging object + outer_transaction: Existing ledger transaction owning the complete delivery update """ verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions) if transactions is not None and len(transactions.keys()) > 0: for i in range(n_retry_times + 1): start_time = time.time() try: - async with _spend_update_tx(prisma_client, _ENTITY_SPEND_MODELS[table_accessor]) as transaction: + async with _spend_update_tx( + prisma_client, _ENTITY_SPEND_MODELS[table_accessor], outer_transaction=outer_transaction + ) as transaction: async with transaction.batch_() as batcher: # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. for entity_id, response_cost in sorted(transactions.items()): @@ -2468,6 +2667,8 @@ class DBSpendUpdateWriter: ) break except Exception as e: + if outer_transaction is not None: + raise await DBSpendUpdateWriter.handle_spend_update_failure( e=e, attempt=i, @@ -2801,6 +3002,12 @@ class DBSpendUpdateWriter: entity_id_field="tag", ) + def _record_request_error_safely(self, *, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None: + try: + self._record_request_error(payload=payload, prisma_client=prisma_client) + except Exception as error: # noqa: BLE001 # error rollup failure must not skip sibling writes + _report_spend_update_step_failure("_record_request_error", payload=payload, exc=error) + @staticmethod def _record_request_error(*, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None: if prisma_client is None: diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 170eb450339..29f727ea41b 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -598,7 +598,7 @@ class SpendLogCleanup: rows: Final = _STORAGE_ROWS.validate_python( await tx.query_raw(trusted_sql(_STORAGE_SQL), list(_SPEND_LOG_TABLES)) ) - except Exception as e: # noqa: BLE001 - a failed measurement skips the size pass, it must not fail the run + except Exception as e: # noqa: BLE001 # a failed measurement skips the size pass, it must not fail the run verbose_proxy_logger.warning("Could not measure spend-log storage: %s: %s", type(e).__name__, e) return None if not rows: @@ -666,7 +666,7 @@ class SpendLogCleanup: await tx.query_raw(trusted_sql(_LOAD_SIZE_STATE_SQL), SPEND_LOG_SIZE_CLEANUP_STATE_PARAM) ) return SizeCleanupState.model_validate_json(rows[0].state_json) if rows else None - except Exception as e: # noqa: BLE001 - an unreadable count restarts from the catalog, it must not fail the run + except Exception as e: # noqa: BLE001 # an unreadable count restarts from the catalog, it must not fail the run verbose_proxy_logger.warning("Could not read the spend-log size cleanup state: %s: %s", type(e).__name__, e) return None @@ -678,7 +678,7 @@ class SpendLogCleanup: await tx.execute_raw( trusted_sql(_SAVE_SIZE_STATE_SQL), SPEND_LOG_SIZE_CLEANUP_STATE_PARAM, state.model_dump_json() ) - except Exception as e: # noqa: BLE001 - a lost save costs one re-seed from the catalog, it must not fail the run + except Exception as e: # noqa: BLE001 # a lost save costs one re-seed from the catalog, it must not fail the run verbose_proxy_logger.warning("Could not save the spend-log size cleanup state: %s: %s", type(e).__name__, e) async def _size_step_cutoff(self, prisma_client: PrismaClient, deadline: float, rows: int) -> datetime | None: diff --git a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py index e9ec040023c..abbeab7f89d 100644 --- a/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py +++ b/litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py @@ -55,3 +55,10 @@ class ToolDiscoveryQueue: """Return and clear all pending rows.""" items, self._pending = list(self._pending.values()), {} return items + + def requeue(self, items: Iterable[ToolDiscoveryQueueItem]) -> None: + """Put rows from a failed write back, ahead of and merged with anything queued since the flush.""" + since_flush: Final = self._pending + self._pending = {} + for item in (*items, *since_flush.values()): + self.add_update(item) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 13a14e95e56..c2d58512fea 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -31,9 +31,30 @@ _CONNECTION_CAPACITY_PHRASES: Final = ( "too many connections for database", "remaining connection slots are reserved", ) +_NOT_ACCEPTING_CONNECTIONS_SQLSTATE: Final = "57P03" +_NOT_ACCEPTING_CONNECTIONS_PHRASES: Final = ( + "the database system is starting up", + "the database system is shutting down", + "the database system is not yet accepting connections", + "the database system is not accepting connections", + "the database system is in recovery mode", +) +_SERVER_TERMINATED_SQLSTATES: Final = frozenset({"57P01", "57P02", "57P05"}) +_SERVER_TERMINATED_PHRASES: Final = ( + "terminating connection due to administrator command", + "terminating connection due to immediate shutdown command", + "terminating connection because of unexpected sigquit signal", + "terminating connection because of crash of another server process", + "terminating connection due to idle-session timeout", +) +_UNREACHABLE_PRISMA_CODE: Final = "P1001" +_UNREACHABLE_PHRASES: Final = ("can't reach database server", "cannot reach database server") _PRISMA_POOL_TIMEOUT_CODE: Final = "P2024" _LOCK_TIMEOUT_SQLSTATE: Final = "55P03" _LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout" +_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014" +_REJECTED_ROW_SQLSTATE_CLASSES: Final = frozenset({"22", "23", "54"}) +_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout" def exception_chain(e: BaseException) -> Iterator[BaseException]: @@ -137,6 +158,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection @staticmethod @@ -199,6 +222,20 @@ class PrismaDBExceptionHandler: return type(e) is prisma.errors.DataError + @staticmethod + def is_rejected_row_error(e: Exception) -> bool: + """True iff Postgres refused the rows themselves: a base ``DataError``, or a raw query failing with a data + exception (SQLSTATE class 22), integrity violation (class 23) or a value too large for an index (class 54). Systemic raw failures such as a missing + table (42P01) or a deadlock (40P01) stay False so the batch is retried instead of dropped.""" + import prisma + + if PrismaDBExceptionHandler.is_prisma_data_error(e): + return True + if not isinstance(e, _exception_types(prisma.errors.RawQueryError)): + return False + sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) + return sqlstate is not None and sqlstate[:2] in _REJECTED_ROW_SQLSTATE_CLASSES + @staticmethod def is_database_transport_error(e: Exception) -> bool: """ @@ -287,6 +324,39 @@ class PrismaDBExceptionHandler: error_message: Final = str(e).lower() return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES) + @staticmethod + def is_database_not_accepting_connections_error(e: Exception) -> bool: + """True iff Postgres refused the connection because it is starting up, shutting + down or in crash recovery (SQLSTATE 57P03). The refusal comes before any statement + runs, so nothing the write carried was applied.""" + import prisma + + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) == _NOT_ACCEPTING_CONNECTIONS_SQLSTATE: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _NOT_ACCEPTING_CONNECTIONS_PHRASES) + + @staticmethod + def is_database_lifecycle_error(e: Exception) -> bool: + """True iff Postgres failed the statement because of a server lifecycle event: + a refused connection during startup, shutdown or recovery (57P03), or a session + the server terminated (57P01, 57P02, 57P05). The server comes back on its own, + so the rows are not at fault. A terminated session may have committed before it + was cut, so only ``is_database_not_accepting_connections_error`` proves a + non-idempotent write never applied.""" + import prisma + + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) in _SERVER_TERMINATED_SQLSTATES: + return True + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _SERVER_TERMINATED_PHRASES) + @staticmethod def is_lock_timeout_error(e: Exception) -> bool: """True iff Postgres cancelled a statement for exceeding ``lock_timeout`` @@ -302,6 +372,17 @@ class PrismaDBExceptionHandler: return True return _LOCK_TIMEOUT_PHRASE in str(e).lower() + @staticmethod + def is_statement_timeout_error(e: Exception) -> bool: + """True iff Postgres cancelled a statement for exceeding ``statement_timeout`` (SQLSTATE 57014).""" + import prisma + + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): + return False + if PrismaDBExceptionHandler.postgres_sqlstate(e) == _STATEMENT_TIMEOUT_SQLSTATE: + return True + return _STATEMENT_TIMEOUT_PHRASE in str(e).lower() + @staticmethod def postgres_sqlstate(e: Exception) -> str | None: """The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None.""" @@ -393,6 +474,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return True + if PrismaDBExceptionHandler.is_database_lifecycle_error(e): + return True if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e): return True if "cached plan must not change result type" in str(e).lower(): @@ -667,3 +750,18 @@ def db_write_failure_is_requeue_safe(e: Exception) -> bool: return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES + + +def db_write_never_sent(e: Exception) -> bool: + """Whether the database could not take the write at all, because it was unreachable (P1001), + refusing connections (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again.""" + if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): + return True + if PrismaDBExceptionHandler.is_database_capacity_error(e): + return True + if _prisma_error_code(e) == _UNREACHABLE_PRISMA_CODE: + return True + if not PrismaDBExceptionHandler.is_prisma_error(e): + return False + error_message: Final = str(e).lower() + return any(phrase in error_message for phrase in _UNREACHABLE_PHRASES) diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index 4566b10b260..f8cdabe9296 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -32,14 +32,15 @@ class _UpsertTable(Protocol): def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... -class _ModelUsageBatch(Protocol): - litellm_dailymodelusage: _UpsertTable +class ModelUsageBatch(Protocol): + @property + def litellm_dailymodelusage(self) -> _UpsertTable: ... def execute_raw(self, query: LiteralString, *args: object) -> None: ... -class _ModelUsageBatchManager(Protocol): - async def __aenter__(self) -> _ModelUsageBatch: ... +class ModelUsageBatchManager(Protocol): + async def __aenter__(self) -> ModelUsageBatch: ... async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... @@ -88,14 +89,14 @@ def _internal_call_origin(metadata: str) -> object: return decoded.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) -def _date_from_start_time(start_time: datetime | str) -> str | None: +def date_from_start_time(start_time: datetime | str) -> str | None: if isinstance(start_time, datetime): return start_time.date().isoformat() return start_time[:10] if len(start_time) >= 10 else None def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransaction | None: - date: Final = _date_from_start_time(payload["startTime"]) + date: Final = date_from_start_time(payload["startTime"]) origin: Final = _internal_call_origin(payload["metadata"]) if date is None or (origin and origin != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN): return None @@ -116,8 +117,8 @@ def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransa ) -def _model_usage_batch(prisma_client: PrismaClient) -> _ModelUsageBatchManager: - batch: Final[_ModelUsageBatchManager] = prisma_client.db.batch_() +def _model_usage_batch(prisma_client: PrismaClient) -> ModelUsageBatchManager: + batch: Final[ModelUsageBatchManager] = prisma_client.db.batch_() return batch @@ -156,26 +157,16 @@ async def flush_model_usage_transactions( "custom_llm_provider": key.custom_llm_provider, "task_type": key.task_type, } - batcher.litellm_dailymodelusage.upsert( - where={"date_model_group_model_custom_llm_provider_task_type": key_fields}, - data={ - "create": { - **key_fields, - "spend": spend, - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "request_count": len(requests), - "successful_requests": successful, - "failed_requests": failed, - }, - "update": { - "spend": {"increment": spend}, - "prompt_tokens": {"increment": prompt_tokens}, - "completion_tokens": {"increment": completion_tokens}, - "request_count": {"increment": len(requests)}, - "successful_requests": {"increment": successful}, - "failed_requests": {"increment": failed}, - }, + queue_model_usage_upsert( + batcher, + key_fields, + { + "spend": spend, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "request_count": len(requests), + "successful_requests": successful, + "failed_requests": failed, }, ) return @@ -183,3 +174,15 @@ async def flush_model_usage_transactions( if attempt >= n_retry_times: raise await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) + + +def queue_model_usage_upsert( + batcher: ModelUsageBatch, key_fields: Mapping[str, str], increments: Mapping[str, int | float] +) -> None: + batcher.litellm_dailymodelusage.upsert( + where={"date_model_group_model_custom_llm_provider_task_type": key_fields}, + data={ + "create": {**key_fields, **increments}, + "update": {name: {"increment": value} for name, value in increments.items()}, + }, + ) diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index f7ec88945a1..9600fb40ad3 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -23,7 +23,11 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, aggregate_tool_discoveries, ) -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe +from litellm.proxy.db.exception_handler import ( + call_with_db_reconnect_retry, + db_write_failure_is_requeue_safe, + db_write_never_sent, +) from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions @@ -134,8 +138,19 @@ async def flush_tool_discovery_queue(prisma_client: "PrismaClient", tool_discove return try: await batch_upsert_tools(prisma_client=prisma_client, items=items) - except Exception as e: # noqa: BLE001 # a lost tool row only delays its discovery, the scheduler job must not fail - verbose_proxy_logger.debug("flush_tool_discovery_queue error (non-blocking): %s", e) + except Exception as e: # noqa: BLE001 # the rows go back to the queue or are dropped, the scheduler job must not fail + if db_write_never_sent(e): + tool_discovery_queue.requeue(items) + verbose_proxy_logger.warning( + "Tool discovery - %d tool rows went back to the queue for the next tick. Error: %s", len(items), str(e) + ) + return + verbose_proxy_logger.error( + "Tool discovery - dropped %d tool rows: the failed statement may have applied or the database " + "refused the data, so re-sending it is not safe. Error: %s", + len(items), + str(e), + ) async def commit_tool_discoveries( diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index e37ecfd582a..8f929119dee 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -4,10 +4,12 @@ from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.gateway_activity import GatewayActivityStore +from litellm.models.plugin_upload import PluginUploadStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository +from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.repositories.user_banner_repository import UserBannerRepository @@ -34,3 +36,10 @@ async def get_gateway_activity_repository(database: DatabaseDep) -> GatewayActiv GatewayActivityRepositoryDep: TypeAlias = Annotated[ GatewayActivityStore | None, Depends(get_gateway_activity_repository) ] + + +async def get_plugin_upload_repository(database: DatabaseDep) -> PluginUploadStore | None: + return None if database is None else PluginUploadRepository(database.prisma) + + +PluginUploadRepositoryDep: TypeAlias = Annotated[PluginUploadStore | None, Depends(get_plugin_upload_repository)] diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py index 94a0c3e2cf1..dd78896725f 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_archive.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -8,8 +8,11 @@ SHA-256 digest published in the index reproducible for identical uploads. import hashlib import io +import lzma import re +import stat import zipfile +import zlib from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType @@ -23,7 +26,7 @@ MAX_ARCHIVE_ENTRIES: Final = 1000 SKILL_MANIFEST_FILENAME: Final = "SKILL.md" _ZIP_ENTRY_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0) -_ZIP_ENTRY_PERMISSIONS: Final = 0o644 << 16 +_ZIP_ENTRY_MODE: Final = 0o644 _FRONTMATTER_PATTERN: Final = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL) _WINDOWS_DRIVE_PATTERN: Final = re.compile(r"^[A-Za-z]:") _EMPTY_FRONTMATTER: Final[Mapping[str, object]] = MappingProxyType({}) @@ -38,19 +41,26 @@ class SkillArchive: declared_description: str | None -def build_skill_archive(stored_content: bytes) -> SkillArchive | None: - """Return the installable archive for an upload, or None when it holds no root SKILL.md.""" +def read_skill_archive_members( + stored_content: bytes, *, for_upload: bool = False +) -> tuple[tuple[str, bytes, int], ...] | None: try: with zipfile.ZipFile(io.BytesIO(stored_content)) as uploaded: - members: Final = _flattened_members(uploaded) - except (zipfile.BadZipFile, OSError, RuntimeError): + return _flattened_members(uploaded, for_upload=for_upload) + except (zipfile.BadZipFile, OSError, RuntimeError, EOFError, lzma.LZMAError, zlib.error): return None + +def build_skill_archive(stored_content: bytes) -> SkillArchive | None: + """Return the installable archive for an upload, or None when it holds no root SKILL.md.""" + members: Final = read_skill_archive_members(stored_content) if members is None: return None - frontmatter: Final = _manifest_frontmatter(next(data for name, data in members if name == SKILL_MANIFEST_FILENAME)) - content: Final = _repack(members) + frontmatter: Final = _manifest_frontmatter( + next(data for name, data, _ in members if name == SKILL_MANIFEST_FILENAME) + ) + content: Final = pack_skill_archive(members) return SkillArchive( content=content, digest=f"sha256:{hashlib.sha256(content).hexdigest()}", @@ -59,24 +69,58 @@ def build_skill_archive(stored_content: bytes) -> SkillArchive | None: ) -def _flattened_members(uploaded: zipfile.ZipFile) -> tuple[tuple[str, bytes], ...] | None: - infos: Final = tuple(info for info in uploaded.infolist() if not info.is_dir()) +def _flattened_members(uploaded: zipfile.ZipFile, *, for_upload: bool) -> tuple[tuple[str, bytes, int], ...] | None: + infos: Final = tuple(info for info in uploaded.infolist() if for_upload or not info.is_dir()) if not infos or len(infos) > MAX_ARCHIVE_ENTRIES: return None if sum(info.file_size for info in infos) > MAX_ARCHIVE_UNPACKED_BYTES: return None - normalized: Final = tuple((info, _normalized_path(info.filename)) for info in infos) + normalized: Final = tuple( + (info, _normalized_path(info.orig_filename if for_upload else info.filename)) for info in infos + ) if any(path is None for _, path in normalized): return None + if for_upload and ( + len(frozenset(path for _, path in normalized)) != len(normalized) + or any(not _safe_upload_entry(info) for info in infos) + ): + return None - prefix: Final = _common_root_prefix(tuple(path for _, path in normalized if path is not None)) - flattened: Final = tuple((info, path[len(prefix) :]) for info, path in normalized if path is not None) + kept: Final = tuple( + (info, path) + for info, path in normalized + if path is not None and not info.is_dir() and (not for_upload or not _is_macos_metadata(path)) + ) + prefix: Final = _common_root_prefix(tuple(path for _, path in kept)) + flattened: Final = tuple((info, path[len(prefix) :]) for info, path in kept) names: Final = frozenset(name for _, name in flattened) if SKILL_MANIFEST_FILENAME not in names or len(names) != len(flattened): return None - return tuple((name, uploaded.read(info)) for info, name in sorted(flattened, key=lambda member: member[1])) + return tuple( + (name, uploaded.read(info), _entry_mode(info, preserve_executable_bits=for_upload)) + for info, name in sorted(flattened, key=lambda member: member[1]) + ) + + +def _entry_mode(info: zipfile.ZipInfo, *, preserve_executable_bits: bool) -> int: + if not preserve_executable_bits or info.create_system != 3: + return _ZIP_ENTRY_MODE + mode: Final = info.external_attr >> 16 + if stat.S_IFMT(mode) not in (0, stat.S_IFREG): + return _ZIP_ENTRY_MODE + return _ZIP_ENTRY_MODE | (mode & 0o111) + + +def _safe_upload_entry(info: zipfile.ZipInfo) -> bool: + return not info.flag_bits & 1 and ( + info.create_system != 3 or stat.S_IFMT(info.external_attr >> 16) in (0, stat.S_IFREG, stat.S_IFDIR) + ) + + +def _is_macos_metadata(path: str) -> bool: + return path.split("/", 1)[0] == "__MACOSX" or path.rsplit("/", 1)[-1] == ".DS_Store" def _common_root_prefix(paths: tuple[str, ...]) -> str: @@ -117,16 +161,16 @@ def _frontmatter_text(frontmatter: Mapping[str, object], key: str) -> str | None return value.strip() or None -def _zip_entry(name: str) -> zipfile.ZipInfo: +def _zip_entry(name: str, mode: int) -> zipfile.ZipInfo: entry: Final = zipfile.ZipInfo(filename=name, date_time=_ZIP_ENTRY_TIMESTAMP) - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = _ZIP_ENTRY_PERMISSIONS + entry.create_system = 3 + entry.external_attr = mode << 16 return entry -def _repack(members: tuple[tuple[str, bytes], ...]) -> bytes: +def pack_skill_archive(members: tuple[tuple[str, bytes, int], ...], compression: int = zipfile.ZIP_DEFLATED) -> bytes: buffer: Final = io.BytesIO() - with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as repacked: - for name, data in members: - repacked.writestr(_zip_entry(name), data) + with zipfile.ZipFile(buffer, mode="w") as repacked: + for name, data, mode in members: + repacked.writestr(_zip_entry(name, mode), data, compress_type=compression) return buffer.getvalue() diff --git a/litellm/proxy/enterprise_billing/billing_metrics.py b/litellm/proxy/enterprise_billing/billing_metrics.py index 166ece28b83..b6f69e8233c 100644 --- a/litellm/proxy/enterprise_billing/billing_metrics.py +++ b/litellm/proxy/enterprise_billing/billing_metrics.py @@ -2,29 +2,33 @@ Push-based OTLP metering for enterprise litellm deployments. Owns a dedicated OpenTelemetry meter provider and an OTLP/HTTP exporter -authenticated to our global collector with a TLS client certificate. The -collector front end terminates mutual TLS: the client certificate presented -here is validated against our CA at the edge, and the verified subject is -what identifies the deployment. It is intentionally isolated from the global -meter provider so the customer's own OTEL metrics are untouched and ours -never leak into their backend. +authenticated to our global collector with a Cognito bearer token. It is +intentionally isolated from the global meter provider so the customer's own +OTEL metrics are untouched and ours never leak into their backend. -The deployment's identity rides on the TLS client certificate, not on the -payload; the secret license key is never sent as an attribute or header. +The secret license key is never sent as an attribute or header. """ import os -import tempfile -from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, Optional +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypedDict +import requests from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.metrics import Counter from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource +from pydantic import BaseModel, ConfigDict +from requests.auth import AuthBase +from typing_extensions import ReadOnly, Unpack from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.url_utils import is_https_url_with_host from litellm.proxy.middleware.billable_request_metrics_middleware import ( BillableCategory, ) @@ -33,23 +37,15 @@ if TYPE_CHECKING: from litellm.proxy._types import EnterpriseLicenseData ENDPOINT_ENV: Final = "LITELLM_BILLING_METRICS_ENDPOINT" -CLIENT_CERT_ENV: Final = "LITELLM_BILLING_METRICS_CLIENT_CERT" -CLIENT_KEY_ENV: Final = "LITELLM_BILLING_METRICS_CLIENT_KEY" -CA_CERT_ENV: Final = "LITELLM_BILLING_METRICS_CA_CERT" +CLIENT_ID_ENV: Final = "LITELLM_BILLING_METRICS_CLIENT_ID" +CLIENT_SECRET_ENV: Final = "LITELLM_BILLING_METRICS_CLIENT_SECRET" +TOKEN_URL_ENV: Final = "LITELLM_BILLING_METRICS_TOKEN_URL" EXPORT_INTERVAL_ENV: Final = "LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS" DEFAULT_EXPORT_INTERVAL_MS: Final = 60_000 SHUTDOWN_FLUSH_TIMEOUT_MS: Final = 5_000 +_TOKEN_REFRESH_SKEW_SECONDS: Final = 60 _METRICS_PATH: Final = "/v1/metrics" -# The cert env vars take a path or the PEM itself. Secret stores that inject -# values as env content cannot mount them as files, so inline PEM is written out. -_PEM_PREFIX: Final = "-----BEGIN" -_PEM_DIR_PREFIX: Final = "litellm-billing-mtls-" -_PEM_FILE_MODE: Final = 0o600 -_CLIENT_CERT_FILENAME: Final = "client.crt" -_CLIENT_KEY_FILENAME: Final = "client.key" -_CA_CERT_FILENAME: Final = "ca.crt" - METRIC_NAME: Final = "litellm.enterprise.billable_requests" METER_NAME: Final = "litellm.enterprise.billing" @@ -59,21 +55,28 @@ AttributeValue = str | int @dataclass(frozen=True, slots=True) class BillingMetricsConfig: endpoint: str - client_cert_path: str - client_key_path: str - ca_cert_path: str | None + token_url: str + client_id: str + client_secret: str = field(repr=False) export_interval_ms: int litellm_version: str license_id: str | None +class _BillingMetricsOAuthToken(BaseModel): + model_config = ConfigDict(frozen=True, strict=True) + + access_token: str + expires_in: int + + def _metrics_endpoint(endpoint: str) -> str: """The OTLP/HTTP metric exporter wants the full URL including the signal path.""" trimmed: Final = endpoint.rstrip("/") return trimmed if trimmed.endswith(_METRICS_PATH) else f"{trimmed}{_METRICS_PATH}" -def _resource_attributes(config: BillingMetricsConfig) -> dict[str, AttributeValue]: +def resource_attributes(config: BillingMetricsConfig) -> dict[str, AttributeValue]: base: Final[dict[str, AttributeValue]] = { "service.name": "litellm-proxy", "litellm.version": config.litellm_version, @@ -82,7 +85,7 @@ def _resource_attributes(config: BillingMetricsConfig) -> dict[str, AttributeVal return {**base, **license_attr} -def _billable_attributes( +def billable_attributes( category: BillableCategory, route: str, status_code: int, model_id: str | None ) -> dict[str, AttributeValue]: base: Final[dict[str, AttributeValue]] = { @@ -94,23 +97,95 @@ def _billable_attributes( return {**base, **model_attr} -def build_mtls_meter_provider(config: BillingMetricsConfig) -> MeterProvider: - """OTLP/HTTP exporter presenting a TLS client certificate. +class BillingMetricsBearerAuth(AuthBase): + def __init__( + self, + *, + client_id: str, + client_secret: str, + token_url: str, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self._client_id: Final = client_id + self._client_secret: Final = client_secret + self._token_url: Final = token_url + self._clock: Final = clock + self._lock: Final = threading.Lock() + self._cached_token: str | None = None + self._refresh_at: float = 0.0 - The collector's load balancer terminates mutual TLS and validates the client - certificate against our CA. Server verification uses the system trust store - (the collector presents a public web-PKI certificate); ca_cert_path overrides - it only for private/test collectors. - """ - exporter: Final = OTLPMetricExporter( - endpoint=_metrics_endpoint(config.endpoint), - # None -> exporter falls back to the system trust store. - certificate_file=config.ca_cert_path, - client_certificate_file=config.client_cert_path, - client_key_file=config.client_key_path, + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + token: Final = self._get_token() + authenticated_request: Final = request.copy() + authenticated_request.headers["Authorization"] = f"Bearer {token}" + return authenticated_request + + def invalidate(self) -> None: + with self._lock: + self._cached_token = None + self._refresh_at = 0.0 + + def _get_token(self) -> str: + with self._lock: + now: Final = self._clock() + if self._cached_token is not None and now < self._refresh_at: + return self._cached_token + token, cache_seconds = self._fetch_token() + self._cached_token = token + self._refresh_at = now + cache_seconds + return token + + def _fetch_token(self) -> tuple[str, float]: + try: + # The OTel periodic exporter's background thread makes blocking token I/O safe here. + response: Final = requests.post( + self._token_url, + auth=(self._client_id, self._client_secret), + data={"grant_type": "client_credentials", "scope": "telemetry/write"}, + timeout=10, + ) + response.raise_for_status() + token_response: Final = _BillingMetricsOAuthToken.model_validate(response.json()) + return token_response.access_token, max(0.0, float(token_response.expires_in) - _TOKEN_REFRESH_SKEW_SECONDS) + except (requests.RequestException, ValueError): + raise requests.RequestException("Billing metrics token request failed") from None + + +class _SessionHookKwargs(TypedDict, total=False): + stream: ReadOnly[object] + timeout: ReadOnly[object] + verify: ReadOnly[object] + cert: ReadOnly[object] + proxies: ReadOnly[object] + + +class _BillingMetricsBearerSession(requests.Session): + def __init__(self, auth: BillingMetricsBearerAuth) -> None: + super().__init__() + self._bearer_auth: Final = auth + self.auth = auth + self.hooks = {"response": [self._invalidate_token_on_unauthorized]} + + def _invalidate_token_on_unauthorized( + self, + response: requests.Response, + **kwargs: Unpack[_SessionHookKwargs], # kwargs-ok: requests hooks forward send options + ) -> requests.Response: + if response.status_code == 401: + self._bearer_auth.invalidate() + return response + + +def build_bearer_meter_provider(config: BillingMetricsConfig) -> MeterProvider: + auth: Final = BillingMetricsBearerAuth( + client_id=config.client_id, + client_secret=config.client_secret, + token_url=config.token_url, ) + session: Final = _BillingMetricsBearerSession(auth) + exporter: Final = OTLPMetricExporter(endpoint=_metrics_endpoint(config.endpoint), session=session) reader: Final = PeriodicExportingMetricReader(exporter, export_interval_millis=config.export_interval_ms) - return MeterProvider(metric_readers=[reader], resource=Resource.create(_resource_attributes(config))) + return MeterProvider(metric_readers=[reader], resource=Resource.create(resource_attributes(config))) class BillingMetricsRecorder: @@ -125,7 +200,7 @@ class BillingMetricsRecorder: ) def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: str | None) -> None: - self._counter.add(1, _billable_attributes(category, route, status_code, model_id)) + self._counter.add(1, billable_attributes(category, route, status_code, model_id)) def shutdown(self) -> None: """Final flush + exporter-thread stop. Without this, up to one export @@ -146,115 +221,34 @@ def _export_interval_ms() -> int: return DEFAULT_EXPORT_INTERVAL_MS -@dataclass(frozen=True, slots=True) -class _CredentialPaths: - client_cert_path: str - client_key_path: str - ca_cert_path: str | None - - -def _is_pem_content(value: str) -> bool: - return value.lstrip().startswith(_PEM_PREFIX) - - -def _write_pem(directory: str, filename: str, pem: str) -> str: - path: Final = os.path.join(directory, filename) - with open(path, "w", encoding="utf-8") as handle: - handle.write(pem if pem.endswith("\n") else f"{pem}\n") - os.chmod(path, _PEM_FILE_MODE) - return path - - -def _resolve_credential_paths(*, client_cert: str, client_key: str, ca_cert: str | None) -> _CredentialPaths: - """ - Accept either a filesystem path or inline PEM content for each credential. - - Secret stores that inject values as environment content rather than mounted - files (ECS tasks reading AWS Secrets Manager, Cloud Run reading Secret - Manager) can only deliver the certificate as a string. The OTLP exporter - takes paths, so inline PEM is written to a private directory once, when the - recorder is built. Raises OSError if that write fails; the caller disables - metering rather than propagating. - """ - inline: Final = tuple(value for value in (client_cert, client_key, ca_cert) if value and _is_pem_content(value)) - if not inline: - return _CredentialPaths(client_cert, client_key, ca_cert) - - # mkdtemp is 0o700, so the 0o600 key file it holds is unreachable by other users. - directory: Final = tempfile.mkdtemp(prefix=_PEM_DIR_PREFIX) - return _CredentialPaths( - client_cert_path=( - _write_pem(directory, _CLIENT_CERT_FILENAME, client_cert) if _is_pem_content(client_cert) else client_cert - ), - client_key_path=( - _write_pem(directory, _CLIENT_KEY_FILENAME, client_key) if _is_pem_content(client_key) else client_key - ), - ca_cert_path=( - _write_pem(directory, _CA_CERT_FILENAME, ca_cert) if ca_cert and _is_pem_content(ca_cert) else ca_cert - ), - ) - - def load_billing_metrics_config( - *, license_data: Optional["EnterpriseLicenseData"], litellm_version: str + *, license_data: "EnterpriseLicenseData | None", litellm_version: str ) -> BillingMetricsConfig | None: - endpoint: Final = os.getenv(ENDPOINT_ENV) - client_cert: Final = os.getenv(CLIENT_CERT_ENV) - client_key: Final = os.getenv(CLIENT_KEY_ENV) - # Optional: only for private/test collectors whose server cert is not on the - # public web PKI. The production collector needs no CA override. - ca_cert: Final = os.getenv(CA_CERT_ENV) - - missing: Final = [ - name - for name, value in ( - (ENDPOINT_ENV, endpoint), - (CLIENT_CERT_ENV, client_cert), - (CLIENT_KEY_ENV, client_key), - ) - if not value - ] - if not endpoint or not client_cert or not client_key: + env: Final = MappingProxyType( + {name: os.getenv(name) or "" for name in (ENDPOINT_ENV, CLIENT_ID_ENV, CLIENT_SECRET_ENV, TOKEN_URL_ENV)} + ) + missing: Final = tuple(name for name, value in env.items() if not value) + if missing: verbose_proxy_logger.warning( "Enterprise billing metrics disabled: licensed deployment missing config (%s)", ", ".join(missing), ) return None - try: - paths: Final = _resolve_credential_paths(client_cert=client_cert, client_key=client_key, ca_cert=ca_cert) - except OSError as exc: - verbose_proxy_logger.warning( - "Enterprise billing metrics disabled: could not write inline certificate content to disk: %s", exc - ) - return None - - # Report the variable names, never their values. A value that is neither a - # readable path nor recognizable PEM is still secret material, and this - # warning would otherwise copy a client key straight into the proxy logs. - unreadable: Final = [ + insecure_urls: Final = tuple( env_name - for env_name, path in ( - (CLIENT_CERT_ENV, paths.client_cert_path), - (CLIENT_KEY_ENV, paths.client_key_path), - (CA_CERT_ENV, paths.ca_cert_path), - ) - if path and not os.path.isfile(path) - ] - if unreadable: - verbose_proxy_logger.warning( - "Enterprise billing metrics disabled: %s did not resolve to a readable certificate file. " - "Set each to a file path, or to inline PEM content beginning with '%s'.", - ", ".join(unreadable), - _PEM_PREFIX, - ) + for env_name, url in ((ENDPOINT_ENV, env[ENDPOINT_ENV]), (TOKEN_URL_ENV, env[TOKEN_URL_ENV])) + if not is_https_url_with_host(url) + ) + if insecure_urls: + verbose_proxy_logger.warning("Enterprise billing metrics disabled: %s must use HTTPS", insecure_urls[0]) return None return BillingMetricsConfig( - endpoint=endpoint, - client_cert_path=paths.client_cert_path, - client_key_path=paths.client_key_path, - ca_cert_path=paths.ca_cert_path, + endpoint=env[ENDPOINT_ENV], + client_id=env[CLIENT_ID_ENV], + client_secret=env[CLIENT_SECRET_ENV], + token_url=env[TOKEN_URL_ENV], export_interval_ms=_export_interval_ms(), litellm_version=litellm_version, license_id=(license_data or {}).get("user_id"), @@ -282,7 +276,7 @@ _ACTIVE_RECORDER: Final = _ActiveRecorderRegistry() def build_billing_metrics_recorder( - *, premium: bool, license_data: Optional["EnterpriseLicenseData"], litellm_version: str + *, premium: bool, license_data: "EnterpriseLicenseData | None", litellm_version: str ) -> BillingMetricsRecorder | None: """Build the recorder, or None when the deployment is not licensed or metering is unconfigured.""" if not premium: @@ -296,14 +290,12 @@ def build_billing_metrics_recorder( return None try: - recorder: Final = BillingMetricsRecorder(build_mtls_meter_provider(config)) + provider: Final = build_bearer_meter_provider(config) + recorder: Final = BillingMetricsRecorder(provider) except Exception as exc: # noqa: BLE001 -- metering must never break proxy startup verbose_proxy_logger.warning("Enterprise billing metrics disabled: failed to initialize exporter: %s", exc) return None _ACTIVE_RECORDER.set(recorder) - # The only positive signal that this component meters. Without it, a silent - # return above is indistinguishable from a working exporter in the logs, and - # a component that carries the cert but no license would look healthy. verbose_proxy_logger.info( "Enterprise billing metrics enabled: exporting to %s every %d ms", config.endpoint, diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 4d910cde603..9b9e0949a1a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -22,7 +22,7 @@ import json import time from collections import Counter, OrderedDict from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeGuard +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard from urllib.parse import urlparse import httpx @@ -507,6 +507,8 @@ class _CompressionResult: class CompresrGuardrail(CustomGuardrail): + server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({COMPRESR_RETRIEVE_TOOL_NAME}) + def __init__( self, api_base: str | None = None, diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py index 44705c4d166..a2688312b38 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py @@ -32,7 +32,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}), unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), fail_on_error=getattr(litellm_params, "fail_on_error", True), - forward_messages_as_sent=getattr(litellm_params, "forward_messages_as_sent", False), + forward_messages_as_sent=litellm_params.forward_messages_as_sent, extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index d69fe4068e0..4880444ada1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -504,8 +504,7 @@ class NomaGuardrail(CustomGuardrail): messages[i]["content"] = anonymized_content break - def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str, position: int): - """Replace the content of the choice at ``position`` with its anonymized version.""" + def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str, position: int) -> None: if not isinstance(response, litellm.ModelResponse): return diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 5f9dc4a7a31..5f99b1a4611 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -1,13 +1,16 @@ import json import re -from collections.abc import AsyncGenerator, AsyncIterable, Mapping, Sequence -from typing import Any, Final, Literal, TypedDict +from collections.abc import AsyncGenerator, AsyncIterable, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from functools import partial, reduce +from itertools import chain, groupby +from types import MappingProxyType +from typing import Any, Final, Literal, TypeAlias, TypedDict, TypeVar from fastapi import HTTPException from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required -from litellm import ChatCompletionToolParam from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache from litellm.completion_extras.litellm_responses_transformation.transformation import ( @@ -19,6 +22,9 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none +from litellm.litellm_core_utils.tool_fields import declared_function_name, qualified_member_name +from litellm.llms.base_llm.guardrail_translation.utils import anthropic_tool_names from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -30,7 +36,10 @@ from litellm.proxy.guardrails.anthropic_sse import ( is_raw_sse_stream, is_sse_error_stream, ) +from litellm.responses.additional_tools import ADDITIONAL_TOOLS_INPUT_ITEM_TYPE from litellm.types.guardrails import GuardrailEventHooks, LitellmParams +from litellm.types.llms.anthropic import ANTHROPIC_HOSTED_TOOLS +from litellm.types.llms.vertex_ai import VertexToolName from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( PermissionError, ToolPermissionRule, @@ -48,8 +57,48 @@ from litellm.types.utils import ( ) GUARDRAIL_NAME: Final = "tool_permission" +_MCP_TOOLSET_TYPE: Final = "mcp_toolset" +_HOSTED_SERVER_NAME_FIELDS: Final = MappingProxyType({"mcp": "server_label", _MCP_TOOLSET_TYPE: "mcp_server_name"}) +_TOOL_CARRYING_INPUT_TYPES: Final = frozenset({"tool_search_output", ADDITIONAL_TOOLS_INPUT_ITEM_TYPE}) +_DECLARATION_KEYS: Final = ("functionDeclarations", "function_declarations") +_NON_TOOL_PART_KEYS: Final = frozenset({*_DECLARATION_KEYS, "cachePoint", "type"}) +_TOOL_CONFIG_KEYS: Final = ("toolConfig", "tool_config") +_FUNCTION_CALLING_CONFIG_KEYS: Final = ("functionCallingConfig", "function_calling_config") +_ALLOWED_FUNCTION_NAMES_KEYS: Final = ("allowedFunctionNames", "allowed_function_names") +_FORCED_FUNCTION_CALLING_MODES: Final = frozenset({"ANY", "VALIDATED"}) +_NAME_IDENTIFIED_TYPES: Final = frozenset({"function", "custom", "namespace", *_HOSTED_SERVER_NAME_FIELDS}) +_TEXT_EDITOR_TOOL_NAMES: Final = frozenset({"str_replace_editor", "str_replace_based_edit_tool"}) +_TYPE_VERSION_SUFFIX: Final = re.compile(r"_\d{8}$") +_GEMINI_TYPED_HOSTED_TOOLS: Final = ("computer_use", "web_search", "web_search_preview") +_NAMED_CHOICE_TYPES: Final = frozenset({"tool", "function", "custom", "allowed_tools", "any", "auto", "none"}) _DECODED_ARGUMENTS: Final = TypeAdapter(object) _REQUEST_DATA: Final = TypeAdapter(object) +_ToolTarget: TypeAlias = tuple[str | None, str | None] +_JudgedCalls: TypeAlias = Mapping[str, tuple[tuple[str, str], ...]] +_GroupedT: Final = TypeVar("_GroupedT") + + +def _snake_case(name: str) -> str: + return re.sub(r"(? str: + head, *rest = name.split("_") + return head + "".join(part.title() for part in rest) + + +_GEMINI_HOSTED_TOOL_KEYS: Final = frozenset( + chain.from_iterable((_snake_case(tool.value), _camel_case(tool.value)) for tool in VertexToolName) +) + + +def _name_identifies_tool(name: str, tool_type: str) -> bool: + stem: Final = _TYPE_VERSION_SUFFIX.sub("", tool_type) + return ( + tool_type in _NAME_IDENTIFIED_TYPES + or name == stem + or (stem == ANTHROPIC_HOSTED_TOOLS.TEXT_EDITOR.value and name in _TEXT_EDITOR_TOOL_NAMES) + ) def _object_mapping(value: object) -> Mapping[str, object] | None: @@ -68,6 +117,356 @@ def _function_tool_name(tool: object) -> str | None: return name if isinstance(name, str) else None +def _non_empty_string(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _request_tool_names(tool: object) -> tuple[str, ...]: + fields: Final = _object_mapping(tool) + custom: Final = ( + _object_mapping(fields.get("custom")) if fields is not None and fields.get("type") == "custom" else None + ) + custom_name: Final = _non_empty_string(custom.get("name")) if custom is not None else None + server_field: Final = _HOSTED_SERVER_NAME_FIELDS.get(str(fields.get("type"))) if fields is not None else None + server_name: Final = ( + _non_empty_string(fields.get(server_field)) if fields is not None and server_field is not None else None + ) + extra_names: Final = tuple(name for name in (_function_name(tool), custom_name, server_name) if name is not None) + return tuple(dict.fromkeys(anthropic_tool_names(tool) + extra_names)) + + +def _function_name(tool: object) -> str | None: + fields: Final = _object_mapping(tool) + function: Final = _object_mapping(fields.get("function")) if fields is not None else None + return _non_empty_string(function.get("name")) if function is not None else None + + +def _namespace_function_targets(tool: object) -> tuple[_ToolTarget, ...]: + function_name: Final = _function_name(tool) + return ((function_name, "function"),) if function_name is not None else () + + +def _gemini_hosted_key_targets(tool: object) -> tuple[_ToolTarget, ...]: + fields: Final = _object_mapping(tool) + if fields is None or fields.get("type") in _GEMINI_TYPED_HOSTED_TOOLS: + return () + keys: Final = tuple(key for key in fields if key != "type") + return (_part_target(fields, keys[0]),) if len(keys) == 1 and keys[0] in _GEMINI_HOSTED_TOOL_KEYS else () + + +def _is_namespace(tool: object) -> bool: + fields: Final = _object_mapping(tool) + return fields is not None and fields.get("type") == "namespace" + + +def _namespace_members(tool: object) -> Sequence[object]: + fields: Final = _object_mapping(tool) + members: Final = _object_list(fields.get("tools")) if fields is not None and _is_namespace(tool) else None + return members or () + + +def _returned_namespace(tool_call: ChatCompletionMessageToolCall) -> str | None: + tags: Final = as_str_mapping(attribute_or_none(tool_call, "provider_specific_fields")) + return _non_empty_string(tags.get("namespace")) if tags is not None else None + + +def _judged_call_names( + tool_call: ChatCompletionMessageToolCall, returned_name: str, judged_names: _JudgedCalls +) -> tuple[tuple[str, str], ...]: + namespace: Final = _returned_namespace(tool_call) + if namespace is not None: + return ((qualified_member_name(namespace, returned_name), returned_name),) + return judged_names.get(returned_name, ()) + + +def _pair_namespace(qualified: str, bare: str) -> str | None: + suffix: Final = qualified_member_name("", bare) + if qualified == bare or not qualified.endswith(suffix): + return None + return _non_empty_string(qualified.removesuffix(suffix)) + + +def _qualified_target(namespace: str | None, target: _ToolTarget) -> _ToolTarget: + name, tool_type = target + return (qualified_member_name(namespace, name), tool_type) if namespace is not None and name is not None else target + + +def _member_name(member: object) -> str | None: + fields: Final = _object_mapping(member) + return _non_empty_string(fields.get("name")) if fields is not None else None + + +def _declaration_targets(declaration: object) -> tuple[_ToolTarget, ...]: + name: Final = _member_name(declaration) + return ((name, "function"),) if name is not None else () + + +@dataclass(frozen=True, slots=True) +class _MemberGroup: + key: str + members: Sequence[object] + targets_of: Callable[[object], tuple[_ToolTarget, ...]] + + +def _member_group( + tool: object, namespace_member_targets: Callable[[str | None, object], tuple[_ToolTarget, ...]] +) -> _MemberGroup | None: + fields: Final = _object_mapping(tool) + if fields is None: + return None + members: Final = _namespace_members(tool) + if members: + return _MemberGroup("tools", members, partial(namespace_member_targets, _member_name(tool))) + key: Final = next((key for key in _DECLARATION_KEYS if _object_list(fields.get(key))), None) + declarations: Final = _object_list(fields.get(key)) if key is not None else None + return _MemberGroup(key, declarations, _declaration_targets) if key is not None and declarations else None + + +def _keyed_tool_fields(tool: object) -> Mapping[str, object] | None: + fields: Final = _object_mapping(tool) + if fields is None or isinstance(fields.get("type"), str) or _request_tool_names(tool): + return None + return fields + + +def _part_keys(fields: Mapping[str, object]) -> tuple[str, ...]: + return tuple(key for key in fields if key not in _NON_TOOL_PART_KEYS) + + +def _part_target(fields: Mapping[str, object], key: str) -> _ToolTarget: + return (None if key in _GEMINI_HOSTED_TOOL_KEYS else _member_name(fields.get(key)), key) + + +def _input_item_tools(item: object) -> Sequence[object] | None: + fields: Final = _object_mapping(item) + carries_tools: Final = fields is not None and _non_empty_string(fields.get("type")) in _TOOL_CARRYING_INPUT_TYPES + return _object_list(fields.get("tools")) if fields is not None and carries_tools else None + + +def _request_list(data: object, key: str) -> Sequence[object] | None: + fields: Final = _object_mapping(data) + return _object_list(fields.get(key)) if fields is not None else None + + +def _input_items(data: object) -> Sequence[object]: + return _request_list(data, "input") or () + + +def _tools_in_items(items: Sequence[object]) -> tuple[object, ...]: + return tuple(chain.from_iterable(_input_item_tools(item) or () for item in items)) + + +def _carries_tools(items: Sequence[object]) -> bool: + return any(_input_item_tools(item) is not None for item in items) + + +def _pair_key(pair: tuple[str, object]) -> str: + return pair[0] + + +def _grouped_values(pairs: Iterable[tuple[str, _GroupedT]]) -> Mapping[str, tuple[_GroupedT, ...]]: + groups: Final = groupby(sorted(pairs, key=_pair_key), key=_pair_key) + return MappingProxyType({key: tuple(dict.fromkeys(value for _, value in group)) for key, group in groups}) + + +def _request_layers(data: object) -> tuple[Mapping[str, object], ...]: + fields: Final = _object_mapping(data) + extra_body: Final = _object_mapping(fields.get("extra_body")) if fields is not None else None + return tuple(layer for layer in (fields, extra_body) if layer is not None) + + +def _layer_sent_tools(layer: Mapping[str, object]) -> tuple[object, ...]: + return ( + *(_request_list(layer, "tools") or ()), + *(_request_list(layer, "functions") or ()), + *_tools_in_items(_input_items(layer)), + ) + + +def _sent_tools(data: object) -> tuple[object, ...]: + return tuple(chain.from_iterable(map(_layer_sent_tools, _request_layers(data)))) + + +def _sent_fields(data: object) -> Mapping[str, object]: + return dict(chain.from_iterable(layer.items() for layer in _request_layers(data))) + + +@dataclass(frozen=True, slots=True) +class _KeptLayer: + fields: Mapping[str, object] + tools: tuple[object, ...] | None + functions: tuple[object, ...] | None + mcp_servers: tuple[object, ...] | None + input: tuple[object, ...] | None + + def sent_tools(self) -> tuple[object, ...]: + kept_input: Final = _input_items(self.fields) if self.input is None else self.input + return (*(self.tools or ()), *(self.functions or ()), *_tools_in_items(kept_input)) + + def kept_lists(self) -> dict[str, object]: + lists: Final = ( + ("tools", self.tools), + ("functions", self.functions), + ("mcp_servers", self.mcp_servers), + ("input", self.input), + ) + return {key: list(kept) for key, kept in lists if kept is not None} + + def overlaid_by(self, over: "_KeptLayer") -> "_KeptLayer": + return _KeptLayer( + fields={**self.fields, **over.fields}, + tools=over.tools if "tools" in over.fields else self.tools, + functions=over.functions if "functions" in over.fields else self.functions, + mcp_servers=over.mcp_servers if "mcp_servers" in over.fields else self.mcp_servers, + input=over.input if "input" in over.fields else self.input, + ) + + +def _mcp_server_target(server: object) -> _ToolTarget: + fields: Final = _object_mapping(server) + return (_non_empty_string(fields.get("name")) if fields is not None else None, _MCP_TOOLSET_TYPE) + + +def _reference_name(part: object) -> str | None: + fields: Final = _object_mapping(part) + is_reference: Final = fields is not None and fields.get("type") == "tool_reference" + return _non_empty_string(fields.get("tool_name")) if fields is not None and is_reference else None + + +def _references_one_of(part: object, names: frozenset[str]) -> bool: + name: Final = _reference_name(part) + return name is not None and name in names + + +def _without_references_to(parts: Sequence[object], names: frozenset[str]) -> Sequence[object]: + removed: Final = [name for name in map(_reference_name, parts) if name is not None and name in names] + kept: Final = [part for part in parts if not _references_one_of(part, names)] + if kept or not removed: + return kept + return [{"type": "text", "text": f"Permission denied: tool {', '.join(removed)} is not available"}] + + +def _tool_result_without_references(fields: Mapping[str, object], names: frozenset[str]) -> object: + parts: Final = _object_list(fields.get("content")) + return {**fields, "content": _without_references_to(parts, names)} if parts else fields + + +def _search_result_without_references(fields: Mapping[str, object], names: frozenset[str]) -> object: + result: Final = _object_mapping(fields.get("content")) + references: Final = _object_list(result.get("tool_references")) if result is not None else None + if result is None or references is None: + return fields + kept: Final = [reference for reference in references if not _references_one_of(reference, names)] + return {**fields, "content": {**result, "tool_references": kept}} + + +def _without_references_in_block(block: object, names: frozenset[str]) -> object: + fields: Final = _object_mapping(block) + if fields is None: + return block + match fields.get("type"): + case "tool_result": + return _tool_result_without_references(fields, names) + case "tool_search_tool_result": + return _search_result_without_references(fields, names) + case _: + return block + + +def _without_references_in_message(message: object, names: frozenset[str]) -> object: + fields: Final = _object_mapping(message) + content: Final = _object_list(fields.get("content")) if fields is not None else None + if fields is None or content is None: + return message + if fields.get("role") == "tool": + return {**fields, "content": _without_references_to(content, names)} + return {**fields, "content": [_without_references_in_block(block, names) for block in content]} + + +def _allowed_tool_entries(tool_choice: Mapping[str, object]) -> Sequence[object] | None: + if tool_choice.get("type") != "allowed_tools": + return None + nested: Final = _object_mapping(tool_choice.get("allowed_tools")) + return _object_list((nested if nested is not None else tool_choice).get("tools")) + + +def _is_removed_entry(entry: object, removed_names: frozenset[str], removed_types: frozenset[str]) -> bool: + names: Final = _request_tool_names(entry) + if names: + return any(name in removed_names for name in names) + fields: Final = _object_mapping(entry) + entry_type: Final = _non_empty_string(fields.get("type")) if fields is not None else None + return entry_type is not None and entry_type in removed_types + + +def _with_allowed_tool_entries(tool_choice: Mapping[str, object], entries: list[object]) -> dict[str, object]: + nested: Final = _object_mapping(tool_choice.get("allowed_tools")) + if nested is None: + return {**tool_choice, "tools": entries} + return {**tool_choice, "allowed_tools": {**nested, "tools": entries}} + + +def _is_denied_target( + target: _ToolTarget, denied_targets: frozenset[_ToolTarget], denied_names: frozenset[str] +) -> bool: + return target in denied_targets or target[0] in denied_names + + +def _kept_members(group: _MemberGroup | None, is_denied: Callable[[_ToolTarget], bool]) -> tuple[object, ...]: + if group is None: + return () + return tuple(member for member in group.members if not any(map(is_denied, group.targets_of(member)))) + + +def _none_tool_choice(tool_choice: object) -> object: + choice: Final = _object_mapping(tool_choice) + return {"type": "none"} if choice is not None and choice.get("type") in ("tool", "any") else "none" + + +def _requires_a_tool(tool_choice: object) -> bool: + choice: Final = _object_mapping(tool_choice) + return tool_choice == "required" or (choice is not None and choice.get("type") == "any") + + +def _kept_allowed_function_names(config: Mapping[str, object], removed_names: frozenset[str]) -> Mapping[str, object]: + listed: Final = tuple((key, _object_list(config.get(key))) for key in _ALLOWED_FUNCTION_NAMES_KEYS) + return { + key: [name for name in names if _non_empty_string(name) not in removed_names] + for key, names in listed + if names is not None + } + + +def _rewritten_function_calling_config( + config: Mapping[str, object], removed_names: frozenset[str], functions_remain: bool +) -> Mapping[str, object]: + kept_names: Final = _kept_allowed_function_names(config, removed_names) + had_names: Final = any(_object_list(config.get(key)) for key in _ALLOWED_FUNCTION_NAMES_KEYS) + every_listed_name_removed: Final = had_names and not any(kept_names.values()) + forces_a_function: Final = str(config.get("mode", "")).upper() in _FORCED_FUNCTION_CALLING_MODES + if not every_listed_name_removed and (functions_remain or not forces_a_function): + return {**config, **kept_names} + return {**{key: value for key, value in config.items() if key not in _ALLOWED_FUNCTION_NAMES_KEYS}, "mode": "NONE"} + + +def _rewritten_tool_config_field( + key: str, value: object, removed_names: frozenset[str], functions_remain: bool +) -> object: + config: Final = _object_mapping(value) + if key not in _FUNCTION_CALLING_CONFIG_KEYS or config is None: + return value + return _rewritten_function_calling_config(config, removed_names, functions_remain) + + +def _rewritten_tool_config(tool_config: object, removed_names: frozenset[str], functions_remain: bool) -> object: + fields: Final = _object_mapping(tool_config) + if fields is None: + return tool_config + return { + key: _rewritten_tool_config_field(key, value, removed_names, functions_remain) for key, value in fields.items() + } + + class _ToolPermissionRuleFields(TypedDict, total=False): """The config-file shape a :class:`ToolPermissionRule` is built from.""" @@ -258,7 +657,7 @@ class ToolPermissionGuardrail(CustomGuardrail): name_required: Final = rule.tool_name is not None type_required: Final = rule.tool_type is not None - name_matched: Final = self._matches_regex(name_pattern, tool_name) if name_required else True + name_matched: Final = self._matches_regex(name_pattern, tool_name or tool_type) if name_required else True type_matched: Final = self._matches_regex(type_pattern, tool_type) if type_required else True overall_match: Final = name_matched and type_matched @@ -320,6 +719,72 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug(message) return is_allowed, None, message + def _is_name_rule_for(self, rule: ToolPermissionRule, tool_name: str, tool_type: str | None) -> bool: + return rule.tool_name is not None and self._rule_matches_tool(rule, tool_name=tool_name, tool_type=tool_type)[0] + + def _decides_call( + self, rule: ToolPermissionRule, tool_name: str, tool_call: ChatCompletionMessageToolCall | None + ) -> bool: + if tool_call is None or not rule.allowed_param_patterns: + return True + arguments, parse_error = self._parse_tool_call_arguments(tool_call) + if parse_error or not arguments: + return True + return self._patterns_match_for_rule(arguments=arguments, rule=rule, tool_name=tool_name)[0] + + def _first_name_rule( + self, tool_name: str, tool_type: str | None, tool_call: ChatCompletionMessageToolCall | None + ) -> ToolPermissionRule | None: + return next( + ( + rule + for rule in self.rules + if self._is_name_rule_for(rule, tool_name, tool_type) and self._decides_call(rule, tool_name, tool_call) + ), + None, + ) + + def _narrower_name( + self, + name: str, + other: str, + tool_type: str | None, + tool_call: ChatCompletionMessageToolCall | None, + ) -> str: + rule: Final = self._first_name_rule(name, tool_type, tool_call) + other_rule: Final = self._first_name_rule(other, tool_type, tool_call) + if other_rule is None or other_rule is rule: + return name + if rule is None or self._is_name_rule_for(rule, other, tool_type): + return other + if self._is_name_rule_for(other_rule, name, tool_type): + return name + return other if other_rule.decision == "deny" else name + + def _deciding_member_name( + self, + qualified: str, + bare: str, + tool_type: str | None, + tool_call: ChatCompletionMessageToolCall | None = None, + ) -> str: + member: Final = self._narrower_name(qualified, bare, tool_type, tool_call) + namespace: Final = _pair_namespace(qualified, bare) + return member if namespace is None else self._narrower_name(member, namespace, tool_type, tool_call) + + def _check_target_permission( + self, target: _ToolTarget, bare_names: Mapping[str, tuple[str, ...]] + ) -> tuple[bool, str | None, str | None]: + name, tool_type = target + candidates: Final = tuple(sorted(bare_names.get(name, ()))) if name is not None else () + if name is None or not candidates: + return self._check_tool_permission(name, tool_type) + results: Final = tuple( + self._check_tool_permission(self._deciding_member_name(name, bare_name, tool_type), tool_type) + for bare_name in candidates + ) + return next((result for result in results if not result[0]), results[0]) + def _parse_tool_call_arguments( self, tool_call: ChatCompletionMessageToolCall ) -> tuple[Mapping[str, object] | None, str | None]: @@ -411,9 +876,9 @@ class ToolPermissionGuardrail(CustomGuardrail): return True, None def _get_permission_for_tool_call( - self, tool_call: ChatCompletionMessageToolCall + self, tool_call: ChatCompletionMessageToolCall, judged_name: str | None = None ) -> tuple[bool, str | None, str | None]: - tool_name: Final = tool_call.function.name if tool_call.function else None + tool_name: Final = judged_name or (tool_call.function.name if tool_call.function else None) tool_type: Final = getattr(tool_call, "type", None) if not tool_name and not tool_type: return self.default_action == "allow", None, None @@ -556,10 +1021,46 @@ class ToolPermissionGuardrail(CustomGuardrail): tool_call for block in content if (tool_call := self._anthropic_tool_use_to_tool_call(block)) is not None ) + def _returned_call_names(self, data: Mapping[str, object]) -> _JudgedCalls: + sent_tools: Final = _sent_tools(_REQUEST_DATA.validate_python(data)) + member_names: Final = tuple(chain.from_iterable(map(self._bare_member_names, sent_tools))) + if not member_names: + return MappingProxyType({}) + top_level_tools: Final = (tool for tool in sent_tools if not _is_namespace(tool)) + top_level_names: Final = frozenset(name for name in map(declared_function_name, top_level_tools) if name) + listed: Final = frozenset(name for name, _ in self._collect_request_tools(data) if name is not None) + own_names: Final = frozenset(bare for _, bare in member_names if bare in listed) | top_level_names + returned_members: Final = _grouped_values( + chain( + ((bare, (name, bare)) for name, bare in member_names if bare not in top_level_names), + ((name, (name, bare)) for name, bare in member_names if name not in top_level_names), + ) + ) + return MappingProxyType( + { + name: (*members, *(((name, name),) if name in own_names else ())) + for name, members in returned_members.items() + } + ) + + def _judged_name(self, tool_call: ChatCompletionMessageToolCall, qualified: str, bare: str) -> str: + return qualified if qualified == bare else self._deciding_member_name(qualified, bare, "function", tool_call) + + def _judge_tool_call( + self, tool_call: ChatCompletionMessageToolCall, judged_names: _JudgedCalls + ) -> tuple[bool, str | None, str | None]: + returned_name: Final = tool_call.function.name if tool_call.function else None + names: Final = _judged_call_names(tool_call, returned_name, judged_names) if returned_name is not None else () + if returned_name is None or not names: + return self._get_permission_for_tool_call(tool_call) + judged: Final = dict.fromkeys(self._judged_name(tool_call, qualified, bare) for qualified, bare in names) + results: Final = tuple(self._get_permission_for_tool_call(tool_call, name) for name in judged) + return next((result for result in results if not result[0]), results[0]) + def _evaluate_tool_calls( - self, tool_calls: Sequence[ChatCompletionMessageToolCall] + self, tool_calls: Sequence[ChatCompletionMessageToolCall], judged_names: _JudgedCalls ) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]: - checked: Final = tuple((tool_call, *self._get_permission_for_tool_call(tool_call)) for tool_call in tool_calls) + checked: Final = tuple((tool_call, *self._judge_tool_call(tool_call, judged_names)) for tool_call in tool_calls) for _tool_call, is_allowed, _rule_id, message in checked: if not is_allowed and message is not None: @@ -619,112 +1120,311 @@ class ToolPermissionGuardrail(CustomGuardrail): if not any(_is_tool_use_block(block) for block in kept_blocks): response["stop_reason"] = "end_turn" # rebind-ok: dropping every tool_use ends the turn - def _get_request_tool_name(self, tool: object) -> tuple[str | None, str | None]: - tool_type: Final = self._get_mapping_value(tool, "type") - if tool_type != "function": - return None, tool_type + def _unqualified_member_targets(self, member: object) -> tuple[_ToolTarget, ...]: + fields: Final = _object_mapping(member) + if fields is None: + return () + targets: Final = self._get_request_tool_targets(member) + own_name: Final = _member_name(member) + judged_names: Final = frozenset(name for name, _ in targets) + own_target: Final = ( + ((own_name, _non_empty_string(fields.get("type"))),) + if own_name is not None and own_name not in judged_names + else () + ) + return targets + own_target - function: Final = self._get_mapping_value(tool, "function") - tool_name: Final = self._get_mapping_value(function, "name") - return tool_name, tool_type + def _namespace_member_targets(self, namespace: str | None, member: object) -> tuple[_ToolTarget, ...]: + return tuple(_qualified_target(namespace, target) for target in self._unqualified_member_targets(member)) - def _get_legacy_function_name(self, function: object) -> str | None: - return self._get_mapping_value(function, "name") + def _bare_member_names(self, tool: object) -> tuple[tuple[str, str], ...]: + namespace: Final = _member_name(tool) + members: Final = _namespace_members(tool) + if namespace is None or not members: + return () + targets: Final = chain.from_iterable(map(self._unqualified_member_targets, members)) + return tuple((qualified_member_name(namespace, name), name) for name, _ in targets if name is not None) - def _get_named_tool_choice(self, data: dict) -> str | None: - tool_choice: Final = data.get("tool_choice") + def _get_request_tool_targets(self, tool: object) -> tuple[_ToolTarget, ...]: + group: Final = _member_group(tool, self._namespace_member_targets) + member_targets: Final = ( + tuple(chain.from_iterable(map(group.targets_of, group.members))) if group is not None else () + ) + keyed: Final = _keyed_tool_fields(tool) + part_targets: Final = tuple(_part_target(keyed, key) for key in _part_keys(keyed)) if keyed is not None else () + return member_targets + part_targets + self._own_tool_targets(tool) + + def _own_tool_targets(self, tool: object) -> tuple[_ToolTarget, ...]: + if _keyed_tool_fields(tool) is not None: + return () + named: Final = _namespace_function_targets(tool) if _namespace_members(tool) else self._named_tool_targets(tool) + return named + _gemini_hosted_key_targets(tool) + + def _named_tool_targets(self, tool: object) -> tuple[_ToolTarget, ...]: + raw_type: Final = self._get_mapping_value(tool, "type") + tool_type: Final = raw_type if isinstance(raw_type, str) else None + anthropic_custom: Final = ( + tool_type == "custom" + and self._get_mapping_value(tool, "input_schema") is not None + and _object_mapping(self._get_mapping_value(tool, "custom")) is None + ) + normalized_type: Final = "function" if tool_type is None or anthropic_custom else tool_type + names: Final = _request_tool_names(tool) + named_targets: Final = tuple((name, normalized_type) for name in names) + if names and all(_name_identifies_tool(name, normalized_type) for name in names): + return named_targets + if names: + return (*named_targets, (None, normalized_type)) + return ((None, tool_type),) if tool_type is not None else () + + def _get_named_tool_choice(self, tool_choice: object) -> str | None: if not tool_choice or tool_choice in ("auto", "none", "required"): return None if isinstance(tool_choice, str): return tool_choice - if self._get_mapping_value(tool_choice, "type") != "function": + choice: Final = _object_mapping(tool_choice) + if choice is None: return None - return self._get_mapping_value(self._get_mapping_value(tool_choice, "function"), "name") + choice_type: Final = choice.get("type") + server_field: Final = _HOSTED_SERVER_NAME_FIELDS.get(str(choice_type)) + if server_field is not None: + return _non_empty_string(choice.get(server_field)) + if choice_type not in ("tool", "function", "custom"): + return None + nested: Final = _object_mapping(choice.get(str(choice_type))) + nested_name: Final = _non_empty_string(nested.get("name")) if nested is not None else None + return nested_name or _non_empty_string(choice.get("name")) - def _get_named_function_call(self, data: dict) -> str | None: + def _get_named_function_call(self, data: Mapping[str, object]) -> str | None: function_call: Final = data.get("function_call") if not function_call or function_call in ("auto", "none"): return None if isinstance(function_call, str): return function_call - return self._get_mapping_value(function_call, "name") + return _non_empty_string(self._get_mapping_value(function_call, "name")) - def _collect_request_tools(self, data: dict) -> list[tuple[str, str | None]]: - request_tools: Final[list[tuple[str, str | None]]] = [] + def _listed_layer_targets(self, layer: Mapping[str, object]) -> tuple[_ToolTarget, ...]: + return ( + *chain.from_iterable(map(self._get_request_tool_targets, _request_list(layer, "tools") or ())), + *map(_mcp_server_target, _request_list(layer, "mcp_servers") or ()), + *chain.from_iterable(map(self._legacy_function_targets, _request_list(layer, "functions") or ())), + ) - for tool in data.get("tools") or []: - tool_name, tool_type = self._get_request_tool_name(tool) - if tool_name is not None: - request_tools.append((tool_name, tool_type)) + def _forced_layer_names(self, layer: Mapping[str, object]) -> tuple[str | None, ...]: + return (self._get_named_tool_choice(layer.get("tool_choice")), self._get_named_function_call(layer)) - for function in data.get("functions") or []: - function_name = self._get_legacy_function_name(function) - if function_name is not None: - request_tools.append((function_name, "function")) + def _collect_request_tools(self, data: Mapping[str, object]) -> tuple[_ToolTarget, ...]: + request: Final = _REQUEST_DATA.validate_python(data) + layers: Final = _request_layers(request) + listed: Final = tuple(chain.from_iterable(map(self._listed_layer_targets, layers))) + listed_names: Final = frozenset(name for name, _ in listed) + forced: Final = chain.from_iterable(map(self._forced_layer_names, layers)) + replayed: Final = map(_function_tool_name, replayed_tool_search_tools(request)) + input_tools: Final = _tools_in_items(tuple(chain.from_iterable(map(_input_items, layers)))) + return ( + *listed, + *((name, "function") for name in forced if name is not None and name not in listed_names), + *((name, "function") for name in replayed if name is not None), + *chain.from_iterable(map(self._get_request_tool_targets, input_tools)), + ) - for forced_tool_name in ( - self._get_named_tool_choice(data), - self._get_named_function_call(data), - ): - if forced_tool_name is not None: - request_tools.append((forced_tool_name, "function")) + def _legacy_function_targets(self, function: object) -> tuple[_ToolTarget, ...]: + targets: Final = self._get_request_tool_targets(function) + as_functions: Final = tuple((name, "function") for name, _ in targets if name is not None) + return tuple(dict.fromkeys(targets + as_functions)) - replayed: Final = map(_function_tool_name, replayed_tool_search_tools(_REQUEST_DATA.validate_python(data))) - request_tools.extend((name, "function") for name in replayed if name is not None) + def _kept_layer( + self, + layer: Mapping[str, object], + denied_targets: frozenset[_ToolTarget], + denied_names: frozenset[str], + ) -> _KeptLayer: + is_denied: Final = partial(_is_denied_target, denied_targets=denied_targets, denied_names=denied_names) + tools: Final = _request_list(layer, "tools") + functions: Final = _request_list(layer, "functions") + mcp_servers: Final = _request_list(layer, "mcp_servers") + return _KeptLayer( + fields=layer, + tools=None if tools is None else self._kept_tool_list(tools, denied_targets, denied_names), + functions=None if functions is None else self._kept_tool_list(functions, denied_targets, denied_names), + mcp_servers=( + None + if mcp_servers is None + else tuple(server for server in mcp_servers if not is_denied(_mcp_server_target(server))) + ), + input=self._kept_input_list(_input_items(layer), denied_targets, denied_names), + ) - return request_tools + def _rewritten_layer_fields( + self, + kept: _KeptLayer, + rewrite_choice: Callable[[object], object], + removed_names: frozenset[str], + functions_remain: bool, + ) -> dict[str, object]: + layer: Final = kept.fields + choice: Final = layer.get("tool_choice") + relaxes_function_call: Final = self._get_named_function_call(layer) in removed_names + return { + **kept.kept_lists(), + **({"tool_choice": rewrite_choice(choice)} if choice is not None else {}), + **({"function_call": "none"} if relaxes_function_call else {}), + **{ + key: _rewritten_tool_config(layer[key], removed_names, functions_remain) + for key in _TOOL_CONFIG_KEYS + if key in layer + }, + } + + def _tool_names_in(self, tools: Sequence[object]) -> frozenset[str]: + targets: Final = chain.from_iterable(self._get_request_tool_targets(tool) for tool in tools) + return frozenset(name for name, _ in targets if name is not None) + + def _referable_names_in(self, tools: Sequence[object]) -> frozenset[str]: + member_names: Final = chain.from_iterable(map(self._bare_member_names, tools)) + namespace_names: Final = (_member_name(tool) for tool in tools if _namespace_members(tool)) + return ( + self._tool_names_in(tools) + | frozenset(bare for _, bare in member_names) + | frozenset(name for name in namespace_names if name is not None) + ) + + def _kept_tools( + self, + tool: object, + denied_targets: frozenset[_ToolTarget], + denied_names: frozenset[str], + ) -> tuple[object, ...]: + is_denied: Final = partial(_is_denied_target, denied_targets=denied_targets, denied_names=denied_names) + group: Final = _member_group(tool, self._namespace_member_targets) + keyed: Final = _keyed_tool_fields(tool) + fields: Final = _object_mapping(tool) + if fields is None or (group is None and keyed is None): + return () if any(map(is_denied, self._get_request_tool_targets(tool))) else (tool,) + own_targets: Final = self._own_tool_targets(tool) + if any(map(is_denied, own_targets)): + return () + kept_members: Final = _kept_members(group, is_denied) + denied_keys: Final = ( + frozenset(key for key in _part_keys(keyed) if is_denied(_part_target(keyed, key))) + if keyed is not None + else frozenset[str]() + ) + if not denied_keys and len(kept_members) == len(group.members if group is not None else ()): + return (tool,) + emptied_group: Final = frozenset({group.key}) if group is not None and not kept_members else frozenset[str]() + dropped_keys: Final = denied_keys | emptied_group + kept_fields: Final = { + key: list(kept_members) if group is not None and key == group.key else value + for key, value in fields.items() + if key not in dropped_keys + } + own_name_left: Final = any(name is not None for name, _ in own_targets) + parts_left: Final = bool(kept_members) or own_name_left or (keyed is not None and bool(_part_keys(kept_fields))) + return (kept_fields,) if parts_left else () + + def _kept_tool_list( + self, + tools: Sequence[object], + denied_targets: frozenset[_ToolTarget], + denied_names: frozenset[str], + ) -> tuple[object, ...]: + return tuple(chain.from_iterable(self._kept_tools(tool, denied_targets, denied_names) for tool in tools)) + + def _kept_input_list( + self, + items: Sequence[object], + denied_targets: frozenset[_ToolTarget], + denied_names: frozenset[str], + ) -> tuple[object, ...] | None: + if not _carries_tools(items): + return None + return tuple(chain.from_iterable(self._kept_input_items(item, denied_targets, denied_names) for item in items)) + + def _kept_input_items( + self, + item: object, + denied_targets: frozenset[_ToolTarget], + denied_names: frozenset[str], + ) -> tuple[object, ...]: + fields: Final = _object_mapping(item) + tools: Final = _input_item_tools(item) + if fields is None or tools is None: + return (item,) + kept: Final = self._kept_tool_list(tools, denied_targets, denied_names) + emptied: Final = bool(tools) and not kept and fields.get("type") == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE + return () if emptied else ({**fields, "tools": list(kept)},) + + def _tool_types_in(self, tools: Sequence[object]) -> frozenset[str]: + targets: Final = chain.from_iterable(self._get_request_tool_targets(tool) for tool in tools) + return frozenset(tool_type for _, tool_type in targets if tool_type is not None) + + def _rewritten_tool_choice( + self, tool_choice: object, removed_names: frozenset[str], removed_types: frozenset[str], tools_remain: bool + ) -> object: + named: Final = self._get_named_tool_choice(tool_choice) + if named is not None and named in removed_names: + return _none_tool_choice(tool_choice) + choice: Final = _object_mapping(tool_choice) + hosted_type: Final = _non_empty_string(choice.get("type")) if choice is not None else None + if hosted_type not in _NAMED_CHOICE_TYPES and hosted_type in removed_types: + return _none_tool_choice(tool_choice) + allowed: Final = _allowed_tool_entries(choice) if choice is not None else None + if choice is not None and allowed is not None: + kept: Final = [entry for entry in allowed if not _is_removed_entry(entry, removed_names, removed_types)] + return _with_allowed_tool_entries(choice, kept) if kept and tools_remain else _none_tool_choice(choice) + if not tools_remain and (named is not None or _requires_a_tool(tool_choice)): + return _none_tool_choice(tool_choice) + return tool_choice def _modify_request_with_permission_errors( self, data: dict, - denied_tool_names: list[str], + denied_targets: frozenset[_ToolTarget], ): """ - Modify the request to replace denied tool_calls blocks with error results - - Args: - data: The model request to modify - denied_tools: List of (tool_use, error) tuples for denied tools + Remove denied tools from the request, and relax any tool_choice that would still require one """ - if not denied_tool_names: + if not denied_targets: return data - verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tool_names)) + verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_targets)) - # Create a mapping of tool_use_id to error result - error_tool_names: Final = set() - for tool_use in denied_tool_names: - error_tool_names.add(tool_use) - - tools: Final[list[ChatCompletionToolParam] | None] = data.get("tools") - if tools is not None: - new_tools: Final = [] - for tool in tools: - tool_name, tool_type = self._get_request_tool_name(tool) - if tool_type == "function" and tool_name in error_tool_names: - continue - new_tools.append(tool) - data["tools"] = new_tools - - functions: Final = data.get("functions") - if functions is not None: - data["functions"] = [ - function for function in functions if self._get_legacy_function_name(function) not in error_tool_names - ] - - named_tool_choice: Final = self._get_named_tool_choice(data) - if named_tool_choice in error_tool_names: - data["tool_choice"] = "none" - - named_function_call: Final = self._get_named_function_call(data) - if named_function_call in error_tool_names: - data["function_call"] = "none" - - messages: Final = without_tool_search_items_loading( - _REQUEST_DATA.validate_python(data), frozenset(denied_tool_names) + denied_names: Final = frozenset(name for name, _ in denied_targets if name is not None) + request: Final = _REQUEST_DATA.validate_python(data) + kept_layers: Final = tuple( + self._kept_layer(layer, denied_targets, denied_names) for layer in _request_layers(request) ) + + sent_tools: Final = _layer_sent_tools(_sent_fields(request)) + kept_sent: Final = reduce(_KeptLayer.overlaid_by, kept_layers) + kept_sent_tools: Final = kept_sent.sent_tools() + removed_names: Final = denied_names | ( + self._referable_names_in(sent_tools) - self._referable_names_in(kept_sent_tools) + ) + kept_types: Final = self._tool_types_in(kept_sent_tools) + removed_types: Final = self._tool_types_in(sent_tools) - kept_types + + tools_remain: Final = bool(kept_sent_tools) or bool(kept_sent.mcp_servers) + rewrite_choice: Final = partial( + self._rewritten_tool_choice, + removed_names=removed_names, + removed_types=removed_types, + tools_remain=tools_remain, + ) + rewritten: Final = tuple( + self._rewritten_layer_fields(kept, rewrite_choice, removed_names, "function" in kept_types) + for kept in kept_layers + ) + data.update(rewritten[0]) + if len(rewritten) > 1 and rewritten[1]: + data["extra_body"] = {**kept_layers[1].fields, **rewritten[1]} # rebind-ok: rewrite mode edits in place + + messages: Final = without_tool_search_items_loading(_REQUEST_DATA.validate_python(data), denied_names) if messages is not None: - data["messages"] = messages # rebind-ok: rewrite mode edits the request in place + data["messages"] = [ # rebind-ok: rewrite mode edits the request in place + _without_references_in_message(message, removed_names) for message in messages + ] return data @@ -843,25 +1543,29 @@ class ToolPermissionGuardrail(CustomGuardrail): ) return data - # Check permissions for each tool - denied_tool_names: Final = [] - for tool_name, tool_type in new_tools: - is_allowed, _, message = self._check_tool_permission(tool_name, tool_type) + member_names: Final = tuple( + chain.from_iterable(map(self._bare_member_names, _sent_tools(_REQUEST_DATA.validate_python(data)))) + ) + bare_names: Final = _grouped_values(member_names) + decisions: Final = tuple((target, self._check_target_permission(target, bare_names)) for target in new_tools) + denied: Final = tuple( + (target, message) + for target, (is_allowed, _, message) in decisions + if not is_allowed and message is not None + ) + for _, message in denied: + verbose_proxy_logger.info("Tool Permission Guardrail: %s", message) + if denied and self.on_disallowed_action == "block": + raise HTTPException( + status_code=400, + detail={ + "error": "Violated guardrail policy", + "detection_message": denied[0][1], + }, + ) - if not is_allowed and message is not None: - verbose_proxy_logger.info("Tool Permission Guardrail: %s", message) - if self.on_disallowed_action == "block": - raise HTTPException( - status_code=400, - detail={ - "error": "Violated guardrail policy", - "detection_message": message, - }, - ) - denied_tool_names.append(tool_name) - - if denied_tool_names: - data = self._modify_request_with_permission_errors(data, denied_tool_names) + if denied: + self._modify_request_with_permission_errors(data, frozenset(target for target, _ in denied)) verbose_proxy_logger.debug("Tool Permission Guardrail Pre-Call Hook: All tools allowed") @@ -908,7 +1612,7 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) - denied_tools: Final = self._evaluate_tool_calls(tool_calls) + denied_tools: Final = self._evaluate_tool_calls(tool_calls, self._returned_call_names(data)) if not denied_tools: verbose_proxy_logger.debug("Tool Permission Guardrail Post-Call Hook: All tools allowed") @@ -932,7 +1636,7 @@ class ToolPermissionGuardrail(CustomGuardrail): Args: user_api_key_dict: User API key information (unused but required by interface) response: The model response to check - request_data: The model request (unused but required by interface) + request_data: The model request """ # Import here to avoid circular imports @@ -949,7 +1653,9 @@ class ToolPermissionGuardrail(CustomGuardrail): stream_chunk_builder(chunks=all_chunks) if not is_raw_sse_stream(all_chunks) else None ) if isinstance(assembled_model_response, ModelResponse): - denied_tools = self._check_assembled_stream(assembled_model_response) + denied_tools = self._check_assembled_stream( + assembled_model_response, self._returned_call_names(request_data) + ) if denied_tools: self._modify_response_with_permission_errors(assembled_model_response, denied_tools) @@ -975,18 +1681,23 @@ class ToolPermissionGuardrail(CustomGuardrail): yield chunk return - for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks): + for sse_chunk in self._anthropic_stream_to_deliver( + anthropic_response, all_chunks, self._returned_call_names(request_data) + ): yield sse_chunk def _anthropic_stream_to_deliver( - self, anthropic_response: ModelResponse, source_chunks: Sequence[object] + self, + anthropic_response: ModelResponse, + source_chunks: Sequence[object], + judged_names: _JudgedCalls, ) -> Sequence[object]: """The frames the client gets: on a failed upstream only what the check read, never the upstream's own. Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly does not, so forwarding those frames would pass a tool call nobody checked. """ - denials: Final = self._check_assembled_stream(anthropic_response) + denials: Final = self._check_assembled_stream(anthropic_response, judged_names) if denials: self._modify_response_with_permission_errors(anthropic_response, denials) elif not anthropic_sse_error_events(source_chunks): @@ -994,7 +1705,7 @@ class ToolPermissionGuardrail(CustomGuardrail): return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True) def _check_assembled_stream( - self, assembled: ModelResponse + self, assembled: ModelResponse, judged_names: _JudgedCalls ) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]: verbose_proxy_logger.debug("Tool Permission Guardrail: Checking response") tool_calls: Final = self._extract_tool_calls_from_response(assembled) @@ -1002,7 +1713,7 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Tool Permission Guardrail: No tool uses found") return () verbose_proxy_logger.debug("Tool Permission Guardrail: Found %s tool calls", len(tool_calls)) - denied_tools: Final = self._evaluate_tool_calls(tool_calls) + denied_tools: Final = self._evaluate_tool_calls(tool_calls, judged_names) if not denied_tools: verbose_proxy_logger.debug("Tool Permission Guardrail Post-Call Hook: All tools allowed") return denied_tools diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 1ddbc139564..55e2e129e36 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from functools import partial from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import billed_guard from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES from litellm.proxy.db.spend_log_batching import spend_log_write_batches from litellm.proxy.utils import PrismaClient +from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ( DailyGuardrailMetricsRepository, DailyGuardrailUsageUnitsRepository, @@ -40,7 +41,7 @@ _RowKey = TypeVar("_RowKey") _RowValue = TypeVar("_RowValue") -class _UsageUnitKey(NamedTuple): +class UsageUnitKey(NamedTuple): guardrail_id: str date: str team_id: str @@ -48,7 +49,7 @@ class _UsageUnitKey(NamedTuple): usage_unit: str -class _UsageUnitIncrement(NamedTuple): +class UsageUnitIncrement(NamedTuple): units: int cost: float """USD for the priced share of units.""" @@ -56,14 +57,14 @@ class _UsageUnitIncrement(NamedTuple): """Units recorded with no known price, the share cost leaves out.""" -def _usage_unit_key(entry: tuple[_UsageUnitKey, _UsageUnitIncrement]) -> _UsageUnitKey: +def _usage_unit_key(entry: tuple[UsageUnitKey, UsageUnitIncrement]) -> UsageUnitKey: return entry[0] -def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement: +def _usage_unit_increment(units: int, cost: float | None) -> UsageUnitIncrement: if cost is None: - return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) - return _UsageUnitIncrement(units=units, cost=cost, untracked_units=0) + return UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) + return UsageUnitIncrement(units=units, cost=cost, untracked_units=0) class _MetricsKey(NamedTuple): @@ -89,22 +90,22 @@ class PendingRollups: def __init__(self) -> None: self.lock: Final = asyncio.Lock() self.metrics: Mapping[_MetricsKey, Mapping[str, int]] = MappingProxyType({}) - self.units: Mapping[_UsageUnitKey, _UsageUnitIncrement] = MappingProxyType({}) + self.units: Mapping[UsageUnitKey, UsageUnitIncrement] = MappingProxyType({}) _PENDING_ROLLUPS: Final = PendingRollups() _NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({}) -_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0, untracked_units=0) +_NO_INCREMENT: Final = UsageUnitIncrement(units=0, cost=0.0, untracked_units=0) def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]: return (*base, *(key for key in extra if key not in base)) -def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement: +def _summed_increments(increments: Iterable[UsageUnitIncrement]) -> UsageUnitIncrement: materialized: Final = tuple(increments) - return _UsageUnitIncrement( + return UsageUnitIncrement( units=sum(i.units for i in materialized), cost=sum(i.cost for i in materialized), untracked_units=sum(i.untracked_units for i in materialized), @@ -112,8 +113,8 @@ def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitI def _merged_unit_rows( - base: Mapping[_UsageUnitKey, _UsageUnitIncrement], extra: Mapping[_UsageUnitKey, _UsageUnitIncrement] -) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: + base: Mapping[UsageUnitKey, UsageUnitIncrement], extra: Mapping[UsageUnitKey, UsageUnitIncrement] +) -> Mapping[UsageUnitKey, UsageUnitIncrement]: return MappingProxyType( { key: _summed_increments((base.get(key, _NO_INCREMENT), extra.get(key, _NO_INCREMENT))) @@ -250,7 +251,7 @@ def _parse_payload_start_time(payload: Mapping[str, object]) -> datetime | None: def _iter_usage_unit_increments( logs_to_process: Sequence[Mapping[str, object]], -) -> Iterator[tuple[_UsageUnitKey, _UsageUnitIncrement]]: +) -> Iterator[tuple[UsageUnitKey, UsageUnitIncrement]]: for payload in logs_to_process: start_time = _parse_payload_start_time(payload) if not payload.get("request_id") or start_time is None: @@ -266,14 +267,14 @@ def _iter_usage_unit_increments( cost_by_unit = billed_guardrail_cost_by_unit(entry) for unit_name, units in usage.items(): if isinstance(units, int) and not isinstance(units, bool) and units > 0: - key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) + key = UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name)) cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None yield key, _usage_unit_increment(units=units, cost=cost) def _sum_usage_unit_increments( logs_to_process: Sequence[Mapping[str, object]], -) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: +) -> Mapping[UsageUnitKey, UsageUnitIncrement]: ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=_usage_unit_key) return MappingProxyType( { @@ -283,8 +284,16 @@ def _sum_usage_unit_increments( ) -async def _upsert_usage_unit_row( - prisma_client: PrismaClient, key: _UsageUnitKey, increment: _UsageUnitIncrement +class GuardrailUsageDatabase(Protocol): + @property + def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... + + +async def upsert_usage_unit_row( + prisma_client: PrismaClient, + key: UsageUnitKey, + increment: UsageUnitIncrement, + outer_transaction: GuardrailUsageDatabase | None = None, ) -> None: row: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsCreateInput] = { "guardrail_id": key.guardrail_id, @@ -314,7 +323,12 @@ async def _upsert_usage_unit_row( "untracked_units": {"increment": increment.untracked_units}, }, } - await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data) + table: Final = ( + outer_transaction.litellm_dailyguardrailusageunits + if outer_transaction is not None + else DailyGuardrailUsageUnitsRepository(prisma_client).table + ) + await table.upsert(where=where, data=data) async def _upsert_metrics_row(prisma_client: PrismaClient, key: _MetricsKey, agg: Mapping[str, int]) -> None: @@ -363,7 +377,7 @@ async def process_spend_logs_guardrail_usage( ) index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {} - for payload in logs_to_process: + for payload in (row for row in logs_to_process if row.get("accounting_log_disabled") is not True): request_id = payload.get("request_id") start_time = _parse_payload_start_time(payload) if not isinstance(request_id, str) or not request_id or start_time is None: @@ -416,7 +430,10 @@ async def process_spend_logs_guardrail_usage( {key: agg for key, agg in daily_guardrail.items() if int(agg["requests_evaluated"]) > 0} ) metrics_rows: Final = _merged_metric_rows(pending_metrics, evaluated_metrics) - unit_rows: Final = _merged_unit_rows(pending_units, _sum_usage_unit_increments(logs_to_process)) + unit_rows: Final = _merged_unit_rows( + pending_units, + _sum_usage_unit_increments(tuple(row for row in logs_to_process if "accounting_payload" not in row)), + ) if not metrics_rows and not index_rows and not unit_rows: return @@ -435,7 +452,7 @@ async def process_spend_logs_guardrail_usage( metrics_rows, partial(_upsert_metrics_row, prisma_client), "daily metrics", sleep ) failed_units: Final = await _upsert_rows_with_retry( - unit_rows, partial(_upsert_usage_unit_row, prisma_client), "usage unit", sleep + unit_rows, partial(upsert_usage_unit_row, prisma_client), "usage unit", sleep ) if failed_metrics or failed_units: async with pending.lock: diff --git a/litellm/proxy/hooks/max_budget_per_session_limiter.py b/litellm/proxy/hooks/max_budget_per_session_limiter.py index 002aeb145a5..636345a7ef7 100644 --- a/litellm/proxy/hooks/max_budget_per_session_limiter.py +++ b/litellm/proxy/hooks/max_budget_per_session_limiter.py @@ -26,6 +26,14 @@ from litellm._logging import verbose_proxy_logger from litellm.caching.redis_cache import log_redis_failure from litellm.exceptions import RateLimitType from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.batch_utils import ( + ChargeOnceJob, + charged_poll_cost, + defers_batch_cost, + job_charged_once_marker_key, + job_to_charge_once, +) +from litellm.litellm_core_utils.core_helpers import finite_cost from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit @@ -44,6 +52,7 @@ _CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapt _OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter( str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True) ) +_CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object]) # Redis Lua script for atomic float increment with TTL. @@ -63,6 +72,30 @@ end return new_val """ +MAX_BUDGET_SESSION_CHARGE_JOB_ONCE_SCRIPT: Final = """ +local spend_key = KEYS[1] +local marker_key = KEYS[2] +local ttl = tonumber(ARGV[3]) + +local polls = redis.call('INCR', marker_key) +redis.call('EXPIRE', marker_key, ttl) +local amount = ARGV[2] +if polls == 1 then + amount = ARGV[1] +end +if tonumber(amount) <= 0 then + return '0' +end + +local existed = redis.call('EXISTS', spend_key) +redis.call('INCRBYFLOAT', spend_key, amount) +if existed == 0 then + redis.call('EXPIRE', spend_key, ttl) +end + +return amount +""" + # Default TTL for session budget counters (1 hour) DEFAULT_MAX_BUDGET_PER_SESSION_TTL: Final = 3600 @@ -87,12 +120,15 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): ) ) - if self.internal_usage_cache.dual_cache.redis_cache is not None: - self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( - MAX_BUDGET_SESSION_INCREMENT_SCRIPT - ) - else: - self.increment_script = None + redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache + self.increment_script = ( + redis_cache.async_register_script(MAX_BUDGET_SESSION_INCREMENT_SCRIPT) if redis_cache is not None else None + ) + self.charge_job_once_script = ( + redis_cache.async_register_script(MAX_BUDGET_SESSION_CHARGE_JOB_ONCE_SCRIPT) + if redis_cache is not None + else None + ) @with_service_target("session_budgets") async def async_pre_call_hook( @@ -140,7 +176,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return None @with_service_target("session_budgets") - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time) -> None: """ After a successful LLM call, increment the session spend by the response cost. """ @@ -168,15 +204,23 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): if max_budget is None: return - response_cost: Final = kwargs.get("response_cost") or 0.0 + callback_fields: Final = _CALLBACK_FIELDS.validate_python(kwargs) + response_cost: Final = finite_cost(charged_poll_cost(callback_fields)) or 0.0 if response_cost <= 0: return + if defers_batch_cost(callback_fields, response_obj): + return + cache_key: Final = self._make_cache_key(str(session_id)) - await self._increment_spend(cache_key, float(response_cost)) + job: Final = job_to_charge_once(callback_fields, response_obj, response_cost) + if job is None: + await self._increment_spend(cache_key, response_cost) + else: + await self._charge_job_once(cache_key, job, response_cost) verbose_proxy_logger.debug( - "MaxBudgetPerSessionHandler: incremented session %s spend by %.6f", + "MaxBudgetPerSessionHandler: charged session %s for a call that cost %.6f", session_id, response_cost, ) @@ -221,6 +265,31 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): def _make_cache_key(self, session_id: str) -> str: return f"{{session_budget:{session_id}}}:spend" + async def _charge_job_once(self, cache_key: str, job: ChargeOnceJob, response_cost: float) -> None: + marker_key: Final = job_charged_once_marker_key(spend_key=cache_key, job=job) + if self.charge_job_once_script is not None: + try: + await self.charge_job_once_script( + keys=[cache_key, marker_key], + args=[str(response_cost), str(job.repeat_poll_cost), self.ttl], + ) + except Exception as e: + log_redis_failure( + verbose_proxy_logger, + logging.WARNING, + "MaxBudgetPerSessionHandler: Redis job charge failed, falling back to in-memory", + e, + ) + else: + return + + polls: Final = await self.internal_usage_cache.dual_cache.in_memory_cache.async_increment( + key=marker_key, value=1, ttl=self.ttl + ) + charge: Final = response_cost if polls == 1 else job.repeat_poll_cost + if charge > 0: + await self._in_memory_increment_spend(cache_key, charge) + @with_service_target("session_budgets") async def _get_current_spend(self, cache_key: str) -> float: """Read current accumulated spend for a session.""" diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 1c10826124e..889fb9fb373 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -19,7 +19,12 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import Span -from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once +from litellm.litellm_core_utils.batch_utils import ( + ChargeOnceJob, + defers_batch_cost, + job_charged_once_marker_key, + job_to_charge_once, +) from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions @@ -1208,18 +1213,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ) return - batch_id: Final = batch_id_to_charge_once( - call_type=kwargs.get("call_type"), - response_obj=response_obj, - response_cost=response_cost, - ) + if defers_batch_cost(kwargs, response_obj): + return + + job: Final = job_to_charge_once(kwargs=kwargs, response_obj=response_obj, response_cost=response_cost) for entity_type, entity_id, resolved in resolved_budgets: await self._charge_entity( entity_type=entity_type, entity_id=entity_id, resolved=resolved, response_cost=response_cost, - batch_id=batch_id, + job=job, ) if self.dual_cache.redis_cache is not None: @@ -1236,7 +1240,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): entity_id: str | None, resolved: ResolvedModelBudget, response_cost: float, - batch_id: str | None, + job: ChargeOnceJob | None, ) -> None: budget_duration: Final = resolved.budget_config.budget_duration if budget_duration is None: @@ -1247,11 +1251,19 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): budget_model=resolved.budget_model, budget_duration=budget_duration, ) - if batch_id is not None and not await self._claim_batch_charge( + if job is not None and not await self._claim_job_charge( spend_key=spend_key, - batch_id=batch_id, + job=job, ttl_seconds=duration_in_seconds(budget_duration), ): + if job.repeat_poll_cost > 0: + await self._charge_entity( + entity_type=entity_type, + entity_id=entity_id, + resolved=resolved, + response_cost=job.repeat_poll_cost, + job=None, + ) return start_time_key: Final = model_budget_start_time_cache_key( entity_type=entity_type, @@ -1362,8 +1374,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): spend=response_cost, ) - async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool: - marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id) + async def _claim_job_charge(self, spend_key: str, job: ChargeOnceJob, ttl_seconds: int) -> bool: + marker_key: Final = job_charged_once_marker_key(spend_key=spend_key, job=job) polls: Final = await self.dual_cache.async_increment_cache( key=marker_key, value=1, ttl=ttl_seconds, refresh_ttl=True ) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index c83551a815c..5c2f6bab650 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -55,7 +55,6 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) -from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, @@ -64,7 +63,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias -from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count +from litellm.litellm_core_utils.token_counter import ( + estimate_media_block_tokens, + offload_token_count, + split_embedding_input_tokens, +) from litellm.llms.azure.ptu_capacity import normalized_tokens from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( @@ -156,6 +159,12 @@ def _resolve_media_models(router: "Router | None", model: str, team_id: str | No return tuple(dict.fromkeys(router.registry().names(model, kind="litellm_base_models", team_id=team_id))) or (model,) +def _media_models( + resolver: Callable[[str, str | None], tuple[str, ...]] | None, model: str, team_id: str | None +) -> tuple[str, ...]: + return resolver(model, team_id) if resolver is not None else (model,) + + def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]: from litellm.proxy.proxy_server import llm_router @@ -709,7 +718,7 @@ class RequestRateLimiterStash: A single mutable instance is shared by every context forked from the request task (the SDK call, streaming generators, and the logging worker's captured context all see the same object), which is what makes the - ``reservation_released`` flag and ``parallel_slot`` clearing effective + released flags and ``parallel_slot`` clearing effective across sibling callbacks: the first release wins, later callbacks observe the cleared state. @@ -746,9 +755,32 @@ class RequestRateLimiterStash: batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = () reservation_released: bool = False + io_reservation_released: bool = False tpm_limited_tags: frozenset[str] = field(default_factory=frozenset) socket_agent_windows: tuple[RateLimitDescriptor, ...] = () + def clear_reserved_tokens(self) -> None: + self.reserved_tokens = 0 + self.ptu_reserved_tokens = 0 + self.itpm_reserved_tokens = 0 + self.otpm_reserved_tokens = 0 + + def mark_reservation_released(self) -> None: + self.reservation_released = True + self.io_reservation_released = True + + def discard_released_tpm_reservation(self) -> None: + if self.reservation_released: + self.reserved_tokens = 0 + self.ptu_reserved_tokens = 0 + self.reservation_released = False + + def discard_released_io_reservation(self) -> None: + if self.io_reservation_released: + self.itpm_reserved_tokens = 0 + self.otpm_reserved_tokens = 0 + self.io_reservation_released = False + @dataclass(frozen=True, slots=True) class CounterRefund: @@ -1355,20 +1387,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) media_source: Final = messages if messages is not None else original_input + is_embedding_call: Final = call_type in EMBEDDING_API_CALL_TYPES projected, media = ( split_media_for_tpm(media_source) if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES) else (original_input, ()) ) - input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input fallback_model: Final = request_model if isinstance(request_model, str) else None resolved_models: Final = ( - media_model_resolver(fallback_model or "", team_id) - if media and media_model_resolver is not None - else (fallback_model or "",) + _media_models(media_model_resolver, fallback_model or "", team_id) if media else (fallback_model or "",) ) - media_tokens: Final = sum( - self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media + embedding_text, embedding_media_tokens = ( + split_embedding_input_tokens( + original_input, + lambda: _media_models(media_model_resolver, fallback_model or "", team_id), + fallback_model, + ) + if include_media and is_embedding_call + else (original_input, 0) + ) + input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else embedding_text + media_tokens: Final = embedding_media_tokens + sum( + estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media ) total_chars: Final = ( @@ -1413,24 +1453,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type) - @staticmethod - def _estimate_media_block_tokens( - block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None - ) -> int: - if block.get("type") == "input_audio": - audio: Final = _callback_kwargs_mapping(block.get("input_audio")) - data: Final = audio.get("data") - return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model) - image: Final = _callback_kwargs_mapping(block.get("image_url")) - url: Final = image.get("url") - detail: Final = image.get("detail") - return estimate_image_tokens( - models, - url if isinstance(url, str) else "", - detail if isinstance(detail, str) else "auto", - fallback_model=fallback_model, - ) - def _is_redis_cluster(self) -> bool: """ Check if the dual cache is using Redis cluster. @@ -4075,12 +4097,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): None if is_responses_request else self._translate_google_genai_native_request(data, call_type) ) is_embedding_request: Final = self._is_embedding_request(data, call_type) - embedding_text: Final = data.get("input") if is_embedding_request else None + request: Final = as_str_mapping(data) + raw_embedding_input: Final = request.get("input") if is_embedding_request and request is not None else None pretokenized_input_tokens: Final = ( - self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None + self._count_pretokenized_embedding_input(raw_embedding_input) if is_embedding_request else None ) if pretokenized_input_tokens is not None: return pretokenized_input_tokens + embedding_text, embedding_media_tokens = ( + split_embedding_input_tokens( + raw_embedding_input, lambda: self._media_model_resolver(model or "", team_id), model + ) + if is_embedding_request + else (None, 0) + ) prompt: Final = data.get("prompt") fallback_text: Final = prompt if prompt is not None else data.get("input") @@ -4130,12 +4160,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",)) ), ) - return estimate + audio_token_estimate + return estimate + audio_token_estimate + embedding_media_tokens except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str): return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN) estimated_input_tokens, _ = self._estimate_input_and_output_tokens( - data=data, call_type=call_type, include_media=False + data=data, + call_type=call_type, + include_media=is_embedding_request, + team_id=team_id, + media_model_resolver=self._media_model_resolver, ) return estimated_input_tokens + audio_token_estimate @@ -4238,6 +4272,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) + stash.discard_released_io_reservation() if itpm_reserved > 0: itpm_scopes: Final = tuple( (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY @@ -4597,6 +4632,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # incremented so post-call reconciliation only applies # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. + stash.discard_released_tpm_reservation() stash.reserved_tokens = estimated_tokens stash.ptu_reserved_tokens = ptu_estimated_tokens stash.ptu_reserved_window_identities = frozenset( @@ -5140,7 +5176,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._response_usage(response_obj) ) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object"))) - if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released: + if ( + not response_usage[2] + and not combined_usage[2] + and aggregate_total <= 0 + and not stash.io_reservation_released + ): return () resolved_usage: Final = ( response_usage @@ -5153,25 +5194,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) billable_input, completion_tokens, _ = resolved_usage - if stash.reservation_released or ( + if stash.io_reservation_released or ( not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities ): return self._build_reservation_aware_tpm_ops( targets=tuple(stash.itpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes, actual_tokens=billable_input, - reserved_tokens=0 if stash.reservation_released else itpm_reserved, + reserved_tokens=0 if stash.io_reservation_released else itpm_reserved, ) + self._build_reservation_aware_tpm_ops( targets=tuple(stash.otpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes, actual_tokens=completion_tokens, - reserved_tokens=0 if stash.reservation_released else otpm_reserved, + reserved_tokens=0 if stash.io_reservation_released else otpm_reserved, ) itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( self._build_project_reservation_ops( targets=tuple(stash.itpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes, actual_tokens=billable_input, reserved_tokens=itpm_reserved, reservation_window_identities=stash.itpm_reserved_window_identities, @@ -5182,7 +5223,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( self._build_project_reservation_ops( targets=tuple(stash.otpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes, actual_tokens=completion_tokens, reserved_tokens=otpm_reserved, reservation_window_identities=stash.otpm_reserved_window_identities, @@ -5402,7 +5443,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): total_tokens = self._aggregate_only_total_tokens(usage=_usage) stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) - reserved_tokens: Final = stash.reserved_tokens if stash is not None else 0 + open_reservation: Final = stash if stash is not None and not stash.reservation_released else None + reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0 reserved_model: Final = stash.reserved_model if stash is not None else None reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the @@ -5464,7 +5506,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reconcile_model=reconcile_model, reserved_scopes=reserved_scopes, reserved_ceiling=stash.ptu_ceiling if stash is not None else None, - reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0, + reserved_tokens=open_reservation.ptu_reserved_tokens if open_reservation is not None else 0, reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(), total_tokens=0 if ptu_skips_socket_usage else total_tokens, ) @@ -5561,18 +5603,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else None, ) - await self._apply_usage_operations( - self._build_success_event_pipeline_operations( - kwargs=kwargs, - response_obj=response_obj, - rate_limit_type=rate_limit_type, - ), - litellm_parent_otel_span, + usage_operations: Final = self._build_success_event_pipeline_operations( + kwargs=kwargs, + response_obj=response_obj, + rate_limit_type=rate_limit_type, ) io_token_operations: Final = self._build_io_token_reservation_ops( kwargs=kwargs, response_obj=response_obj, ) + if stash is not None: + stash.clear_reserved_tokens() + await self._apply_usage_operations(usage_operations, litellm_parent_otel_span) if io_token_operations: if isinstance(io_token_operations, list): await self.async_increment_tokens_with_ttl_preservation( @@ -5733,10 +5775,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # already released it (proxy-level rejection that also bubbles up # here as an LLM-error callback). max_parallel_requests is its # own counter and is always decremented per call. - reserved_tokens, itpm_reserved, otpm_reserved = ( - (0, 0, 0) - if stash is None or stash.reservation_released - else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) + reserved_tokens: Final = 0 if stash is None or stash.reservation_released else stash.reserved_tokens + itpm_reserved, otpm_reserved = ( + (0, 0) + if stash is None or stash.io_reservation_released + else (stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) ) tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call) @@ -5806,6 +5849,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else () ) + if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0): + stash.mark_reservation_released() if pipeline_operations: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call( pipeline_operations, parent_otel_span=litellm_parent_otel_span @@ -5820,8 +5865,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations=project_operations, parent_otel_span=litellm_parent_otel_span, ) - if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0): - stash.reservation_released = True if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call): await self._apply_usage_operations( self._build_success_event_pipeline_operations( @@ -5927,7 +5970,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): Idempotent: the slot release clears the stashed acquisition (and slot removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM - refund is guarded by the stash's ``reservation_released`` flag — if + refund is guarded by the stash's two released flags — if both this hook and async_log_failure_event end up running in the same flow, only the first release/refund applies. A mid-stream failure relayed here with recovered partial usage settles the reservation at @@ -5953,11 +5996,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) stash.batch_tpd_refund_ops = () - if stash.reservation_released: - return - reserved_tokens: Final = stash.reserved_tokens - itpm_reserved: Final = stash.itpm_reserved_tokens - otpm_reserved: Final = stash.otpm_reserved_tokens + reserved_tokens: Final = 0 if stash.reservation_released else stash.reserved_tokens + itpm_reserved: Final = 0 if stash.io_reservation_released else stash.itpm_reserved_tokens + otpm_reserved: Final = 0 if stash.io_reservation_released else stash.otpm_reserved_tokens if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0: return tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data) @@ -6024,6 +6065,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): itpm_reserved, otpm_reserved, ) + stash.mark_reservation_released() if combined_ops: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=list(combined_ops), @@ -6040,7 +6082,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations=project_ops, parent_otel_span=user_api_key_dict.parent_otel_span, ) - stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e) return diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 980f0a5d410..ff67f2cb7c6 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -4,17 +4,19 @@ from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone +from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast from pydantic import TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger -from litellm.batches.batch_utils import batch_cost_is_final from litellm.caching.redis_batch import flush_post_call_redis_batches from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.batch_utils import defers_batch_cost, repeat_poll_cost from litellm.litellm_core_utils.core_helpers import ( budget_reservation_from_metadata, finite_cost, @@ -24,7 +26,6 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, - UsageCaptureObservation, UsageObservation, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost @@ -38,6 +39,7 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import ( + ACCOUNTING_LEDGER_KWARG, DBSpendUpdateWriter, debitable_model_access_groups, get_llm_router, @@ -64,10 +66,9 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export deployment_params_provider, get_request_model_access_groups, - sanitize_error_information_for_spend_logs, + sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export should_store_prompts_and_responses_in_spend_logs, ) -from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs from litellm.proxy.utils import ( REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY, @@ -77,9 +78,7 @@ from litellm.proxy.utils import ( from litellm.types.router import Deployment from litellm.types.utils import ( CallTypes, - LiteLLMBatch, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -226,6 +225,32 @@ def _guardrail_info_lifted_from_litellm_metadata( return {"standard_logging_guardrail_information": guardrail_info} +def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object: + metadata: Final = _str_keyed_fields(request_data.get("metadata")) + lifted: Final = _guardrail_info_lifted_from_litellm_metadata( + _str_keyed_fields(request_data.get("litellm_metadata")), + guardrail_info_already_set="standard_logging_guardrail_information" in metadata, + ) + return {**metadata, **lifted}.get("standard_logging_guardrail_information") + + +def failure_row_response_cost(request_data: Mapping[str, object]) -> float: + stashed_cost: Final = request_data.get("response_cost") + recovered_stream_cost: Final = ( + max(float(stashed_cost), 0.0) + if isinstance(request_data.get("combined_usage_object"), litellm.Usage) + and isinstance(stashed_cost, (int, float)) + else 0.0 + ) + finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) + finalized_amount: Final = finalized_failure.get("response_cost") + return ( + float(finalized_amount) + if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) + else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data)) + ) + + def _redaction_opt_in_lifted_from_litellm_metadata( litellm_metadata: ObjectMapping, *, caller_set_headers: bool ) -> ObjectMapping: @@ -350,26 +375,13 @@ def _attempt_spend_row_kwargs( "combined_usage_object": attempt.usage, "response_cost": attempt.response_cost, "_accounting_usage_observed": UsageObservation.ESTIMATED, - "_accounting_usage_snapshot": UsageCaptureObservation.from_usage( - attempt.usage, UsageObservation.ESTIMATED, attempt.response_cost - ), - "_accounting_prior_captures": (), } row.pop("billed_failed_attempts", None) - row.pop("_accounting_capture", None) call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id") if isinstance(call_id, str): row["litellm_call_id"] = f"{call_id}{suffix}" row["parent_request_id"] = call_id metadata["parent_request_id"] = call_id - captured_attempts: Final = request_data.get("_accounting_billed_attempts") - if isinstance(captured_attempts, tuple): - for captured in TypeAdapter(tuple[object, ...]).validate_python(captured_attempts): - if ( - isinstance(captured, CapturedUsage) - and captured.payload["litellm_call_id"] == row["litellm_call_id"] - ): - row["_accounting_capture"] = captured standard_logging_object: Final = request_data.get("standard_logging_object") if isinstance(standard_logging_object, dict): payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict @@ -405,7 +417,7 @@ class _ProxyDBLogger(CustomLogger): async def async_log_success_event( self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime ) -> None: - if self.spend_event_producer is None or not is_offloadable_success(response_obj): + if self.spend_event_producer is None or not is_offloadable_success(response_obj, kwargs.get("call_type")): await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time) return event: Final = build_spend_event( @@ -455,9 +467,6 @@ class _ProxyDBLogger(CustomLogger): "Failed to invalidate budget reservation counters after failure release failed" ) - capture_metadata: Final = await self._try_capture_failure_usage( - request_data, original_exception, user_api_key_dict - ) if completed_cost and tracking_disabled: return @@ -492,19 +501,10 @@ class _ProxyDBLogger(CustomLogger): # ``.get("traceback")`` / truthy checks, and the TypedDict marks # the field as optional, so omitting is type-safe. _error_information.pop("traceback", None) - # Strip echoed request input + apply DB-size cap before storing in - # the spend-log metadata column (LIT-2992). Result is never None - # here because the input above is constructed non-None. - _error_information = cast( - StandardLoggingPayloadErrorInformation, - sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), - ) _metadata["error_information"] = _error_information - _metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture - {**_metadata, **capture_metadata} - if capture_metadata is not None - else await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( + _metadata = ( # rebind-ok: enrich identity before building failure spend rows + await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( metadata=_metadata, original_exception=original_exception ) ) @@ -585,19 +585,7 @@ class _ProxyDBLogger(CustomLogger): # recovered cost onto request_data (the usage rides along in # ``combined_usage_object`` for the token columns), so attribute the # real partial spend to this failure row instead of zero. - recovered_stream_cost: Final = ( - max(float(request_data.get("response_cost") or 0.0), 0.0) - if isinstance(request_data.get("combined_usage_object"), litellm.Usage) - else 0.0 - ) - finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) - finalized_amount: Final = finalized_failure.get("response_cost") - recovered_response_cost: Final = ( - float(finalized_amount) - if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) - else recovered_stream_cost - + guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information")) - ) + recovered_response_cost: Final = failure_row_response_cost(request_data) failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it( request_data.get("standard_logging_object"), @@ -623,106 +611,6 @@ class _ProxyDBLogger(CustomLogger): ), ) - async def _failure_capture_metadata( - self, request_data: Mapping[str, object], exception: Exception, auth: UserAPIKeyAuth - ) -> Mapping[str, object] | None: - from litellm.proxy.proxy_server import usage_capture - - if usage_capture.meter is None: - return None - captured_identity: Final = ( - _batch_accounting_for(request_data.get("litellm_call_id")) is not None - and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP - ) - current_identity: Final = { - **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth), - "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth), - } - metadata: Final = ( - { - **dict.fromkeys(current_identity), - **{ - key: value - for key, value in _REQUEST_DATA.validate_python( - get_litellm_metadata_from_kwargs(dict(request_data)) - ).items() - if key in current_identity or key == "tags" - }, - } - if captured_identity - else current_identity - ) - return await self._enrich_failure_metadata_unless_db_stalled( - metadata=metadata, original_exception=exception, resolve_missing_key_identity=not captured_identity - ) - - async def _try_capture_failure_usage( - self, - request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer - original_exception: Exception, - auth: UserAPIKeyAuth, - ) -> Mapping[str, object] | None: - try: - metadata: Final = await self._failure_capture_metadata(request_data, original_exception, auth) - self._capture_failure_usage(request_data, original_exception, auth, metadata) - except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing or reservation release - verbose_proxy_logger.exception("Usage capture failed on the request failure path") - return None - return metadata - - def _capture_failure_usage( - self, - request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer - original_exception: Exception, - auth: UserAPIKeyAuth, - metadata: Mapping[str, object] | None = None, - ) -> None: - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.db.exception_handler import exception_chain - from litellm.proxy.proxy_server import usage_capture - from litellm.proxy.spend_tracking.usage_capture import UsageAcknowledgementError - - if any(isinstance(error, UsageAcknowledgementError) for error in exception_chain(original_exception)): - return - usage_meter: Final = usage_capture.meter - - if usage_meter is None: - return - logging_obj: Final = request_data.get("litellm_logging_obj") - details: Final[Mapping[str, object]] = ( - _str_keyed_fields(logging_obj.model_call_details) if isinstance(logging_obj, Logging) else {} - ) - params: Final = _str_keyed_fields(request_data.get("litellm_params") or details.get("litellm_params")) - failure_data: Final = { - **details, - **request_data, - "litellm_params": { - **params, - "metadata": { - **_str_keyed_fields(params.get("metadata")), - **( - metadata - if metadata is not None - else { - **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth), - "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth), - } - ), - "status": "failure", - }, - }, - } - start: Final = logging_obj.start_time if isinstance(logging_obj, Logging) else request_data.get("start_time") - if not isinstance(start, datetime): - return - end: Final = datetime.now(timezone.utc) - captured: Final = tuple( - capture_callback(usage_meter, row, original_exception, start, end) - for _, row in _failure_spend_rows(failure_data, 0.0, None, self._deployment_lookup) - ) - request_data["_accounting_capture"] = captured[0] # rebind-ok: share one receipt with the legacy writer - request_data["_accounting_billed_attempts"] = captured[1:] # rebind-ok: share attempt receipts - async def _write_failure_spend_rows( self, user_api_key_dict: UserAPIKeyAuth, @@ -771,6 +659,7 @@ class _ProxyDBLogger(CustomLogger): token: Final = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) end_user_id: Final = None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id + accounting_rows: Final = tuple((cost, dict(row)) for cost, row in spend_rows) charged: Final = [ await self._spend_writer().update_database( token=token, @@ -778,17 +667,17 @@ class _ProxyDBLogger(CustomLogger): user_id=user_api_key_dict.user_id, end_user_id=end_user_id, team_id=user_api_key_dict.team_id, - kwargs=dict(row_kwargs), + kwargs=row_kwargs, completion_response=original_exception, start_time=start_time, end_time=datetime.now(), org_id=user_api_key_dict.org_id, project_id=user_api_key_dict.project_id, ) - for row_cost, row_kwargs in spend_rows + for row_cost, row_kwargs in accounting_rows ] counted_rows: Final = tuple( - row for row, row_charged in zip(spend_rows, charged, strict=True) if row_charged and row[0] > 0 + row for row, row_charged in zip(accounting_rows, charged, strict=True) if row_charged and row[0] > 0 ) if counted_rows and not ProxyUpdateSpend.disable_spend_updates(): await self._increment_failure_spend_counters( @@ -867,6 +756,7 @@ class _ProxyDBLogger(CustomLogger): router=get_llm_router(), ), project_id=user_api_key_dict.project_id, + **_ledger_counter_options(row_kwargs.get(ACCOUNTING_LEDGER_KWARG)), ) except Exception: # noqa: BLE001 # one row's counter failure must not keep the other rows' counters from landing verbose_proxy_logger.exception("Failed to raise spend counters for a failure spend row") @@ -908,20 +798,11 @@ class _ProxyDBLogger(CustomLogger): resolve_missing_key_identity=str(kwargs.get("call_type")) not in _CAPTURED_IDENTITY_CALL_TYPES, ) _write_spend_metadata_to_kwargs(kwargs=kwargs, metadata=metadata) - kwargs.update(await captured_callback_kwargs(kwargs, completion_response, start_time, end_time)) budget_reservation: Final = _get_budget_reservation_from_metadata(metadata=metadata) if accounting is not None and (budget_reservation is not None or litellm.apply_search_free_tier): accounting.write_started = True - if ( - isinstance(completion_response, LiteLLMBatch) - and kwargs.get("call_type") == CallTypes.aretrieve_batch.value - and not batch_cost_is_final(completion_response) - ): - verbose_proxy_logger.debug( - "Cost tracking deferred for batch %s still in status %s", - completion_response.id, - completion_response.status, - ) + if defers_batch_cost(kwargs, completion_response): + verbose_proxy_logger.debug("Cost tracking deferred for a batch whose cost is not final yet") await _release_budget_reservation(budget_reservation=budget_reservation) return user_id: Final = cast(str | None, metadata.get("user_api_key_user_id", None)) @@ -983,25 +864,28 @@ class _ProxyDBLogger(CustomLogger): agent_id=metadata.get("billing_agent_id") or metadata.get("agent_id"), ): ## UPDATE DATABASE - charged: Final = await _update_database_and_spend_counters( - proxy_logging_obj=proxy_logging_obj, - increment_spend_counters=increment_spend_counters, - user_api_key=user_api_key, - user_id=user_id, - end_user_id=end_user_id, - team_id=team_id, - org_id=org_id, + charged_cost: Final = await _charge_request_cost( + partial( + _update_database_and_spend_counters, + proxy_logging_obj=proxy_logging_obj, + increment_spend_counters=increment_spend_counters, + user_api_key=user_api_key, + user_id=user_id, + end_user_id=end_user_id, + team_id=team_id, + org_id=org_id, + completion_response=completion_response, + start_time=start_time, + end_time=end_time, + request_tags=tags, + model_access_groups=model_access_groups, + project_id=project_id, + ), kwargs=kwargs, - completion_response=completion_response, - start_time=start_time, - end_time=end_time, response_cost=response_cost, budget_reservation=budget_reservation, - request_tags=tags, - model_access_groups=model_access_groups, - project_id=project_id, ) - if not charged: + if charged_cost is None: return # update cache (fire-and-forget for backward compat: @@ -1011,7 +895,7 @@ class _ProxyDBLogger(CustomLogger): token=user_api_key, user_id=user_id, end_user_id=end_user_id, - response_cost=response_cost, + response_cost=charged_cost, team_id=team_id, parent_otel_span=parent_otel_span, tags=tags, @@ -1022,7 +906,7 @@ class _ProxyDBLogger(CustomLogger): token=user_api_key, key_alias=key_alias, end_user_id=end_user_id, - response_cost=response_cost, + response_cost=charged_cost, max_budget=end_user_max_budget, ) elif budget_reservation is not None: @@ -1316,6 +1200,58 @@ def _get_request_tags_for_cost_tracking( return None +class _LedgerCounterOptions(TypedDict): + windows_from_ledger: ReadOnly[NotRequired[bool]] + + +def _ledger_counter_options(ledger: object) -> _LedgerCounterOptions: + return {"windows_from_ledger": True} if ledger is True else {} + + +class _ChargeSpend(Protocol): + async def __call__( + self, + *, + kwargs: dict[str, object], # mutable-ok: update_database takes the logging kwargs dict itself + response_cost: float, + budget_reservation: dict[str, object] | None, # mutable-ok: reconcile stamps the shared reservation dict + ) -> bool: ... + + +async def _charge_request_cost( + charge_spend: _ChargeSpend, + kwargs: dict[str, object], # mutable-ok: the first charge passes the caller's logging kwargs dict on unchanged + response_cost: float, + budget_reservation: dict[str, object] | None, # mutable-ok: reconcile stamps the shared reservation dict +) -> float | None: + """What the request charged, since a poll of a job an earlier poll charged still pays for its own guardrails.""" + if await charge_spend(kwargs=kwargs, response_cost=response_cost, budget_reservation=budget_reservation): + return response_cost + standard_logging_payload: Final = kwargs.get("standard_logging_object") + cost_after_job: Final = repeat_poll_cost(kwargs.get("call_type"), response_cost, standard_logging_payload) + if cost_after_job <= 0: + return None + charged: Final = await charge_spend( + kwargs={ + **kwargs, + "response_cost": cost_after_job, + **( + { + "standard_logging_object": { + **_OBJECT_MAPPING.validate_python(standard_logging_payload), + "response_cost": cost_after_job, + } + } + if isinstance(standard_logging_payload, Mapping) + else {} + ), + }, + response_cost=cost_after_job, + budget_reservation=None, + ) + return cost_after_job if charged else None + + class _IncrementSpendCounters(Protocol): """The ``increment_spend_counters`` coroutine :func:`_update_database_and_spend_counters` awaits.""" @@ -1332,6 +1268,7 @@ class _IncrementSpendCounters(Protocol): request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, project_id: str | None = None, + windows_from_ledger: bool = False, ) -> None: ... @@ -1469,6 +1406,7 @@ async def _update_database_and_spend_counters_in_batch( request_started_at=start_time, model_access_groups=model_access_groups, project_id=project_id, + **_ledger_counter_options(_BATCH_CALLBACK_FIELDS.validate_python(kwargs).get(ACCOUNTING_LEDGER_KWARG)), ) except Exception: if budget_reservation is not None: diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 54a2d4062d2..d2aac9cae38 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -47,7 +47,7 @@ from litellm.proxy.auth.password_policy import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast -from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for +from litellm.proxy.common_utils.openai_error_payload import openai_shaped_proxy_exception, proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import ( AUTH_OBJECTS_TARGET, object_permission_cache_key, @@ -1048,6 +1048,7 @@ async def _check_user_info_v2_access( 1. Proxy admins / proxy admin viewers can access any user 2. User can access their own info 3. Team admins can access info of users in their teams + 4. Org admins can access info of users in their organizations Raises on unexpected DB errors so they surface as 500s, not silent 404s. """ @@ -1090,9 +1091,33 @@ async def _check_user_info_v2_access( if team.team_id in (target_user.teams or []): return target_user + if user_api_key_dict.user_id is not None and await _target_in_caller_admin_org( + prisma_client=prisma_client, + caller_user_id=user_api_key_dict.user_id, + target_user_id=target_user_id, + ): + return await _fetch_target_user() + return None +async def _target_in_caller_admin_org( + prisma_client: "PrismaClient", + caller_user_id: str, + target_user_id: str, +) -> bool: + admin_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many( + where={"user_id": caller_user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value} + ) + if not admin_memberships: + return False + admin_org_ids: Final = {membership.organization_id for membership in admin_memberships} + target_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many( + where={"user_id": target_user_id} + ) + return any(membership.organization_id in admin_org_ids for membership in target_memberships) + + @router.get( "/v2/user/info", tags=["Internal User management"], @@ -1197,6 +1222,8 @@ async def user_info_v2( ), ) except Exception as e: + if isinstance(e, HTTPException) and e.status_code < 500: + raise openai_shaped_proxy_exception(e, str(e.detail), e.status_code) verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/prompt_caching_requests.py b/litellm/proxy/management_endpoints/prompt_caching_requests.py index 2f00d263d84..d52c0255b08 100644 --- a/litellm/proxy/management_endpoints/prompt_caching_requests.py +++ b/litellm/proxy/management_endpoints/prompt_caching_requests.py @@ -14,9 +14,7 @@ from litellm.proxy.spend_tracking.savings import ( marks_gateway_injection, prompt_caching_savings_for_request, ) -from litellm.proxy.spend_tracking.spend_tracking_utils import ( - _query_raw_rows, # pyright: ignore[reportPrivateUsage] # existing typed spend-query adapter; rows validated below -) +from litellm.proxy.utils import query_raw_within_statement_timeout from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.prompt_caching_requests import ( @@ -158,7 +156,7 @@ async def get_prompt_caching_requests( else cursor_start_time ) rows: Final = _REQUEST_ROWS.validate_python( - await _query_raw_rows( + await query_raw_within_statement_timeout( prisma_client, prompt_caching_requests_sql(filter), start.isoformat(), @@ -166,8 +164,8 @@ async def get_prompt_caching_requests( page_size + 1, cursor_time.isoformat() if cursor_time is not None else None, cursor_request_id, + what="Prompt caching requests", ) - or () ) def current_router() -> "Router | None": diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f505406fe0e..54b9b26123e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -224,7 +224,12 @@ from litellm.proxy.management_helpers.utils import ( add_new_member, management_endpoint_wrapper, ) -from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy +from litellm.proxy.utils import ( + PrismaClient, + ProxyLogging, + handle_exception_on_proxy, + query_raw_within_statement_timeout, +) from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.chunked_in import delete_many_in, find_many_in from litellm.repositories.daily_activity_buckets import api_key_from_bucket @@ -7473,6 +7478,9 @@ class _TeamUserSpendDbRow(TypedDict): failed_requests: ReadOnly[int] +_TEAM_USER_SPEND_ROWS: Final = TypeAdapter(tuple[_TeamUserSpendDbRow, ...]) + + @router.get( "/team/spend/by_user", response_model=TeamUserSpendResponse, @@ -7523,12 +7531,16 @@ async def get_team_spend_by_user( own_user_only: Final = scope.api_key_filter is not None user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else () async with db_span("team_user_spend", "LiteLLM_SpendLogs"): - rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw( - _team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only), - start_date, - end_date, - *scoped_team_ids, - *user_param, + rows: Final = _TEAM_USER_SPEND_ROWS.validate_python( + await query_raw_within_statement_timeout( + prisma_client, + _team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only), + start_date, + end_date, + *scoped_team_ids, + *user_param, + what="Spend per user", + ) ) results: Final = tuple( TeamUserSpendRow( diff --git a/litellm/proxy/management_helpers/user_list_sorting.py b/litellm/proxy/management_helpers/user_list_sorting.py index d3d2e323cb0..407a188ff1a 100644 --- a/litellm/proxy/management_helpers/user_list_sorting.py +++ b/litellm/proxy/management_helpers/user_list_sorting.py @@ -15,6 +15,10 @@ class _UserId(BaseModel): _USER_IDS: Final = TypeAdapter(tuple[_UserId, ...]) +def _csv_values(raw: str | None) -> tuple[str, ...]: + return tuple(value.strip() for value in (raw or "").split(",") if value.strip()) + + async def sorted_user_ids( prisma_client: PrismaClient, *, @@ -30,9 +34,9 @@ async def sorted_user_ids( organization_ids: str | None, search: str | None, ) -> Sequence[str]: - ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip()) - sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip()) - org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip()) + ids: Final = _csv_values(user_ids) + sso_ids: Final = _csv_values(sso_user_ids) + org_ids: Final = _csv_values(organization_ids) expression: Final = ( "CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END" if sort_by == "status" diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index eacba25257e..013336e1d5d 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -367,7 +367,7 @@ _UpstreamSessionKey: TypeAlias = tuple[ str, str, str, - tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool], + tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool, bool], ] @@ -6253,9 +6253,9 @@ class MCPServerManager: "server_name": server_name, "mcp_rate_limit_server_name": server.alias or server.server_name or server.name, "user_api_key_auth": checked_auth, - "user_api_key_user_id": (getattr(checked_auth, "user_id", None) if checked_auth else None), - "user_api_key_team_id": (getattr(checked_auth, "team_id", None) if checked_auth else None), - "user_api_key_end_user_id": (getattr(checked_auth, "end_user_id", None) if checked_auth else None), + "user_api_key_user_id": checked_auth.user_id if checked_auth else None, + "user_api_key_team_id": checked_auth.team_id if checked_auth else None, + "user_api_key_end_user_id": checked_auth.end_user_id if checked_auth else None, "user_api_key_hash": (getattr(checked_auth, "api_key_hash", None) if checked_auth else None), "incoming_bearer_token": caller_bearer.get_secret_value() if caller_bearer is not None else None, "headers": logging_safe_mcp_headers(raw_headers), diff --git a/litellm/proxy/mcp_server/upstream.py b/litellm/proxy/mcp_server/upstream.py index 96a1c2db9ba..3dd245947db 100644 --- a/litellm/proxy/mcp_server/upstream.py +++ b/litellm/proxy/mcp_server/upstream.py @@ -231,6 +231,7 @@ async def prepare_upstream_client( server_url="", transport_type=transport, protocol_version=protocol_version, + skip_tool_output_validation=server.skip_tool_output_validation, auth_type=server.auth_type, auth_value=auth_value, timeout=timeout, @@ -265,6 +266,7 @@ async def prepare_upstream_client( server_url=server.url or "", transport_type=transport, protocol_version=protocol_version, + skip_tool_output_validation=server.skip_tool_output_validation, upstream_transport=upstream_transport, auth_type=server.auth_type, timeout=timeout, @@ -308,6 +310,7 @@ async def prepare_upstream_client( server_url=server.url or "", transport_type=transport, protocol_version=protocol_version, + skip_tool_output_validation=server.skip_tool_output_validation, upstream_transport=upstream_transport, auth_type=server.auth_type, auth_value=auth_value, diff --git a/litellm/proxy/middleware/request_body_decompression_middleware.py b/litellm/proxy/middleware/request_body_decompression_middleware.py index 3e144d7bf4c..2c36a621387 100644 --- a/litellm/proxy/middleware/request_body_decompression_middleware.py +++ b/litellm/proxy/middleware/request_body_decompression_middleware.py @@ -20,7 +20,11 @@ from litellm.constants import ( ) from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.common_utils.anthropic_error_payload import proxy_error_type, route_error_content -from litellm.proxy.middleware.request_size_limit_middleware import MaxRequestSizeGetter, send_request_too_large +from litellm.proxy.middleware.request_size_limit_middleware import ( + MaxRequestSizeGetter, + effective_max_request_size_mb, + send_request_too_large, +) RFC_9659_ZSTD_WINDOW_LOG_MAX: Final = 23 GZIP_MAGIC: Final = b"\x1f\x8b" @@ -337,6 +341,7 @@ class RequestBodyDecompressionMiddleware: max_decompression_ratio: int = MAX_REQUEST_BODY_DECOMPRESSION_RATIO, max_decoded_bodies_in_flight_mb: float = MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB, codecs: Mapping[str, Codec] = AVAILABLE_CODECS, + route_max_request_size_mb: Mapping[tuple[str, str], float] | None = None, ) -> None: self.app = app self.get_max_request_size_mb: Final = get_max_request_size_mb @@ -344,6 +349,7 @@ class RequestBodyDecompressionMiddleware: self.max_decompression_ratio: Final = max_decompression_ratio self.budget: Final = _DecodedBodyBudget(int(max_decoded_bodies_in_flight_mb * 1024 * 1024)) self.codecs: Final = codecs + self.route_max_request_size_mb: Final = MappingProxyType(dict(route_max_request_size_mb or {})) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: coding: Final = _coding_for(scope) @@ -378,7 +384,10 @@ class RequestBodyDecompressionMiddleware: async def _decode_then_call( self, scope: Scope, receive: Receive, send: Send, codec: Codec, prefix: _Prefix ) -> None: - max_mb: Final = _effective_max_mb(self.max_decompressed_body_size_mb, self.get_max_request_size_mb()) + request_max_mb: Final = effective_max_request_size_mb( + scope, self.get_max_request_size_mb(), self.route_max_request_size_mb + ) + max_mb: Final = _effective_max_mb(self.max_decompressed_body_size_mb, request_max_mb) limits: Final = _Limits(max_mb=max_mb, max_ratio=self.max_decompression_ratio) match await _receive_body(scope, receive, prefix, limits.max_bytes): case _Body(data=data): diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py index 15bbf05bd01..f2883168d65 100644 --- a/litellm/proxy/middleware/request_size_limit_middleware.py +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -1,7 +1,10 @@ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final +from fastapi import HTTPException +from starlette._utils import get_route_path from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm.proxy.common_utils.anthropic_error_payload import route_error_content @@ -10,8 +13,22 @@ MaxRequestSizeGetter = Callable[[], int | float | None] RequestSizeLimitEnabledGetter = Callable[[], bool] -class RequestEntityTooLarge(Exception): - pass +class RequestEntityTooLarge(HTTPException): + def __init__(self, max_request_size_mb: float) -> None: + super().__init__(status_code=413, detail=f"Request size is too large. Max size is {max_request_size_mb} MB") + + +def effective_max_request_size_mb( + scope: Scope, + configured: float | None, + route_limits: Mapping[tuple[str, str], float], +) -> float | None: + global_limit: Final = configured if configured is not None and configured > 0 else None + method: Final = scope.get("method") + route_limit: Final = route_limits.get((method, get_route_path(scope))) if isinstance(method, str) else None + if route_limit is None: + return global_limit + return min(route_limit, global_limit) if global_limit is not None else route_limit class RequestSizeLimitMiddleware: @@ -28,19 +45,29 @@ class RequestSizeLimitMiddleware: app: ASGIApp, get_max_request_size_mb: MaxRequestSizeGetter, is_request_size_limit_enabled: RequestSizeLimitEnabledGetter, + route_max_request_size_mb: Mapping[tuple[str, str], float] | None = None, ) -> None: self.app = app self.get_max_request_size_mb = get_max_request_size_mb self.is_request_size_limit_enabled = is_request_size_limit_enabled + self.route_max_request_size_mb = MappingProxyType(dict(route_max_request_size_mb or {})) + + def _max_request_size_mb(self, scope: Scope) -> float | None: + configured: Final = self.get_max_request_size_mb() + return effective_max_request_size_mb( + scope, + configured if self.is_request_size_limit_enabled() else None, + self.route_max_request_size_mb, + ) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] != "http": await self.app(scope, receive, send) return - max_request_size_mb: Final = self.get_max_request_size_mb() + max_request_size_mb: Final = self._max_request_size_mb(scope) max_request_size_bytes: Final = _mb_to_bytes(max_request_size_mb) - if max_request_size_bytes is None or not self.is_request_size_limit_enabled(): + if max_request_size_bytes is None or max_request_size_mb is None: await self.app(scope, receive, send) return @@ -61,7 +88,7 @@ class RequestSizeLimitMiddleware: received_body_bytes += len(message.get("body", b"")) if received_body_bytes > max_request_size_bytes: - raise RequestEntityTooLarge + raise RequestEntityTooLarge(max_request_size_mb) return message async def tracking_send(message: Message) -> None: diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index c0d81abeba7..18ce18da936 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -21,8 +21,8 @@ from typing import ( from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict -from litellm.batches.batch_utils import batch_cost_is_final from litellm.constants import MAX_FILE_LIST_LIMIT +from litellm.litellm_core_utils.batch_utils import batch_cost_is_final from litellm.proxy._types import ProxyException from litellm.repositories.managed_file_repository import ManagedFileRepository from litellm.repositories.table_repositories import ( diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index be9f80694b8..627164abb93 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -31,7 +31,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm import get_llm_provider -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS, @@ -58,6 +58,7 @@ from litellm.llms.anthropic.common_utils import ( anthropic_pass_through_api_base, merge_anthropic_beta_headers, ) +from litellm.llms.azure.common_utils import entra_rejection_in from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, @@ -1417,13 +1418,12 @@ async def handle_bedrock_count_tokens( if deployment is not None: model_litellm_params: Final = deployment.get("litellm_params", {}) - # Get the resolved model ID from the configuration if "model" in model_litellm_params: resolved_model = model_litellm_params["model"] # Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery for key, value in model_litellm_params.items(): - if key != "user_api_key_dict": # Don't overwrite user_api_key_dict + if key != "user_api_key_dict": litellm_params[key] = value verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params) @@ -2382,6 +2382,16 @@ async def _relay_router_model( ) except httpx.HTTPStatusError as upstream_error: return await _relay_upstream_response(upstream_error.response) + except Exception as error: + rejection: Final = entra_rejection_in(error) + if rejection is None: + raise + raise ProxyException( + message=redact_internal_details_from_client_message(rejection.message), + type=ProxyErrorTypes.auth_error, + param=None, + code=rejection.status_code, + ) from error if not is_streaming_request: return await _relay_upstream_response(cast(httpx.Response, result)) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_websocket_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_websocket_passthrough_logging_handler.py index f3bad54ffb2..90eb76cc9d8 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_websocket_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_websocket_passthrough_logging_handler.py @@ -158,9 +158,7 @@ def _responses_turn(event: Mapping[str, object], session_model: str) -> _Complet return _CompletedTurn( model=ended.response.model or session_model, response=ended.response, - usage=ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # the one shared Responses usage converter - ended.response.usage - ), + usage=ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(ended.response.usage), ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 0fbfa0b06a7..0c89627e74d 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -23,7 +23,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text -from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY +from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, PgBouncerSettings, @@ -409,6 +409,7 @@ class ProxyInitializationHelpers: log_config: str | None = None, keepalive_timeout: int | None = None, timeout_worker_healthcheck: int | None = None, + max_request_header_size: int | None = None, ) -> dict: """ Get the arguments for `uvicorn` worker @@ -427,6 +428,9 @@ class ProxyInitializationHelpers: "port": port, "server_header": False, "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), + "h11_max_incomplete_event_size": ( + max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE + ), } if log_config is not None: print_startup_message(f"Using log_config: {log_config}") @@ -680,6 +684,7 @@ class ProxyInitializationHelpers: ssl_keyfile_path: str, max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, + max_request_header_size: int | None = None, ): """ Run litellm with `gunicorn` @@ -754,7 +759,7 @@ class ProxyInitializationHelpers: gunicorn_options: Final = { "bind": f"{host}:{port}", "workers": num_workers, # default is 1 - "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout), + "worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout, max_request_header_size), "graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS, "preload": True, # Add the preload flag, "accesslog": "-", # Log to stdout @@ -1108,6 +1113,18 @@ class ProxyInitializationHelpers: help="Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)", envvar="KEEPALIVE_TIMEOUT", ) +@click.option( + "--max_request_header_size", + default=None, + show_default=f"{DEFAULT_MAX_REQUEST_HEADER_SIZE} bytes with h11", + type=click.IntRange(min=1), + help=( + "Maximum bytes buffered for an incomplete HTTP request header block. " + "Applies when uvicorn or --run_gunicorn uses h11. Parser selection is unchanged; httptools ignores this setting. " + "Ignored under --run_hypercorn / --run_granian." + ), + envvar="MAX_REQUEST_HEADER_SIZE", +) @click.option( "--timeout_worker_healthcheck", default=None, @@ -1253,6 +1270,7 @@ def run_server( check_migrations_against_existing_data: bool, keepalive_timeout, timeout_worker_healthcheck, + max_request_header_size: int | None, max_requests_before_restart: int | None, max_requests_before_restart_jitter: int | None, limit_concurrency: int | None, @@ -1747,6 +1765,7 @@ def run_server( log_config=log_config, keepalive_timeout=keepalive_timeout, timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), + max_request_header_size=max_request_header_size, ) # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: @@ -1821,6 +1840,7 @@ def run_server( ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, + max_request_header_size=max_request_header_size, ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ed65ab71220..9a80761b477 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6,6 +6,7 @@ import importlib import inspect import io import itertools +import logging import os import random import re @@ -60,6 +61,7 @@ import anyio import anyio.lowlevel import websockets import websockets.exceptions +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config from pydantic.fields import FieldInfo, PydanticUndefined from pydantic.main import IncEx @@ -215,6 +217,7 @@ from litellm.types.utils import ( TextCompletionResponse, TokenCountMessagesFormat, TokenCountResponse, + TranscriptionResponse, echoed_cost_map_pricing_fields, is_server_derived_pricing_key, pricing_override_fields, @@ -324,6 +327,7 @@ from litellm._internal_context import service_target, with_service_target from litellm._logging import ( STARTUP_OUTPUT_SETTINGS, print_startup_message, + redact_internal_details_from_client_message, redact_string, verbose_proxy_logger, verbose_router_logger, @@ -373,6 +377,11 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + TRANSCRIPTION_STREAM_DONE_FRAME, + GuardedTranscriptionReplay, + guarded_transcription_events, +) from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, @@ -415,9 +424,13 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( + router as model_deprecation_alerts_router, +) from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) +from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload from litellm.proxy.auth.auth_checks import ( ROLE_BASED_PERMISSIONS_ADAPTER, ExperimentalUIJWTToken, @@ -471,6 +484,7 @@ from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # lega user_api_key_auth, user_api_key_auth_accepted_websocket_for_model, user_api_key_auth_live_websocket, + user_api_key_auth_realtime_websocket, user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router @@ -521,6 +535,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -536,6 +551,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10 resolve_inference_model, safe_get_request_headers, ) +from litellm.proxy.common_utils.http_parsing_utils import ( + resolve_realtime_route_model as _resolve_realtime_route_model, +) from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( @@ -555,9 +573,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, attribute_of, + error_status_code, headers_with_litellm_call_id, litellm_call_id_headers, openai_error_param, + openai_error_type, proxy_exception_for, route_failure_proxy_exception, with_litellm_call_id, @@ -862,8 +882,6 @@ from litellm.proxy.spend_tracking.spend_event_producer import ( SpendEventProducer, build_spend_event_producer, ) -from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime -from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings from litellm.proxy.telemetry.endpoints import router as telemetry_router from litellm.proxy.telemetry.middleware import TelemetryMiddleware from litellm.proxy.telemetry.runtime import TelemetryRuntime @@ -1171,6 +1189,27 @@ server_root_path: Final = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() premium_user_data: Optional["EnterpriseLicenseData"] = _license_check.airgapped_license_data + + +def _set_premium_user(is_premium: bool) -> None: + global premium_user, premium_user_data + premium_user = is_premium # rebind-ok: config load can switch the license in use + premium_user_data = _license_check.airgapped_license_data # rebind-ok: tracks the license in use + + +def _use_config_license(config_license: str, env_license_is_premium: bool) -> bool: + env_license: Final = _license_check.license_str + env_license_data: Final = _license_check.airgapped_license_data + _license_check.license_str = config_license + if _license_check.is_premium(): + return True + if not env_license_is_premium: + return False + _license_check.license_str = env_license + _license_check.airgapped_license_data = env_license_data + return True + + global_max_parallel_request_retries_env: Final[str | None] = os.getenv("LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES") proxy_state: Final = ProxyState() SENSITIVE_DATA_MASKER: Final = SensitiveDataMasker() @@ -1356,7 +1395,6 @@ async def _flush_spend_around_logging_drain_on_shutdown( finally: await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock)) await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock)) - await usage_capture.stop(GracefulShutdownManager.remaining(clock=clock)) await flush_spend() @@ -1638,6 +1676,8 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode(general_settings_view()) + async with AsyncExitStack() as boot_database: enforce_master_key_boot_verdict( await with_stored_secrets_counted( @@ -2893,9 +2933,15 @@ def configured_max_request_size_mb() -> int | float | None: return MAX_REQUEST_SIZE_MB_SETTING.validate_python(general_settings.get("max_request_size_mb")) +_ROUTE_MAX_REQUEST_SIZE_MB: Final[Mapping[tuple[str, str], float]] = MappingProxyType( + {("POST", "/claude-code/plugins/upload"): 11} +) + + app.add_middleware( RequestBodyDecompressionMiddleware, get_max_request_size_mb=lambda: configured_max_request_size_mb() if premium_user is True else None, + route_max_request_size_mb=_ROUTE_MAX_REQUEST_SIZE_MB, ) app.add_middleware( CORSMiddleware, @@ -3265,39 +3311,9 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False): spend_event_producer: SpendEventProducer | None = None -usage_capture: Final = UsageCaptureRuntime() - - -def _configure_usage_capture() -> None: - settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings) - capture_settings: Final = settings.get("accounting_capture") - if settings.get("accounting_sync_ack") and capture_settings is None: - raise ValueError("accounting_sync_ack requires accounting_capture") - if capture_settings is None: - if usage_capture.meter is not None: - raise ValueError("Disabling accounting_capture requires a worker restart") - return - if ( - prisma_client is None - or CollectorSettings().enabled - or os.getenv("SPEND_LOGS_URL") - or litellm.apply_search_free_tier - ): - raise ValueError( - "Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier" - ) - from litellm.proxy.db.accounting_ledger import AccountingLedger - - ledger: Final = AccountingLedger.for_client(prisma_client) - writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch) - usage_capture.start( - writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch - ) - def cost_tracking(): global prisma_client, spend_event_producer - _configure_usage_capture() if prisma_client is not None: from litellm.integrations.shadow_eval_logger import ShadowEvalLogger @@ -3605,6 +3621,7 @@ async def increment_spend_counters( request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, project_id: str | None = None, + windows_from_ledger: bool = False, ): """ Atomically increment spend counters for budget enforcement. @@ -3641,6 +3658,7 @@ async def increment_spend_counters( request_started_at=request_started_at, model_access_groups=model_access_groups, project_id=project_id, + windows_from_ledger=windows_from_ledger, ) @@ -3670,6 +3688,50 @@ def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bo user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key) +async def accounting_window_keys(token: str | None, team_id: str | None) -> tuple[Mapping[str, object], ...]: + scopes: Final = tuple( + (entity_type, entity_id, cache_key) + for entity_type, entity_id, cache_key in ( + (Litellm_EntityType.KEY, token, token), + (Litellm_EntityType.TEAM, team_id, f"team_id:{team_id}"), + ) + if entity_id is not None and cache_key is not None + ) + snapshots: Final = tuple( + [ + await _accounting_scope_windows(entity_type, entity_id, cache_key) + for entity_type, entity_id, cache_key in scopes + ] + ) + from itertools import chain + + return tuple(chain.from_iterable(snapshots)) + + +async def _accounting_scope_windows( + entity_type: Litellm_EntityType, entity_id: str, cache_key: str +) -> tuple[Mapping[str, object], ...]: + from litellm.proxy.spend_tracking.budget_reservation import fixed_budget_window + + cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) + limits: Final = ( + await _stored_budget_limits(entity_type, entity_id) if cached is None else _cached_budget_limits(cached) + ) + if not isinstance(limits, list): + return () + windows: Final = TypeAdapter(tuple[object, ...]).validate_python(limits) + return tuple( + { + "entity_type": entity_type.value, + "entity_id": entity_id, + "window_duration": fixed[0], + "window_start": fixed[1].isoformat(), + } + for window in windows + if (fixed := fixed_budget_window(window)) is not None + ) + + async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object: """Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache.""" if prisma_client is None: @@ -3711,6 +3773,7 @@ async def _increment_spend_counters_batched( request_started_at: datetime | None, model_access_groups: Sequence[str] | None, project_id: str | None = None, + windows_from_ledger: bool = False, ): """Runs inside one spend counter batch: the reservation reconcile and the warm checks share a single MGET, and the reconcile adjustments go out in the same INCRBYFLOAT pipeline as the counter increments.""" @@ -3752,15 +3815,16 @@ async def _increment_spend_counters_batched( if window_counter not in reserved_counter_keys else None ) - await _enqueue_window_spend_row_update( - entity_type=entity_type, - entity_id=entity_id, - reset_at=window_reset_at, - window_duration=duration, - window_start=window_start, - increment=cost, - request_started_at=request_started_at, - ) + if not windows_from_ledger: + await _enqueue_window_spend_row_update( + entity_type=entity_type, + entity_id=entity_id, + reset_at=window_reset_at, + window_duration=duration, + window_start=window_start, + increment=cost, + request_started_at=request_started_at, + ) return pending_window async def _window_increments( @@ -6672,7 +6736,6 @@ class ProxyConfig: def _load_environment_variables(self, config: dict): ## ENVIRONMENT VARIABLES - global premium_user environment_variables: Final = config.get("environment_variables", None) if environment_variables: for key, value in environment_variables.items(): @@ -6712,7 +6775,7 @@ class ProxyConfig: # check if litellm_license in general_settings if "LITELLM_LICENSE" in environment_variables: _license_check.license_str = os.getenv("LITELLM_LICENSE", None) - premium_user = _license_check.is_premium() + _set_premium_user(_license_check.is_premium()) def _set_config_environment_variable(self, key: str, value: str) -> None: if key in os.environ and os.environ[key] != value: @@ -6764,7 +6827,6 @@ class ProxyConfig: prompt_injection_detection_obj, \ redis_usage_cache, \ store_model_in_db, \ - premium_user, \ open_telemetry_logger, \ health_check_details, \ proxy_batch_polling_interval, \ @@ -6777,6 +6839,12 @@ class ProxyConfig: self._load_environment_variables(config=config) + config_license: Final = _GENERAL_SETTINGS_VIEW.validate_python( + _GENERAL_SETTINGS_VIEW.validate_python(config).get("general_settings") or {} + ).get("litellm_license") + if isinstance(config_license, str): + _set_premium_user(_use_config_license(config_license, env_license_is_premium=premium_user)) + ## Coordination Redis (before cache init, so the explicit block wins) coordination_redis_cache: Final = self._init_coordination_redis(config=config) if coordination_redis_cache is not None: @@ -7353,7 +7421,7 @@ class ProxyConfig: allowed_ips: Final = general_settings.get("allowed_ips", None) if allowed_ips is not None and premium_user is False: raise ValueError( - "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) ## BUDGET RESCHEDULER ## proxy_budget_rescheduler_min_time = general_settings.get( @@ -7409,11 +7477,6 @@ class ProxyConfig: if general_settings.get("enforced_params") is not None and premium_user is not True: raise ValueError("Trying to use `enforced_params`" + CommonProxyErrors.not_premium_user.value) - # check if litellm_license in general_settings - if "litellm_license" in general_settings: - _license_check.license_str = general_settings["litellm_license"] - premium_user = _license_check.is_premium() - router_params: Final[dict] = { "cache_responses": litellm.cache is not None, # cache if user passed in cache values } @@ -10715,6 +10778,26 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob return _resolve +def _transcription_sse_error_frame(client_error: Exception) -> str: + status_code: Final = error_status_code(client_error, 500) + error_returned: Final = json.dumps( + { + "error": sse_error_payload(client_error)[1] + if isinstance(client_error, HTTPException) + else ProxyException( + message=redact_internal_details_from_client_message( + str(getattr(client_error, "message", client_error)) + ), + type=openai_error_type(client_error, status_code), + param=openai_error_param(client_error), + code=status_code, + ).to_dict() + }, + default=str, + ) + return f"data: {error_returned}\n\n" + + async def async_data_generator( response, user_api_key_dict: UserAPIKeyAuth, @@ -13345,6 +13428,13 @@ async def completion( _response.choices[0].text = e.message return _response except Exception as e: + if is_provider_overload(e): + raise await base_llm_response_processor.handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -13751,11 +13841,13 @@ def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, o "/v1/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) @router.post( "/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) async def audio_transcriptions( request: Request, @@ -13843,34 +13935,72 @@ async def audio_transcriptions( cache_key: Final = hidden_params.get("cache_key", None) or "" api_base: Final = hidden_params.get("api_base", None) or "" response_cost: Final = hidden_params.get("response_cost", None) or "" - response_call_id: Final = hidden_params.get("litellm_call_id", None) or "" + response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {} + stream_logging_obj: Final = getattr(response, "logging_obj", None) - fastapi_response.headers.update( - ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - call_id=response_call_id, - request_data=data, - hidden_params=hidden_params, - **additional_headers, - ) + custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=user_api_key_dict.allowed_model_region, + call_id=response_call_id, + request_data=data, + hidden_params=hidden_params, + litellm_logging_obj=stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None, + **additional_headers, ) # Call response headers hook (matches base_process_llm_request behavior) callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook( data=data, user_api_key_dict=user_api_key_dict, - response=response, + response=response.response if isinstance(response, GuardedTranscriptionReplay) else response, request_headers=dict(request.headers), ) - if callback_headers: - fastapi_response.headers.update(callback_headers) + response_headers: Final = {**custom_headers, **(callback_headers or {})} + + if hasattr(response, "__aiter__"): + + async def transcription_event_stream( + stream: AsyncIterator[TranscriptionStreamEvent], + ) -> AsyncGenerator[str, None]: + try: + async for event in stream: + yield f"data: {event.model_dump_json(exclude_unset=True)}\n\n" + yield TRANSCRIPTION_STREAM_DONE_FRAME + except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event + replacement: Final = await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=stream_error, + request_data=( + {**data, "litellm_logging_obj": stream_logging_obj} + if isinstance(stream_logging_obj, LiteLLMLoggingObj) + else data + ), + ) + yield _transcription_sse_error_frame(stream_error if replacement is None else replacement) + yield TRANSCRIPTION_STREAM_DONE_FRAME + finally: + with anyio.CancelScope(shield=True): + close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) + if callable(close): + close_result: Final = close() + if inspect.isawaitable(close_result): + await close_result + + return StreamingResponse( + guarded_transcription_events(response.response) + if isinstance(response, GuardedTranscriptionReplay) + else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)), + media_type="text/event-stream", + headers=response_headers, + ) + + fastapi_response.headers.update(response_headers) return response except Exception as e: @@ -14019,6 +14149,8 @@ async def _reject_realtime_session( @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") +@app.websocket("/v1/realtime/translations") +@app.websocket("/realtime/translations") async def realtime_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."), @@ -14027,7 +14159,7 @@ async def realtime_websocket_endpoint( None, description="Comma-separated list of guardrail names to apply to this request.", ), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_realtime_websocket), ): requested_protocols: Final = tuple( protocol.strip() @@ -14035,15 +14167,13 @@ async def realtime_websocket_endpoint( if protocol.strip() ) - route_model = model + is_translation: Final = websocket.url.path.endswith("/realtime/translations") + route_model: Final = _resolve_realtime_route_model(model, intent, is_translation) if route_model is None: - if intent == "transcription": - route_model = "gpt-realtime-whisper" - else: - await _reject_realtime_session( - websocket, user_api_key_dict, code=1008, reason="model query parameter is required" - ) - return + await _reject_realtime_session( + websocket, user_api_key_dict, code=1008, reason="model query parameter is required" + ) + return assert route_model is not None try: await can_key_call_resolved_model( @@ -14064,6 +14194,7 @@ async def realtime_websocket_endpoint( model=model, intent=intent, guardrails=guardrails, + is_translation=is_translation, ) @@ -14077,13 +14208,20 @@ async def _route_realtime_websocket_session( guardrails: str | None, request_path: str | None = None, live_session_start: Mapping[str, object] | None = None, + is_translation: bool = False, ) -> None: # Only use explicit parameters, not all query params - query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))) + query_model: Final = route_model if is_translation else model + query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys + RealtimeQueryParams, + dict(_realtime_query_params_template(query_model, intent)), + ) + data: dict[str, object] = { "model": route_model, "websocket": websocket, "query_params": query_params, # Only explicit params + "realtime_mode": "translation" if is_translation else "realtime", } # Pass guardrails into data so pre-call guardrail processing picks them up if guardrails: @@ -16898,9 +17036,6 @@ async def _apply_search_filter_to_models( team_public_model_name: Final = (m.get("model_info") or {}).get("team_public_model_name") or "" if search_lower in team_public_model_name.lower(): return True - # Also match the underlying LiteLLM model name (e.g. - # "openrouter/deepseek/deepseek-chat"), so users can find - # deployments by typing the provider or the upstream model id. litellm_params: Final[Mapping[str, object]] = m.get("litellm_params") or MappingProxyType({}) litellm_model: Final = litellm_params.get("model") return isinstance(litellm_model, str) and search_lower in litellm_model.lower() @@ -18409,8 +18544,6 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(registry.deployments(source="alias")) all_models.extend(alias_models) - all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router)) - allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, llm_router=llm_router, @@ -18419,6 +18552,15 @@ async def model_info_v1( proxy_logging_obj=proxy_logging_obj, ) + all_models = await run_model_listing( + partial( + expand_wildcard_deployments_for_model_info, + all_models, + llm_router, + allowed_model_names=allowed_model_names, + ) + ) + all_models = _filter_v1_model_info_deployments( all_models=all_models, allowed_model_names=allowed_model_names, @@ -18502,8 +18644,9 @@ async def model_deprecations( Reads `deprecation_date` metadata from `model_prices_and_context_window.json` (and any per-deployment `model_info.deprecation_date` overrides) for the - models configured on this proxy. An entry also names its `successor_model` - when the deployment sets `model_info.successor_model`. + models configured on this proxy. An entry also names its `successor_model`: + `model_info.successor_model` when the deployment sets one, else the + replacement the cost map records from the provider's deprecation notice. Parameters: warn_within_days: Window (in days) used to bucket "imminent" models, @@ -18523,7 +18666,11 @@ async def model_deprecations( -H "Authorization: Bearer $LITELLM_MASTER_KEY" ``` """ - return collect_model_deprecations(llm_router=llm_router, warn_within_days=warn_within_days) + return collect_model_deprecations( + llm_router=llm_router, + warn_within_days=warn_within_days, + successor_log_level=logging.DEBUG, + ) def get_model_group_info( @@ -22290,6 +22437,7 @@ app.include_router(prompt_caching_requests_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) +app.include_router(model_deprecation_alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) @@ -22307,6 +22455,7 @@ app.add_middleware( RequestSizeLimitMiddleware, get_max_request_size_mb=configured_max_request_size_mb, is_request_size_limit_enabled=lambda: premium_user is True, + route_max_request_size_mb=_ROUTE_MAX_REQUEST_SIZE_MB, ) app.add_middleware( AdmissionControlMiddleware, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 044b8a1de6e..5ded2fa5f5e 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1553,6 +1553,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "EMPIRIOLABS", + "provider_display_name": "EmpirioLabs AI", + "litellm_provider": "empiriolabs", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.empiriolabs.ai/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "empiriolabs/aplomb-1" + }, { "provider": "EMPOWER", "provider_display_name": "Empower", @@ -2641,6 +2669,34 @@ ], "default_model_placeholder": "nace/drex-latest" }, + { + "provider": "PARALLEL_AI", + "provider_display_name": "Parallel AI", + "litellm_provider": "parallel_ai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.parallel.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "parallel_ai/decision" + }, { "provider": "NEBIUS", "provider_display_name": "Nebius", @@ -3539,6 +3595,34 @@ ], "default_model_placeholder": "reka/reka-flash" }, + { + "provider": "WALLY", + "provider_display_name": "Wally", + "litellm_provider": "wally", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://inference.runanywhere.ai/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "wally/glm-5.3-flash" + }, { "provider": "Sail", "provider_display_name": "Sail", diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 9dbf6bcc43d..17c52fc54bd 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,19 +2,24 @@ import json import time -from collections.abc import Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager from typing import TYPE_CHECKING, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status -from pydantic import ConfigDict, TypeAdapter, with_config +from fastapi.responses import PlainTextResponse +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict +import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -43,7 +48,26 @@ router: Final = APIRouter() _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" -_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") + + +@contextmanager +def _invalid_request_body_answers_400() -> Generator[None, None, None]: + try: + yield + except ValidationError as error: + raise ProxyException(message=str(error), type="invalid_request_error", param=None, code=400) from error + + +_TRANSLATION_SECRET_BODY: Final = { + "requestBody": inline_request_body( + RealtimeClientSecretRequest, + {"model": "gpt-realtime-translate", "session": {"type": "translation", "model": "gpt-realtime-translate"}}, + ) +} +_TRANSLATION_SDP_BODY: Final = { + "requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}} +} @with_config(ConfigDict(strict=True, extra="allow")) @@ -140,19 +164,60 @@ def _set_transcription_model_on_session( } +def nested_transcription_upstream_model(model: str, team_id: str | None, llm_router: "Router | None") -> str: + deployment_model: Final = ( + llm_router.registry().resolve(model, kind="upstream", team_id=team_id) if llm_router is not None else None + ) + if deployment_model is None: + return model + try: + return litellm.get_llm_provider(model=deployment_model)[0] + except litellm.BadRequestError: + return deployment_model + + +async def _authorize_and_bind_nested_transcription_models( + session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization + user_api_key_dict: UserAPIKeyAuth, + llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list + llm_router: "Router | None", +) -> None: + nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + if nested_models: + _set_transcription_model_on_session( + session=session_data, + model=nested_transcription_upstream_model(nested_models[0], user_api_key_dict.team_id, llm_router), + ) + + async def _prepare_client_secret_session( req: RealtimeClientSecretRequest, user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, llm_router: "Router | None", + forced_session_type: str | None = None, ) -> tuple[str, dict | None, str]: - session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None) - session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None + requested_session_type: Final = req.session.type if req.session else None + if forced_session_type is None and requested_session_type == "translation": + raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") + session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) + session_data: Final[dict | None] = ( + req.session.model_dump(exclude_unset=True) if req.session else ({} if session_type == "translation" else None) + ) if session_data is not None: session_data["type"] = session_type session_model: Final = req.session.model if req.session else None - model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL + requested_model: Final = session_model or req.model + default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL + model: str = requested_model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -160,6 +225,15 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) + if session_data is not None and (requested_model or session_type == "translation"): + session_data["model"] = model + if session_type == "translation": + await _authorize_and_bind_nested_transcription_models( + session_data=session_data, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {}) @@ -194,12 +268,13 @@ def _encode_realtime_token_payload( team_id: str | None, expires_at: int | None, session_type: str = "realtime", + caller_auth: Mapping[str, object] | None = None, ) -> str: """ Encode metadata with the upstream ephemeral key so /realtime/calls can route without requiring model as a query param. """ - payload: Final[dict[str, str | int | None]] = { + payload: Final[Mapping[str, object]] = { "v": _REALTIME_TOKEN_VERSION, "ephemeral_key": ephemeral_key, "model_id": model_id, @@ -207,10 +282,24 @@ def _encode_realtime_token_payload( "team_id": team_id or "", "expires_at": expires_at, "session_type": session_type, + **({} if caller_auth is None else {"caller_auth": caller_auth}), } return json.dumps(payload, separators=(",", ":")) +def _caller_auth_for_token(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]: + return user_api_key_dict.model_dump(mode="json", exclude_none=True, exclude={"api_key", "parent_otel_span"}) + + +def _caller_auth_from_token( + decoded_payload: Mapping[str, object] | None, user_id: str | None, team_id: str | None +) -> UserAPIKeyAuth: + caller_auth: Final = None if decoded_payload is None else as_str_mapping(decoded_payload.get("caller_auth")) + if caller_auth is None: + return UserAPIKeyAuth(user_id=user_id, team_id=team_id) + return UserAPIKeyAuth.model_validate({**caller_auth, "api_key": caller_auth.get("token")}) + + def _header_value(headers: Mapping[str, str], name: str, default: str) -> str: return headers.get(name, default) @@ -249,11 +338,24 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) +@router.post( + "/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) async def create_realtime_client_secret( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ) -> RealtimeClientSecretResponse: + """Create a short-lived client token for a Realtime conversation, transcription, or translation session""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, @@ -269,13 +371,16 @@ async def create_realtime_client_secret( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeClientSecretRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeClientSecretRequest.model_validate(body) + is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( req=req, user_api_key_dict=user_api_key_dict, llm_model_list=llm_model_list, llm_router=llm_router, + forced_session_type="translation" if is_translation_request else None, ) data = {"model": model} @@ -299,17 +404,20 @@ async def create_realtime_client_secret( proxy_config=proxy_config, ) + call_type: Final = ( + "acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret" + ) data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, - call_type="acreate_realtime_client_secret", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="acreate_realtime_client_secret", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -366,6 +474,7 @@ async def create_realtime_client_secret( team_id=getattr(user_api_key_dict, "team_id", None), expires_at=expires_at if isinstance(expires_at, int) else None, session_type=session_type, + caller_auth=_caller_auth_for_token(user_api_key_dict) if session_type == "translation" else None, ) encrypted_token: Final[str] = encrypt_value_helper(token_payload) upstream_json["value"] = encrypted_token @@ -392,10 +501,27 @@ async def create_realtime_client_secret( "/openai/v1/realtime/calls", tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", + openapi_extra=_TRANSLATION_SDP_BODY, + tags=["realtime"], +) +@router.post( + "/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", + openapi_extra=_TRANSLATION_SDP_BODY, + tags=["realtime"], +) async def proxy_realtime_calls( request: Request, fastapi_response: Response, ) -> Response: + """Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, @@ -417,6 +543,7 @@ async def proxy_realtime_calls( media_type="application/json", ) + is_translation_request: Final = "/realtime/translations/calls" in request.url.path encrypted_token: Final = auth_header.removeprefix("Bearer ").strip() decrypted_token_value: Final = decrypt_value_helper( value=encrypted_token, @@ -434,33 +561,39 @@ async def proxy_realtime_calls( if decoded_payload is not None: # Check token expiry expires_at: Final = decoded_payload.get("expires_at") - if expires_at is not None and isinstance(expires_at, int): - if time.time() > expires_at: - return Response( - content=json.dumps({"error": "Token has expired"}), - status_code=http_status.HTTP_401_UNAUTHORIZED, - media_type="application/json", - ) + if isinstance(expires_at, int) and time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL user_id = decoded_payload.get("user_id") or None team_id = decoded_payload.get("team_id") or None - session_type = _coerce_realtime_session_type(decoded_payload.get("session_type")) + raw_session_type: Final = decoded_payload.get("session_type") + session_type = _coerce_realtime_session_type(raw_session_type) + if is_translation_request != (raw_session_type == "translation"): + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) else: - # Backward compatibility: older tokens contained only encrypted upstream key. + if is_translation_request: + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL) user_id = None team_id = None session_type = "realtime" - # Build a minimal UserAPIKeyAuth with user/team IDs from the token - # so spend tracking and budget enforcement work correctly. - minimal_auth: Final = UserAPIKeyAuth( - user_id=user_id, - team_id=team_id, - ) + token_caller_auth: Final = _caller_auth_from_token(decoded_payload, user_id, team_id) data: dict = {} try: @@ -487,22 +620,23 @@ async def proxy_realtime_calls( data=data, request=request, general_settings=general_settings, - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, version=version, proxy_config=proxy_config, ) + call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls" data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, data=data, - call_type="arealtime_calls", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="arealtime_calls", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -510,7 +644,7 @@ async def proxy_realtime_calls( except Exception as e: await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, original_exception=e, request_data=data, ) @@ -581,7 +715,8 @@ async def create_realtime_transcription_session( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeTranscriptionSessionRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeTranscriptionSessionRequest.model_validate(body) model: Final[str] = req.resolved_model() or "gpt-realtime-whisper" await can_key_call_resolved_model( @@ -592,6 +727,7 @@ async def create_realtime_transcription_session( ) transcription_session: Final = {k: v for k, v in body.items() if k != "model"} + _set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True) data = {"model": model, "transcription_session": transcription_session} data = await add_litellm_data_to_request( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 3f76bcf69ad..c4cd7c2d411 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -2162,7 +2162,7 @@ _WS_FRAME: Final = TypeAdapter(dict[str, object]) _GUARDRAIL_INFORMATION: Final = TypeAdapter(list[dict[str, object]]) _PII_TOKENS: Final = TypeAdapter(dict[str, str]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" -_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools") +_GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools", "tool_choice", "extra_body") _CLIENT_METADATA_KEYS: Final = ("metadata", "litellm_metadata") _CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policies") _METADATA_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config") diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index ecc31f3b0a7..148d465099f 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -621,7 +621,9 @@ RouteType = Literal[ "alist_input_items", "_arealtime", # private function for realtime API "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "acreate_realtime_transcription_session", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 179b741fe7c..78f16d9f1fd 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1615,9 +1615,18 @@ model LiteLLM_ClaudeCodePluginTable { updated_at DateTime? @default(now()) @updatedAt created_by String? + uploaded_archive LiteLLM_ClaudeCodePluginUpload? + @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginUpload { + plugin_name String @id + content Bytes + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_name], references: [name], onDelete: Cascade, onUpdate: Cascade) +} + + // User/team-scoped memory store with a GLOBAL unique key. // `value` is a string (typically markdown/text meant for LLM context); // `metadata` is an optional JSON envelope for structured tags without schema changes. @@ -2062,74 +2071,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/litellm/proxy/shutdown/graceful_uvicorn_worker.py b/litellm/proxy/shutdown/graceful_uvicorn_worker.py index 6ae5ae60c3f..0bbbfe0a4a9 100644 --- a/litellm/proxy/shutdown/graceful_uvicorn_worker.py +++ b/litellm/proxy/shutdown/graceful_uvicorn_worker.py @@ -1,10 +1,21 @@ from uvicorn.workers import UvicornWorker +from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE -def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]: + +def build_graceful_shutdown_uvicorn_worker( + timeout_graceful_shutdown: float, + max_request_header_size: int | None = None, +) -> type[UvicornWorker]: class GracefulShutdownUvicornWorker(UvicornWorker): def __init__(self, *args: object) -> None: - self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown} + self.CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": timeout_graceful_shutdown, + "h11_max_incomplete_event_size": ( + max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE + ), + } super().__init__(*args) return GracefulShutdownUvicornWorker diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index dcfd6148f54..af7eb742aa7 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1264,6 +1264,15 @@ def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float return default_reserved_cost +def fixed_budget_window(window: object) -> tuple[str, datetime] | None: + fields: Final = _coerce_window(window) + start: Final = get_budget_window_start(window) + duration: Final = fields.get("budget_duration") + if not fields.get("reset_at") or not isinstance(duration, str) or start is None: + return None + return duration, start + + def get_budget_window_start(window: object) -> datetime | None: window_dict: Final = _coerce_window(window) budget_duration: Final = window_dict.get("budget_duration") diff --git a/litellm/proxy/spend_tracking/session_page.py b/litellm/proxy/spend_tracking/session_page.py new file mode 100644 index 00000000000..810afbfa7b7 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_page.py @@ -0,0 +1,221 @@ +"""Which sessions a logs page shows and how many match, read from the newest rows instead of every row. Reads grow +until one settles both, so the result is what grouping every row would give.""" + +import math +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" +AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" +SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( + f"(call_type = {AGENT_CALL_TYPE_SQL}) DESC, " + f'CASE WHEN call_type = {AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' + f'call_type IN {MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' +) + + +class _ReadRow(TypedDict): + session_key: ReadOnly[str | None] + api_key: ReadOnly[str | None] + last_activity: ReadOnly[str | None] + settled: ReadOnly[bool | None] + rows_read: ReadOnly[int] + session_count: ReadOnly[int] + + +class _SupportsQueryRaw(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ... + + +@dataclass(frozen=True, slots=True) +class Session: + key: str + api_key: str + last_activity: str + + +@dataclass(frozen=True, slots=True) +class SessionPage: + sessions: tuple[Session, ...] + has_more: bool + next_cursor: str | None + total: int + total_is_capped: bool + + +@dataclass(frozen=True, slots=True) +class _Request: + cursor: tuple[str, str, str] | None + offset: int + page_limit: int + sort_desc: bool + status: str | None + cap: int + + def first_read_size(self) -> int | None: + """Every row when oldest first, or when filtered by outcome, which any of a session's rows can decide. Otherwise + the fewest that could settle a page: a session past the cap and an older row.""" + return self.cap + 2 if self.sort_desc and self.status is None else None + + +@dataclass(frozen=True, slots=True) +class _Read: + candidates: tuple[Session, ...] + last_settled: bool + rows_read: int + session_count: int + saw_every_row: bool + + def settles(self, request: _Request) -> bool: + page_settled: Final = ( + request.page_limit <= 0 + or self.saw_every_row + or (len(self.candidates) > request.page_limit and self.last_settled) + ) + return page_settled and (self.saw_every_row or self.session_count > request.cap) + + def next_size(self, request: _Request) -> int: + rows_per_session: Final = self.rows_read / self.session_count + return max(2 * self.rows_read, math.ceil(1.25 * (request.cap + 1) * rows_per_session)) + + +_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]] + + +def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: + if session_cursor is None or session_cursor.count("|") < 2: + return None + last_activity, _, rest = session_cursor.partition("|") + api_key, _, session_key = rest.partition("|") + if not last_activity or not session_key: + return None + return (last_activity, session_key, api_key) + + +async def read_session_page( + db: _SupportsQueryRaw, + *, + where_clause: str, + params: Sequence[object], + status: str | None, + page: int, + page_size: int, + sort_desc: bool, + session_cursor: str | None, + cap: int, +) -> SessionPage: + """One page of sessions after ``session_cursor``, or after ``(page - 1) * page_size`` sessions, keeping only those + whose outcome is ``status`` when given. ``where_clause`` reads ``params`` as ``$1`` onward.""" + cursor: Final = _parse_session_cursor(session_cursor) + offset: Final = (page - 1) * page_size if cursor is None else 0 + request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, status, cap) + return await _session_page(_sql_reader(db, where_clause, params, request), request) + + +async def _session_page(read: _Reader, request: _Request) -> SessionPage: + last = await read(request.first_read_size()) # rebind-ok: each read replaces the last until one settles the page + while not last.settles(request): + last = await read(last.next_size(request)) + has_more: Final = len(last.candidates) > request.page_limit + sessions: Final = last.candidates[: request.page_limit] + final: Final = sessions[-1] if has_more and sessions else None + return SessionPage( + sessions=sessions, + has_more=has_more, + next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None, + total=min(last.session_count, request.cap), + total_is_capped=last.session_count > request.cap, + ) + + +def _sessions_sql(request: _Request, status_index: int) -> str: + """Each session's last activity, from its rows in ``scanned``, keeping only sessions whose outcome is the status.""" + if request.status is None: + return ( + 'SELECT session_key, api_key, MAX("startTime") AS last_activity FROM scanned GROUP BY session_key, api_key' + ) + return f""" + SELECT session_key, api_key, last_activity + FROM ( + SELECT DISTINCT ON (session_key, api_key) + session_key, + api_key, + status, + MAX("startTime") OVER (PARTITION BY session_key, api_key) AS last_activity + FROM scanned + ORDER BY session_key, api_key, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS outcomes + WHERE COALESCE(status, 'success') = ${status_index} + """ + + +def _sql_reader(db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], request: _Request) -> _Reader: + status_index: Final = len(params) + 1 + status_args: Final = () if request.status is None else (request.status,) + cursor_index: Final = status_index + len(status_args) + after_cursor: Final = ( + f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} " + f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})" + if request.cursor + else "" + ) + outcome_columns: Final = ', status, call_type, "endTime", request_id' if request.status is not None else "" + direction: Final = "DESC" if request.sort_desc else "ASC" + page_rows: Final = request.page_limit + 1 if request.page_limit > 0 else 0 + skip: Final = f"OFFSET {request.offset}" if page_rows and request.offset else "" + + async def read(size: int | None) -> _Read: + # LEFT JOIN keeps the summary row when no session is on the page. + rows: Final = await db.query_raw( + f""" + WITH scanned AS ( + SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"{outcome_columns} + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + {f'ORDER BY "startTime" DESC LIMIT {size}' if size is not None else ""} + ), + sessions AS ({_sessions_sql(request, status_index)}), + summary AS ( + SELECT COUNT(*) AS rows_read, + MIN("startTime") AS oldest, + (SELECT COUNT(*) FROM sessions) AS session_count + FROM scanned + ) + SELECT page.session_key, + page.api_key, + page.last_activity::text AS last_activity, + page.last_activity > summary.oldest AS settled, + summary.rows_read, + summary.session_count + FROM summary + LEFT JOIN LATERAL ( + SELECT session_key, api_key, last_activity + FROM sessions + {after_cursor} + ORDER BY last_activity {direction}, session_key {direction}, api_key {direction} + LIMIT {page_rows} {skip} + ) AS page ON TRUE + ORDER BY page.last_activity {direction}, page.session_key {direction}, page.api_key {direction} + """, + *params, + *status_args, + *(request.cursor or ()), + ) + rows_read: Final = int(rows[0]["rows_read"]) + return _Read( + candidates=tuple( + Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"])) + for row in rows + if row["session_key"] is not None + ), + last_settled=bool(rows[-1]["settled"]), + rows_read=rows_read, + session_count=int(rows[0]["session_count"]), + saw_every_row=size is None or rows_read < size, + ) + + return read diff --git a/litellm/proxy/spend_tracking/spend_event.py b/litellm/proxy/spend_tracking/spend_event.py index 031e84896b2..d054519cb9c 100644 --- a/litellm/proxy/spend_tracking/spend_event.py +++ b/litellm/proxy/spend_tracking/spend_event.py @@ -28,6 +28,7 @@ from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.types.interactions import InteractionsAPIResponse from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import LiteLLMBatch, Usage +from litellm.types.videos.utils import is_video_status_call_type SPEND_EVENT_VERSION: Final = 1 CACHE_OFF_KEY: Final = "Cache OFF" @@ -205,9 +206,10 @@ class SpendEventDecodeError: reason: str -def is_offloadable_success(response_obj: object) -> bool: - """Batch retrieves and interaction polls branch on the concrete response class, so they stay in-process.""" - return not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) +def is_offloadable_success(response_obj: object, call_type: object) -> bool: + """Batch retrieves and interaction polls branch on the concrete response class, and a video status poll on + the status and video id the event leaves out, so they stay in-process.""" + return not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) and not is_video_status_call_type(call_type) def _json_fallback(value: object) -> str: diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 89200792c0b..fcb9eb4a280 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3,6 +3,7 @@ import collections import json import os from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone from functools import partial from itertools import groupby @@ -52,6 +53,13 @@ from litellm.proxy.auth.authorization_dependencies import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.session_page import ( + AGENT_CALL_TYPE_SQL, + MCP_CALL_TYPES_SQL, + SESSION_KEY_EXPR, + SESSION_REPRESENTATIVE_ORDER_SQL, + read_session_page, +) from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, @@ -65,7 +73,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_spend_by_team, get_spend_by_team_and_customer, ) -from litellm.proxy.utils import handle_exception_on_proxy +from litellm.proxy.utils import handle_exception_on_proxy, query_raw_within_statement_timeout from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import SpendLogsRepository from litellm.repositories.team_repository import TeamRepository @@ -87,23 +95,15 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 _DECODED_JSON: Final = TypeAdapter(object) -_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" -_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" -_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" -_AGENT_CALL_TYPE_SQL: Final = "'asend_message'" -_SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( - f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, " - f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' - f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' -) +_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key" _BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" _SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( { - "mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}", - "agent": f"call_type = {_AGENT_CALL_TYPE_SQL}", + "mcp": f"call_type IN {MCP_CALL_TYPES_SQL}", + "agent": f"call_type = {AGENT_CALL_TYPE_SQL}", "batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}", "llm": ( - f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} " + f"(call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} " f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})" ), } @@ -268,6 +268,20 @@ async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args: return await _query_raw(prisma_client, sql_query, *args) +async def _query_spend_logs(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]: + return cast( # cast-ok: callers declare the row shape their SELECT list produces, as with _query_raw + "Sequence[_RowT]", await query_raw_within_statement_timeout(prisma_client, sql_query, *args) + ) + + +@dataclass(frozen=True, slots=True) +class _SpendLogsReader: + prisma_client: PrismaClient + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + return await query_raw_within_statement_timeout(self.prisma_client, query, *args) + + class _TeamTable(Protocol): """The subset of the Prisma team table API this module uses.""" @@ -1567,10 +1581,7 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, api_key) - if db_response is None: - return [] - + db_response = await _query_spend_logs(prisma_client, sql_query, start_date_obj, end_date_obj, api_key) return db_response elif internal_user_id is not None: verbose_proxy_logger.debug("Getting /spend for internal_user_id: %s", internal_user_id) @@ -1610,12 +1621,9 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await _query_raw_or_none( + db_response = await _query_spend_logs( prisma_client, sql_query, start_date_obj, end_date_obj, internal_user_id ) - if db_response is None: - return [] - return db_response elif team_id is not None and customer_id is not None: return await get_spend_by_team_and_customer( @@ -1677,10 +1685,7 @@ async def get_global_spend_report( group_by_day; """ - db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) - if db_response is None: - return [] - + db_response = await _query_spend_logs(prisma_client, sql_query, start_date_obj, end_date_obj) return db_response elif group_by == "api_key": sql_query = """ @@ -1718,12 +1723,11 @@ async def get_global_spend_report( ORDER BY total_cost DESC; """ - db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj) - if db_response is None: - return [] - + db_response = await _query_spend_logs(prisma_client, sql_query, start_date_obj, end_date_obj) return db_response + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -1967,7 +1971,7 @@ async def get_key_spend_report( caller_value=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), scope_name="api_key", ) - db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + db_response: Sequence[Mapping[str, object]] = await _query_spend_logs( prisma_client, _scoped_spend_report_sql(scope_column="api_key"), start_date_obj, @@ -2009,7 +2013,7 @@ async def get_user_spend_report( caller_value=user_api_key_dict.user_id, scope_name="internal_user_id", ) - db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + db_response: Sequence[Mapping[str, object]] = await _query_spend_logs( prisma_client, _scoped_spend_report_sql(scope_column="user"), start_date_obj, @@ -2051,7 +2055,7 @@ async def get_team_spend_report( caller_value=user_api_key_dict.team_id, scope_name="team_id", ) - db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + db_response: Sequence[Mapping[str, object]] = await _query_spend_logs( prisma_client, _scoped_spend_report_sql(scope_column="team_id"), start_date_obj, @@ -2093,7 +2097,7 @@ async def get_organization_spend_report( organization_id=organization_id, prisma_client=prisma_client, ) - db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none( + db_response: Sequence[Mapping[str, object]] = await _query_spend_logs( prisma_client, _ORG_SPEND_REPORT_SQL, start_date_obj, @@ -2948,23 +2952,6 @@ async def ui_view_spend_logs( sql_params.append(json.dumps(used_client_oauth_token)) p += 1 - if status_filter is not None and group_by_session is True and not is_search_lookup: - session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" - sql_conditions.append( - f"""({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT session_key, api_key FROM ( - SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) - {_SESSION_KEY_EXPR} AS session_key, api_key, status - FROM "LiteLLM_SpendLogs" - WHERE {session_filter_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} - ) AS session_outcomes - WHERE COALESCE(status, 'success') = ${p} - )""" - ) - sql_params.append(status_filter) - p += 1 - if ( group_by_session is True and not is_v2 @@ -2977,12 +2964,30 @@ async def ui_view_spend_logs( sql_conditions=sql_conditions, sql_params=sql_params, next_param_index=p, + status_filter=status_filter, page=page, page_size=page_size, sort_desc=order_direction != "asc", session_cursor=session_cursor, ) + if status_filter is not None and group_by_session is True and not is_search_lookup: + session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" + sql_conditions.append( + f"""({_SESSION_GROUP_KEY_SQL}) IN ( + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {session_filter_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${p} + )""" + ) + sql_params.append(status_filter) + p += 1 + # Build the ORDER BY expression. ttft_ms is computed from # completionStartTime - startTime; non-streaming rows (where # completionStartTime is null or equals endTime) yield NULL, so we @@ -3017,7 +3022,7 @@ async def ui_view_spend_logs( LIMIT ${p} ) AS bounded_matches """ - count_rows: Final[Sequence[_SpendLogsCountRow] | None] = await _query_raw_or_none( + count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_spend_logs( prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 ) raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 @@ -3031,7 +3036,7 @@ async def ui_view_spend_logs( {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {joined_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id LIMIT ${p} OFFSET ${p + 1} @@ -3048,7 +3053,7 @@ async def ui_view_spend_logs( ) sql_params.extend([page_size, skip]) - data: Final[Sequence[Mapping[str, object]]] = await _query_raw(prisma_client, sql_query, *sql_params) + data: Final[Sequence[Mapping[str, object]]] = await _query_spend_logs(prisma_client, sql_query, *sql_params) if request_id is not None and not is_v2 and not is_admin_view: await _assert_user_owns_fetched_spend_rows( @@ -3080,22 +3085,6 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) -class _SessionPageRow(TypedDict): - session_key: ReadOnly[str] - api_key: ReadOnly[str] - last_activity: ReadOnly[str] - - -def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: - if session_cursor is None or session_cursor.count("|") < 2: - return None - last_activity, _, rest = session_cursor.partition("|") - api_key, _, session_key = rest.partition("|") - if not last_activity or not session_key: - return None - return (last_activity, session_key, api_key) - - async def _fetch_session_representatives( prisma_client: "PrismaClient", where_clause: str, @@ -3104,19 +3093,21 @@ async def _fetch_session_representatives( session_keys: Sequence[tuple[str, str]], ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + # Lets Postgres use the session_id and request_id indexes. rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where_clause} + AND (session_id = ANY(${next_param_index}::text[]) OR request_id = ANY(${next_param_index}::text[])) AND ({_SESSION_GROUP_KEY_SQL}) IN ( SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) ) - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives """ - rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place + rep_rows: Final[Sequence[dict[str, object]]] = await _query_spend_logs( # mutable-ok: rows are enriched in place prisma_client, rep_query, *sql_params, @@ -3129,117 +3120,32 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] -async def _count_grouped_sessions( - prisma_client: "PrismaClient", - where_clause: str, - sql_params: Sequence[object], - next_param_index: int, -) -> tuple[int, bool]: - """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - return ( - (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) - ) - - async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], sql_params: Sequence[object], next_param_index: int, + status_filter: str | None, page: int, page_size: int, sort_desc: bool, session_cursor: str | None, ) -> Mapping[str, object]: - """ - One row per session, keyset-paginated by session last activity. - - Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra - table): rows sharing a ``session_id`` and ``api_key`` form a session, rows - without a session id are singletons keyed by ``request_id``. A page is the - next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, - api_key)``, resumed from the ``session_cursor`` keyset - ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. A request for ``page > 1`` - without a cursor (the UI jumping straight to the last page, or back to a - page it never walked through) falls back to ``OFFSET (page - 1) * - page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` - window the capped ``total`` promises, so a page never runs past that total - and one starting at or past it returns no rows without a query. Each session is represented - by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` - exactly like the flat listing, and the response carries - ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). A page that runs out of sessions while still - holding some is itself the end of the list, so its ``total`` is - ``offset + len(page)`` and the grouped count query is skipped; a page that - starts past the end says nothing about the total, so that one is counted. - """ + """One row per session, keyset-paginated by last activity. ``read_session_page`` picks and counts the sessions, + and each is shown by its final agent outcome or newest non-MCP row.""" where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" - cmp_op: Final = "<" if sort_desc else ">" - direction: Final = "DESC" if sort_desc else "ASC" - - cursor: Final = _parse_session_cursor(session_cursor) - having_clause: Final = ( - f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} ' - f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})" - if cursor - else "" + session_page: Final = await read_session_page( + _SpendLogsReader(prisma_client), # pyright: ignore[reportArgumentType] # rows are the TypedDict the session SQL selects + where_clause=where_clause, + params=sql_params, + status=status_filter, + page=page, + page_size=page_size, + sort_desc=sort_desc, + session_cursor=session_cursor, + cap=SPEND_LOGS_PAGINATION_COUNT_CAP, ) - cursor_params: Final[tuple[object, ...]] = cursor if cursor else () - limit_index: Final = next_param_index + len(cursor_params) - offset: Final = (page - 1) * page_size if cursor is None else 0 - page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) - offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () - offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" - - page_query: Final = f""" - SELECT {_SESSION_KEY_EXPR} AS session_key, - api_key, - MAX("startTime")::text AS last_activity - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - {having_clause} - ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} {offset_clause} - """ - page_rows: Final[Sequence[_SessionPageRow]] = ( - () - if page_limit <= 0 - else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) - ) - - has_more: Final = len(page_rows) > page_limit - visible_rows: Final = page_rows[:page_limit] - next_cursor: Final = ( - f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" - if has_more and visible_rows - else None - ) - - page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 - page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list - total_records, total_is_capped = ( - (offset + len(page_rows), False) - if page_ends_the_list - else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) - ) - - session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) + session_keys: Final = tuple((session.key, session.api_key) for session in session_page.sessions) data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row await _fetch_session_representatives( prisma_client=prisma_client, @@ -3253,18 +3159,17 @@ async def _ui_session_grouped_spend_logs( ) _hydrate_spend_log_metadata(data) - total_pages: Final = (total_records + page_size - 1) // page_size response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response( prisma_client, data, - total_records, + session_page.total, page, page_size, - total_pages, + (session_page.total + page_size - 1) // page_size, enrich_session_counts=True, - total_is_capped=total_is_capped, + total_is_capped=session_page.total_is_capped, ) - return {**response, "next_session_cursor": next_cursor, "has_more": has_more} + return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more} class RequestResponsePayload(NamedTuple): @@ -4705,7 +4610,7 @@ async def _build_ui_spend_logs_response( if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) is not None } ) - rows: Final[Sequence[_SessionSpendRow]] = await _query_raw( + rows: Final[Sequence[_SessionSpendRow]] = await _query_spend_logs( prisma_client, f""" SELECT s.*, COALESCE(m.session_models, ARRAY[]::text[]) AS session_models @@ -4720,16 +4625,16 @@ async def _build_ui_spend_logs_response( ) ), 0)::bigint AS session_total_duration_ms, COUNT(*) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} )::int AS mcp_tool_call_count, COALESCE(SUM(spend) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} ), 0)::double precision AS mcp_tool_call_spend, COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count, COUNT(*) FILTER ( - WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} + WHERE call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} )::int AS session_llm_count, - COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count, + COUNT(*) FILTER (WHERE call_type = {AGENT_CALL_TYPE_SQL})::int AS session_agent_count, COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens, COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens, COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens @@ -4776,7 +4681,7 @@ async def _build_ui_spend_logs_response( if row.get("session_id") and row.get("api_key") is not None for models in (TypeAdapter(list[str]).validate_python(row.get("session_models") or ()),) } - except PrismaError: + except (PrismaError, HTTPException): verbose_proxy_logger.debug( "Failed to enrich session spend aggregates for spend logs UI", exc_info=True, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 8b0ff718fec..fc54d1da0b2 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -3,7 +3,7 @@ import json import os import re import secrets -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from datetime import datetime as dt from functools import reduce @@ -38,6 +38,12 @@ from litellm.constants import ( ) from litellm.integrations.otel.model.metadata import caller_session_id from litellm.integrations.otel.model.trace_controls import TraceControls +from litellm.litellm_core_utils.batch_utils import ( + BATCH_COST_REQUEST_ID_SUFFIX, + charged_poll_cost, + charged_video_id, + pays_only_its_guardrail, +) from litellm.litellm_core_utils.classifier_logging import ( CLASSIFIER_AUDIT_FIELDS, classifier_audit_fields, @@ -58,7 +64,6 @@ from litellm.litellm_core_utils.internal_call_metadata import ( ) from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, - UsageObservation, coerce_model_access_groups, is_valid_sha256_hash, request_model_access_groups_from_litellm_params, @@ -75,7 +80,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error -from litellm.proxy.utils import PrismaClient, hash_token +from litellm.proxy.utils import PrismaClient, hash_token, query_raw_within_statement_timeout from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, summary_only_compaction_content, @@ -99,6 +104,7 @@ from litellm.types.utils import ( VectorStoreSearchResponse, batch_chat_usage, ) +from litellm.types.videos.utils import VIDEO_COST_REQUEST_ID_SUFFIX from litellm.utils import get_end_user_id_for_cost_tracking if TYPE_CHECKING: @@ -155,9 +161,7 @@ def _is_non_secret_key_value(value: str, *, key_alias: object = None) -> bool: ) -def _redact_logged_api_key( - value: str | None, *, already_redacted: bool = False, key_alias: object = None -) -> str | None: +def redact_logged_api_key(value: str | None, *, already_redacted: bool = False, key_alias: object = None) -> str | None: if not isinstance(value, str) or not value: return None stripped: Final = re.sub(r"(?i)^bearer ", "", value) @@ -321,7 +325,7 @@ def _get_spend_logs_metadata( and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias) and _trusted_hash == _raw_key ) - clean_metadata["user_api_key"] = _redact_logged_api_key( + clean_metadata["user_api_key"] = redact_logged_api_key( _raw_key, already_redacted=_already_redacted, key_alias=_key_alias ) clean_metadata["applied_guardrails"] = applied_guardrails @@ -343,7 +347,7 @@ def _get_spend_logs_metadata( return clean_metadata -BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost" +JOB_POLL_GUARDRAIL_COST_REQUEST_ID_SUFFIX: Final = "_guardrail_cost" def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> str: @@ -371,6 +375,13 @@ def get_spend_logs_id( ) if resolved_id is None: return None + response_cost: Final = charged_poll_cost(kwargs) + charged_video: Final = charged_video_id(call_type, kwargs, resolved_id, response_cost) + if charged_video is not None: + return f"{charged_video}{VIDEO_COST_REQUEST_ID_SUFFIX}" + call_id: Final = kwargs.get("litellm_call_id") + if pays_only_its_guardrail(call_type, kwargs, response_cost) and isinstance(call_id, str) and call_id: + return f"{call_id}{JOB_POLL_GUARDRAIL_COST_REQUEST_ID_SUFFIX}" return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, kwargs)}" @@ -592,12 +603,12 @@ def get_logging_payload( start_time: datetime, end_time: datetime, llm_router: "Router | None" = None, - capture_usage: Callable[[SpendLogsPayload, str | None], None] | None = None, ) -> SpendLogsPayload: if kwargs is None: kwargs = {} rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError) + failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None response_served_from_cache: Final = served_from_cache(response_obj) if response_obj is None: response_obj = {} @@ -648,10 +659,12 @@ def get_logging_payload( usage = _usage # A request that failed mid-stream has no usable response_obj usage, and a stream - # continued on a fallback only has the fallback's. combined_usage_object holds the - # usage of every call the client's answer took, so it wins in both cases. + # continued on a fallback or a success that bills a failed follow-up only has its own. + # combined_usage_object holds the usage of every call the row bills, so it wins in each case. _combined_usage: Final = kwargs.get("combined_usage_object") - if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage): + if ( + not usage or "continued_partial_usage" in kwargs or "failed_attempts_billed_on_success" in kwargs + ) and isinstance(_combined_usage, litellm.Usage): usage = _combined_usage.model_dump() id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata) @@ -677,14 +690,14 @@ def get_logging_payload( and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias) and _trusted_hash == api_key ) - api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or "" + api_key = redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or "" if ( standard_logging_payload is not None ): # [TODO] migrate completely to sl payload. currently missing pass-through endpoint data api_key = ( api_key - or _redact_logged_api_key( + or redact_logged_api_key( standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True, key_alias=standard_logging_payload["metadata"].get("user_api_key_alias"), @@ -888,14 +901,11 @@ def get_logging_payload( additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values drop_requester_ip_address: Final = requester_ip_address_logging_disabled() - error_information: Final = clean_metadata["error_information"] if drop_requester_ip_address: clean_metadata["requester_ip_address"] = None - if drop_requester_ip_address and isinstance(error_information, Mapping): - clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked - StandardLoggingPayloadErrorInformation, - _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)), - ) + clean_metadata["error_information"] = sanitize_error_information_for_spend_logs( + clean_metadata["error_information"], original_exception=failure_exception + ) if litellm.cache is None: cache_key = "Cache OFF" @@ -982,14 +992,6 @@ def get_logging_payload( payload.get("spend"), ) - if capture_usage is not None: - observed_usage: Final = ( - StandardLoggingPayloadSetup.get_usage_as_dict({"usage": usage}) - if usage and kwargs.get("_accounting_usage_observed") is not UsageObservation.UNKNOWN - else None - ) - capture_usage(payload, safe_dumps(observed_usage) if observed_usage is not None else None) - # Explicitly clear large intermediate objects to reduce memory pressure del response_obj_dict, usage, clean_metadata, additional_usage_values @@ -1094,7 +1096,7 @@ async def _query_raw_rows( sql_query: str, *args: object, ) -> Sequence[Mapping[str, object]] | None: - return await prisma_client.db.query_raw(sql_query, *args) + return await query_raw_within_statement_timeout(prisma_client, sql_query, *args) async def get_spend_by_team( diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py deleted file mode 100644 index 19480a66678..00000000000 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ /dev/null @@ -1,1084 +0,0 @@ -from __future__ import annotations - -import asyncio -import json -import re -from collections.abc import Awaitable, Callable, Iterator, Mapping -from dataclasses import dataclass, replace -from datetime import datetime, timezone -from decimal import Decimal -from hashlib import sha256 -from itertools import chain, groupby, islice, repeat -from typing import TYPE_CHECKING, Final - -from pydantic import BaseModel, JsonValue, TypeAdapter - -from litellm._logging import verbose_proxy_logger -from litellm.accounting import ( - Dimension, - DimensionName, - InboundUsageEvent, - Metric, - MetricName, - OutboundUsageEvent, - Outcome, -) -from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY -from litellm.proxy.spend_tracking.usage_meter import ( - DurableUsageMeter, - UsageCaptureError, - UsageCaptureSettings, - usage_delivery_targets, -) -from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams - -if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy._types import SpendLogsPayload - - -from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite - - -class UsageAcknowledgementError(UsageCaptureError): - def __init__(self, cause: Exception) -> None: - super().__init__("Usage persistence did not acknowledge commit") - self.__cause__ = cause - - -@dataclass(frozen=True, slots=True) -class CapturedUsage: - payload: SpendLogsPayload - writes: tuple[UsageWrite, ...] - receipt: asyncio.Future[None] - - -_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) - - -_FIELDS: Final = TypeAdapter(dict[str, object]) - - -_INSTANT: Final = TypeAdapter(datetime) - - -_DIMENSIONS: Final = ( - ("api_key", DimensionName.KEY), - ("user", DimensionName.USER), - ("team_id", DimensionName.TEAM), - ("organization_id", DimensionName.ORGANIZATION), - ("end_user", DimensionName.END_USER), - ("model", DimensionName.MODEL), - ("model_group", DimensionName.MODEL_GROUP), - ("model_id", DimensionName.DEPLOYMENT), - ("custom_llm_provider", DimensionName.PROVIDER), - ("mcp_namespaced_tool_name", DimensionName.TOOL), - ("agent_id", DimensionName.AGENT), -) - - -_USAGE_METRICS: Final = tuple( - metric - for metric in MetricName - if metric not in (MetricName.SPEND, MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS) -) - - -def _fields(value: object) -> Mapping[str, object]: - return _FIELDS.validate_python(value) if isinstance(value, Mapping) else {} - - -def _text(value: object) -> str | None: - return value if isinstance(value, str) and value else None - - -def _utc_time(value: datetime | str) -> datetime: - parsed: Final = _INSTANT.validate_python(value) - return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) - - -def _json(value: object) -> str: - return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) - - -def _usage_metrics(usage: Mapping[str, JsonValue] | None) -> tuple[Metric, ...] | None: - if usage is None: - return None - return tuple( - Metric(name=metric, value=Decimal(str(value))) - for metric in _USAGE_METRICS - if isinstance(value := usage.get(metric.value), (int, float)) and not isinstance(value, bool) - ) - - -@dataclass(frozen=True, slots=True) -class UsageCaptureContext: - caller_id: str | None - attempt_id: str - parent_request_id: str | None - retry_of_attempt_id: str | None - job_id: str | None - charge_eligible: bool - request_count_eligible: bool - ownership: str - pricing: str - finalized_cost: str | None = None - allocation: str | None = None - outcome: Outcome | None = None - started_at: datetime | None = None - ended_at: datetime | None = None - phase: str = "execution" - extra_dimensions: tuple[Dimension, ...] = () - provider_timestamps: str = "{}" - usage_source: str = "measured" - included_attempt_ids: tuple[str, ...] = () - related_attempt_ids: tuple[str, ...] = () - - -def _database_usage_write(event: InboundUsageEvent | OutboundUsageEvent, envelope: Mapping[str, object]) -> UsageWrite: - from sys import maxsize - - from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure - - normalized: Final = _FIELDS.validate_python(safe_json_structure(envelope, max_depth=maxsize)) - if normalized.get("event") != event.model_dump(mode="json"): - raise UsageCaptureError("Usage event text cannot be normalized without changing identity or attribution") - serialized: Final = _json(normalized) - _JSON_OBJECT.validate_json(serialized) - return UsageWrite(event, serialized, usage_delivery_targets(event.dimensions)) - - -def usage_writes( - payload: SpendLogsPayload, usage_json: str | None, context: UsageCaptureContext -) -> tuple[UsageWrite, ...]: - fields: Final = _FIELDS.validate_python(payload) - tags: Final = TypeAdapter(tuple[str, ...]).validate_json(payload["request_tags"]) - dimensions: Final = ( - *(Dimension(name=name, value=value) for field, name in _DIMENSIONS if (value := _text(fields.get(field)))), - *(Dimension(name=DimensionName.TAG, value=tag) for tag in dict.fromkeys(tags)), - *context.extra_dimensions, - ) - outcome: Final = context.outcome or (Outcome.FAILED if payload["status"] == "failure" else Outcome.SUCCEEDED) - started: Final = context.started_at or _utc_time(payload["startTime"]) - ended: Final = context.ended_at or _utc_time(payload["endTime"]) - usage: Final = None if usage_json is None else _JSON_OBJECT.validate_json(usage_json) - outbound: Final = OutboundUsageEvent( - event_id=context.attempt_id, - attempt_id=context.attempt_id, - request_id=context.caller_id, - retry_of_attempt_id=context.retry_of_attempt_id, - call_type=CallTypes(payload["call_type"]), - started_at=started, - ended_at=ended, - outcome=outcome, - dimensions=dimensions, - metrics=_usage_metrics(None if context.usage_source == "unknown" else usage), - ) - envelope: Final = { - "event": outbound.model_dump(mode="json"), - "usage": usage, - "usage_source": "unknown" if usage is None else context.usage_source, - "included_attempt_ids": context.included_attempt_ids, - "related_attempt_ids": context.related_attempt_ids, - "pricing": _JSON_OBJECT.validate_json(context.pricing), - "charge_eligible": context.charge_eligible, - "request_count_eligible": context.request_count_eligible, - "ownership": context.ownership, - "parent_request_id": context.parent_request_id, - "job_id": context.job_id, - "phase": context.phase, - "provider_timestamps": _JSON_OBJECT.validate_json(context.provider_timestamps), - "finalized_cost": None - if context.finalized_cost is None - else _JSON_OBJECT.validate_json(context.finalized_cost), - "allocation": None if context.allocation is None else _JSON_OBJECT.validate_json(context.allocation), - } - outbound_write: Final = _database_usage_write(outbound, envelope) - if not context.request_count_eligible or context.caller_id is None: - return (outbound_write,) - inbound: Final = InboundUsageEvent( - event_id=f"{context.caller_id}:inbound", - request_id=context.caller_id, - call_type=outbound.call_type, - started_at=started, - ended_at=ended, - outcome=outcome, - dimensions=dimensions, - metrics=( - Metric(name=MetricName.REQUESTS, value=Decimal(1)), - Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(int(outcome is Outcome.SUCCEEDED))), - Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(int(outcome is not Outcome.SUCCEEDED))), - ), - ) - return ( - _database_usage_write(inbound, {"event": inbound.model_dump(mode="json")}), - outbound_write, - ) - - -_INCLUDED_IN_PARENT: Final = object() - - -_PRICING_FIELDS: Final = frozenset( - { - "service_tier", - "data_residency", - "vertex_location", - "region_name", - "base_model", - "custom_pricing", - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_second", - "output_cost_per_second", - "input_cost_per_character", - "output_cost_per_character", - "size", - "quality", - "n", - "duration", - "seconds", - "search_context_size", - "additional_response_cost", - } -) - - -def _selected_pricing(fields: Mapping[str, object]) -> Mapping[str, JsonValue]: - return _JSON_OBJECT.validate_python( - { - key: value - for key, value in fields.items() - if key in _PRICING_FIELDS or key in CustomPricingLiteLLMParams.model_fields - } - ) - - -def _response_fields(response: object) -> Mapping[str, object]: - if isinstance(response, BaseModel): - return _FIELDS.validate_python(response.model_dump()) - return _fields(response) - - -def _job_time(response: Mapping[str, object], key: str) -> datetime | None: - stamp: Final = response.get(key) - if isinstance(stamp, (int, float)) and not isinstance(stamp, bool): - return datetime.fromtimestamp(stamp, tz=timezone.utc) - return None - - -def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], response: object) -> UsageCaptureContext: - from dataclasses import asdict - - from litellm.litellm_core_utils.hidden_params import get_hidden_params - from litellm.litellm_core_utils.internal_call_metadata import is_background_response, is_unbilled_non_inference_call - from litellm.litellm_core_utils.litellm_logging import ( - Logging, - UsageCaptureObservation, - UsageObservation, - mask_api_base_credentials, - ) - from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost - from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation - from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups - - metadata: Final = _JSON_OBJECT.validate_json(payload["metadata"]) - params: Final = _fields(kwargs.get("litellm_params")) - request_metadata: Final = _fields(params.get("litellm_metadata") or params.get("metadata")) - standard: Final = _fields(kwargs.get("standard_logging_object")) - logging_obj: Final = kwargs.get("litellm_logging_obj") - raw_response: Final = _response_fields(response) - hidden: Final = _fields(get_hidden_params(response)) - tool: Final = _fields(kwargs.get("mcp_tool_call_metadata")) - tool_pricing: Final = _fields(tool.get("mcp_server_cost_info")) - caller: Final = payload["litellm_call_id"] - cached: Final = payload["cache_hit"].lower() == "true" - included_in_parent: Final = kwargs.get("_accounting_internal_aggregate") is _INCLUDED_IN_PARENT - billed_id: Final = payload["request_id"] - terminal: Final = raw_response.get("status") in { - "completed", - "complete", - "failed", - "cancelled", - "expired", - "incomplete", - } - deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal - management: Final = ( - (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal) - or (is_background_response(response) and not terminal) - or is_unbilled_non_inference_call(payload["call_type"], request_metadata, response) - ) - attempt: Final = ( - f"{caller}_cache_hit" - if cached and caller - else f"{caller}:status" - if management - else f"{billed_id}:included" - if included_in_parent - else billed_id - ) - if not attempt or attempt == "None" or (cached and not caller): - raise UsageCaptureError("Usage capture requires a stable execution identity") - retry: Final = metadata.get(BILLED_RETRY_ATTEMPT_METADATA_KEY) is True - origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) - parent: Final = _text(kwargs.get("parent_request_id")) or _text(request_metadata.get("parent_request_id")) - finalized_value: Final = ( - logging_obj.finalized_cost if isinstance(logging_obj, Logging) else kwargs.get("_accounting_finalized_cost") - ) - finalized: Final = finalized_value if isinstance(finalized_value, FinalizedCost) else None - allocation_value: Final = ( - logging_obj.cost_allowance_allocation - if isinstance(logging_obj, Logging) - else kwargs.get("_accounting_allocation") - ) - allocation: Final = allocation_value if isinstance(allocation_value, SearchAllowanceAllocation) else None - pricing_snapshot: Final = kwargs.get("_accounting_pricing_snapshot") - observation: Final = kwargs.get("_accounting_usage_snapshot") - captured_pricing: Final = { - "model_map_information": ( - _JSON_OBJECT.validate_json(pricing_snapshot.model_map_information) - if isinstance(pricing_snapshot, PricingCapture) - else metadata.get("model_map_information") - ), - "cost_breakdown": metadata.get("cost_breakdown"), - **( - _JSON_OBJECT.validate_json(pricing_snapshot.inputs) - if isinstance(pricing_snapshot, PricingCapture) - else _pricing_inputs(kwargs, logging_obj, response) - ), - "tool": { - "name": tool.get("name"), - "cost_per_query": _fields(tool_pricing.get("tool_name_to_cost_per_query")).get( - _text(tool.get("name")) or "", tool_pricing.get("default_cost_per_query") - ), - } - if tool - else None, - "response_inputs": { - "audio_transcription_duration": hidden.get("audio_transcription_duration", raw_response.get("duration")), - "billed_units": _fields(raw_response.get("meta")).get("billed_units"), - "batch_models": hidden.get("batch_models"), - "batch_successful_requests": hidden.get("batch_successful_requests"), - "batch_failed_requests": hidden.get("batch_failed_requests"), - "service_tier": raw_response.get("service_tier"), - "region_name": hidden.get("region_name"), - }, - "reported_cost": observation.response_cost - if isinstance(observation, UsageCaptureObservation) - else standard.get( - "response_cost", kwargs.get("response_cost", _fields(get_hidden_params(response)).get("response_cost")) - ), - "adjustments": ( - _JSON_OBJECT.validate_json(pricing_snapshot.adjustments) - if isinstance(pricing_snapshot, PricingCapture) - else None - ), - } - return UsageCaptureContext( - caller_id=None if deferred else parent if retry or included_in_parent else caller, - attempt_id=attempt, - parent_request_id=parent, - retry_of_attempt_id=_text(request_metadata.get("retry_of_attempt_id")), - job_id=_text(raw_response.get("id")) if deferred or management else None, - charge_eligible=not cached and not included_in_parent and not management, - request_count_eligible=bool(caller) and not origin and not retry and not deferred and not included_in_parent, - ownership="included_in_parent" - if included_in_parent - else "aggregate" - if kwargs.get("additional_response_cost") is not None - else "attempt", - pricing=_json(captured_pricing), - finalized_cost=_json( - { - "response_cost": finalized.amount, - "cost_breakdown": finalized.logging_breakdown(), - "allowance_discount": finalized.allowance_discount, - } - ) - if finalized is not None - else ( - _json({"response_cost": standard.get("response_cost"), "cost_breakdown": standard.get("cost_breakdown")}) - if standard.get("cost_finalized") is True - else None - ), - allocation=_json(asdict(allocation)) if allocation is not None else None, - started_at=_job_time(raw_response, "created_at") if deferred else None, - ended_at=( - _job_time(raw_response, "completed_at") - or _job_time(raw_response, "failed_at") - or _job_time(raw_response, "cancelled_at") - or _job_time(raw_response, "expired_at") - ) - if deferred - else None, - outcome=( - Outcome.CANCELLED - if raw_response.get("status") == "cancelled" - or isinstance(response, asyncio.CancelledError) - or getattr(response, "status_code", None) == 499 - else Outcome.FAILED - if raw_response.get("status") in {"failed", "expired", "incomplete"} or isinstance(response, BaseException) - else None - ), - phase="terminal" if deferred else "management" if management else "execution", - usage_source="estimated" - if kwargs.get("_accounting_usage_observed") is UsageObservation.ESTIMATED - else "measured", - included_attempt_ids=TypeAdapter(tuple[str, ...]).validate_python( - kwargs.get("_accounting_included_attempt_ids", ()) - ), - related_attempt_ids=tuple(write.event.event_id for write in _prior_writes(kwargs)), - provider_timestamps=_json( - { - key: raw_response.get(key) - for key in ("created_at", "completed_at", "failed_at", "cancelled_at", "expired_at") - if raw_response.get(key) is not None - } - ) - if deferred - else "{}", - extra_dimensions=( - *( - (Dimension(name=DimensionName.TOOL, value=tool_name),) - if not payload.get("mcp_namespaced_tool_name") - and (tool_name := _text(tool.get("namespaced_tool_name"))) - else () - ), - *( - (Dimension(name=DimensionName.API_BASE, value=mask_api_base_credentials(api_base)),) - if (api_base := _text(payload["api_base"])) - else () - ), - *( - Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value=group) - for group in get_request_model_access_groups(kwargs) - ), - *( - (Dimension(name=DimensionName.PROJECT, value=project),) - if (project := _text(request_metadata.get("user_api_key_project_id"))) - else () - ), - *( - (Dimension(name=DimensionName.SERVICE_TIER, value=tier),) - if (tier := _text(raw_response.get("service_tier") or kwargs.get("service_tier"))) - else () - ), - ), - ) - - -class _CaptureBuilder: - def __init__(self, kwargs: Mapping[str, object], response: object) -> None: - self.kwargs: Final = kwargs - self.response: Final = response - self.writes: tuple[UsageWrite, ...] = () - - def observe(self, payload: SpendLogsPayload, usage: str | None) -> None: - from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation, UsageObservation - - snapshot: Final = self.kwargs.get("_accounting_usage_snapshot") - if isinstance(snapshot, UsageCaptureError): - raise snapshot - effective: Final = snapshot.usage_json if isinstance(snapshot, UsageCaptureObservation) else usage - parsed: Final = _JSON_OBJECT.validate_json(effective) if effective is not None else {} - batch: Final = _fields(parsed.get("batch_usage_by_model")) - unknown: Final = (bool(batch) or bool(parsed.get("batch_unknown_usage_requests"))) and all( - value is None for value in batch.values() - ) - observed: Final = ( - _json( - { - "batch_usage_by_model": batch, - "batch_unknown_usage_requests": parsed.get("batch_unknown_usage_requests"), - **( - {"batch_usage_by_request": parsed["batch_usage_by_request"]} - if "batch_usage_by_request" in parsed - else {} - ), - } - ) - if unknown - else effective - ) - context: Final = capture_context(payload, self.kwargs, self.response) - provenance: Final = ( - "unknown" - if unknown - else "estimated" - if isinstance(snapshot, UsageCaptureObservation) and snapshot.provenance is UsageObservation.ESTIMATED - else context.usage_source - ) - self.writes = usage_writes(payload, observed, replace(context, usage_source=provenance)) - - -def _prior_captures(kwargs: Mapping[str, object]) -> tuple[CapturedUsage, ...]: - prior: Final = kwargs.get("_accounting_prior_captures", ()) - return ( - tuple( - item for item in TypeAdapter(tuple[object, ...]).validate_python(prior) if isinstance(item, CapturedUsage) - ) - if isinstance(prior, tuple) - else () - ) - - -def _prior_writes(kwargs: Mapping[str, object]) -> tuple[UsageWrite, ...]: - return tuple(chain.from_iterable(item.writes for item in _prior_captures(kwargs))) - - -def capture_callback( - meter: DurableUsageMeter, - kwargs: Mapping[str, object], - response: object, - start_time: datetime, - end_time: datetime, -) -> CapturedUsage: - from litellm.proxy.db.db_spend_update_writer import get_llm_router - from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - - existing: Final = kwargs.get("_accounting_capture") - if isinstance(existing, CapturedUsage): - return existing - builder: Final = _CaptureBuilder(kwargs, response) - payload: Final = get_logging_payload( - dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe - ) - prerequisites: Final = tuple(item.receipt for item in _prior_captures(kwargs)) - return CapturedUsage(payload, builder.writes, meter.submit(builder.writes, prerequisites=prerequisites)) - - -def _batch_rows(response: object) -> tuple[Mapping[str, JsonValue], ...]: - usage: Final = _response_fields(_response_fields(response).get("usage")) - rows: Final = usage.get("batch_usage_by_request") - return TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(rows) if rows else () - - -def _batch_root(write: UsageWrite, rows: tuple[Mapping[str, JsonValue], ...]) -> UsageWrite: - event: Final = write.event.model_copy(update={"metrics": None}) - envelope: Final = _JSON_OBJECT.validate_json(write.envelope) - return _database_usage_write( - event, - { - **envelope, - "event": event.model_dump(mode="json"), - "usage": { - "batch_row_count": len(rows), - "batch_rows_sha256": sha256(_json(rows).encode()).hexdigest(), - "batch_charge_eligible": envelope["charge_eligible"], - }, - "usage_source": "unknown", - "charge_eligible": False, - "ownership": "aggregate", - }, - ) - - -def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int) -> UsageWrite: - envelope: Final = _JSON_OBJECT.validate_json(root.envelope) - pricing: Final = _JSON_OBJECT.validate_python(envelope["pricing"]) - model: Final = _text(row["model"]) - usage: Final = _JSON_OBJECT.validate_python(row["usage"]) if row["usage"] is not None else None - identity: Final = f"{root.event.event_id}:row:{index}" - event: Final = OutboundUsageEvent.model_validate( - { - **root.event.model_dump(), - "event_id": identity, - "attempt_id": identity, - "metrics": _usage_metrics(usage), - "dimensions": ( - *(dimension for dimension in root.event.dimensions if dimension.name is not DimensionName.MODEL), - *((Dimension(name=DimensionName.MODEL, value=model),) if model else ()), - ), - } - ) - return _database_usage_write( - event, - { - **envelope, - "event": event.model_dump(mode="json"), - "usage": usage, - "usage_source": "unknown" if usage is None else "measured", - "charge_eligible": _fields(envelope["usage"])["batch_charge_eligible"], - "ownership": "attempt", - "parent_request_id": root.event.event_id, - "finalized_cost": None, - "allocation": None, - "pricing": { - **pricing, - "model_map_information": _fields(pricing.get("batch_models")).get(model or ""), - "batch_models": {}, - "reported_cost": None, - "cost_breakdown": None, - "response_inputs": { - key: value - for key, value in _fields(pricing.get("response_inputs")).items() - if key not in {"batch_models", "batch_successful_requests"} - }, - }, - }, - ) - - -@dataclass(frozen=True, slots=True) -class _BatchCapture: - payload: SpendLogsPayload - root: UsageWrite - rows: bytes - prerequisites: tuple[asyncio.Future[None], ...] - - -def _prepare_batch_capture( - kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime -) -> _BatchCapture: - from litellm.proxy.db.db_spend_update_writer import get_llm_router - from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - - rows: Final = tuple(sorted(_batch_rows(response), key=_json)) - builder: Final = _CaptureBuilder(kwargs, response) - payload: Final = get_logging_payload( - dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe - ) - if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None: - raise UsageCaptureError("Batch rows require one terminal job observation") - return _BatchCapture( - payload, - _batch_root(builder.writes[0], rows), - b"\n".join(_json((row, sum(1 for _ in group))).encode("utf-8") for row, group in groupby(map(_json, rows))), - tuple(prior.receipt for prior in _prior_captures(kwargs)), - ) - - -def _batch_groups(rows: bytes) -> Iterator[tuple[str, int]]: - return (TypeAdapter(tuple[str, int]).validate_json(match.group()) for match in re.finditer(rb"[^\n]+", rows)) - - -def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]: - expanded: Final = chain.from_iterable(repeat(row, count) for row, count in _batch_groups(rows)) - return (_batch_row_write(root, _JSON_OBJECT.validate_json(row), index) for index, row in enumerate(expanded)) - - -def _batch_write_bound(root: UsageWrite, rows: bytes) -> int: - last_index: Final = sum(count for _, count in _batch_groups(rows)) - 1 - return max( - len(root.envelope.encode("utf-8")), - max( - len(_batch_row_write(root, _JSON_OBJECT.validate_json(row), last_index).envelope.encode("utf-8")) - for row, _ in _batch_groups(rows) - ), - ) - - -def _batch_capacity( - meter: DurableUsageMeter, root: UsageWrite, rows: bytes, *, extra_bytes: int = 0 -) -> tuple[int, int, int]: - retained: Final = len(rows) + len(root.envelope.encode("utf-8")) + extra_bytes - bound: Final = _batch_write_bound(root, rows) - chunk_bytes: Final = max( - bound, - min( - bound * min(meter.settings.batch_size, sum(count for _, count in _batch_groups(rows))), - meter.settings.max_pending_bytes // 4, - (meter.settings.max_pending_bytes - retained) // 2, - ), - ) - return retained + chunk_bytes, chunk_bytes, bound - - -async def _persist_batch_capture( - meter: DurableUsageMeter, - load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]], - root: UsageWrite, - rows: bytes, - prerequisites: tuple[asyncio.Future[None], ...], - chunk_bytes: int, - reservation: object | None = None, -) -> None: - for receipt in prerequisites: - await asyncio.shield(receipt) - await meter.record_writes((root,)) - accepted: Final = await load((root.event.event_id,)) - if len(accepted) != 1: - raise UsageCaptureError("Committed batch snapshot could not be loaded") - frozen: Final = _reuse_terminal_snapshot(root, accepted[0]) - retained, capacity, bound = _batch_capacity(meter, frozen, rows, extra_bytes=len(root.envelope.encode("utf-8"))) - if reservation is not None: - meter.resize_producer(reservation, retained, capacity) - write_capacity: Final = capacity if reservation is not None else chunk_bytes - if bound > write_capacity: - raise UsageCaptureError("Batch observation exceeds reserved producer write capacity") - group_size: Final = min(meter.settings.batch_size, max(1, write_capacity // bound)) - remaining: Final = _batch_writes(frozen, rows) - while group := tuple(islice(remaining, group_size)): - await meter.record_writes(group) - - -async def capture_batch_callback( - meter: DurableUsageMeter, - load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None, - kwargs: Mapping[str, object], - response: object, - start_time: datetime, - end_time: datetime, -) -> CapturedUsage: - if load is None: - raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader") - capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) - await _persist_batch_capture( - meter, load, capture.root, capture.rows, capture.prerequisites, meter.settings.max_pending_bytes - ) - receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() - receipt.set_result(None) - return CapturedUsage(capture.payload, (capture.root,), receipt) - - -async def captured_callback_kwargs( - kwargs: Mapping[str, object], response: object, start_time: datetime | None, end_time: datetime | None -) -> Mapping[str, object]: - from litellm.proxy.proxy_server import usage_capture - - usage_meter: Final = usage_capture.meter - - if usage_meter is None: - return kwargs - if start_time is None or end_time is None: - verbose_proxy_logger.error("Usage capture requires callback start and end timestamps") - return kwargs - try: - captured: Final = ( - usage_capture.submit_batch(kwargs, response, start_time, end_time) - if _batch_rows(response) - else capture_callback(usage_meter, kwargs, response, start_time, end_time) - ) - except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing - verbose_proxy_logger.exception("Usage capture failed before legacy cost tracking") - return kwargs - return {**kwargs, "_accounting_capture": captured} - - -async def capture_proxy_response(logging_obj: object, response: object, *, background: bool = False) -> None: - from fastapi import HTTPException - - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.proxy_server import general_settings, usage_capture - - settings: Final = _FIELDS.validate_python(general_settings) - usage_meter: Final = usage_capture.meter - - if usage_meter is None or not isinstance(logging_obj, Logging): - return - sync_ack: Final = ( - settings.get("accounting_sync_ack") is True - and logging_obj.stream is not True - and not background - and logging_obj.call_type in {"acompletion", "atext_completion", "aembedding", "aresponses"} - ) - try: - capture_logging_usage(usage_meter, logging_obj, response, logging_obj.start_time, datetime.now(timezone.utc)) - captured: Final = logging_obj.model_call_details.get("_accounting_capture") - if sync_ack: - if not isinstance(captured, CapturedUsage): - raise UsageCaptureError("Usage capture was not admitted") - timeout: Final = TypeAdapter(float).validate_python( - settings.get("accounting_sync_ack_timeout_seconds", 20.0) - ) - await asyncio.wait_for(asyncio.shield(captured.receipt), timeout=timeout) - except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure - verbose_proxy_logger.exception("Usage capture failed before the response boundary") - if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed": - acknowledgement: Final = UsageAcknowledgementError(error) - raise HTTPException(503, str(acknowledgement)) from acknowledgement - - -def capture_logging_usage( - meter: DurableUsageMeter, - logging_obj: object, - response: object, - start_time: datetime, - end_time: datetime, - freeze_only: bool = False, -) -> CapturedUsage | UsageCaptureError | None: - from litellm._internal_context import internal_parent_request_id, is_internal_call - from litellm.batches.batch_utils import batch_cost_is_final - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import LiteLLMBatch - - if not isinstance(logging_obj, Logging): - return - internal: Final = is_internal_call.get() - if isinstance(response, LiteLLMBatch) and batch_cost_is_final(response) and response.usage is None: - return - try: - if not internal and isinstance(logging_obj.model_call_details.get("_accounting_capture"), CapturedUsage): - return capture_callback(meter, logging_obj.model_call_details, response, start_time, end_time) - normalized_result: Final = logging_obj.usage_capture_response(response) - prepared: Final = logging_obj.prepared_cost - normalized: Final = ( - {**_response_fields(normalized_result), "usage": prepared.usage.model_dump()} - if prepared is not None - else normalized_result - ) - if normalized is None: - return - details: Final = _FIELDS.validate_python(logging_obj.model_call_details) - params: Final = _fields(details.get("litellm_params")) - metadata: Final = _fields(params.get("litellm_metadata") or params.get("metadata")) - if metadata.get("batch_ignore_default_logging") is True: - return - existing_pricing: Final = details.get("_accounting_pricing_snapshot") - pricing: Final = ( - existing_pricing - if isinstance(existing_pricing, PricingCapture) and not internal - else _capture_pricing(logging_obj, normalized, details) - ) - if freeze_only or _batch_rows(normalized): - if not internal: - logging_obj.record_usage_pricing(pricing) - return - captured: Final = capture_callback( - meter, - { - **details, - "litellm_logging_obj": logging_obj, - "_accounting_capture": None if internal else details.get("_accounting_capture"), - "_accounting_pricing_snapshot": pricing, - "_accounting_internal_aggregate": _INCLUDED_IN_PARENT if internal else None, - "parent_request_id": internal_parent_request_id.get() if internal else details.get("parent_request_id"), - }, - normalized, - start_time, - end_time, - ) - if not internal: - logging_obj.record_usage_capture(captured) - except Exception as error: # noqa: BLE001 # failed capture must not prevent normal logging and legacy charging - verbose_proxy_logger.exception("Usage capture failed at the execution boundary") - return UsageCaptureError(str(error)) - else: - return captured - - -@dataclass(frozen=True, slots=True) -class PricingCapture: - model_map_information: str - adjustments: str - inputs: str - - -def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture: - import litellm - from litellm.litellm_core_utils.cost_discount import resolve_cost_discount - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - params: Final = _fields(details.get("litellm_params")) - prepared: Final = logging_obj.prepared_cost - provider: Final = _text(details.get("custom_llm_provider")) - builtins: Final = logging_obj.standard_built_in_tools_params - pricing: Final = ( - prepared.payload["model_map_information"] - if prepared is not None - else StandardLoggingPayloadSetup.get_model_cost_information( - base_model=logging_obj.get_deployment_model_for_cost(), - custom_pricing=bool(params.get("custom_pricing")), - custom_llm_provider=_text(details.get("custom_llm_provider")), - init_response_obj=normalized, - api_base=_text(params.get("api_base")), - router_model_id=logging_obj.get_router_model_id(), - ) - ) - adjustments: Final = _json( - { - "discount": resolve_cost_discount( - litellm.cost_discount_config, - provider, - logging_obj.get_deployment_model_for_cost(), - _text(params.get("region_name") or params.get("vertex_location")), - ), - "margin": litellm.cost_margin_config.get(provider or "", litellm.cost_margin_config.get("global")), - "web_search_options": ( - _selected_pricing(_fields(builtins.get("web_search_options"))) - if builtins.get("web_search_options") is not None - else None - ), - "file_search": builtins.get("file_search") is not None, - } - ) - batch_usage: Final = _fields(_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model")) - return PricingCapture( - _json(pricing), - adjustments, - _json( - { - **_pricing_inputs(details, logging_obj, normalized), - "batch_models": { - model: StandardLoggingPayloadSetup.get_model_cost_information( - base_model=model, - custom_pricing=bool(params.get("custom_pricing")), - custom_llm_provider=provider, - init_response_obj={"model": model}, - api_base=_text(params.get("api_base")), - router_model_id=logging_obj.get_router_model_id(), - ) - for model in batch_usage - }, - } - ), - ) - - -def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]: - from litellm.litellm_core_utils.litellm_logging import Logging - - return { - "request": _selected_pricing(kwargs), - "deployment": _selected_pricing(_fields(kwargs.get("litellm_params"))), - "optional_params": _selected_pricing(_fields(kwargs.get("optional_params"))), - "logging": logging_obj.usage_capture_pricing(response) if isinstance(logging_obj, Logging) else None, - "model_info": _selected_pricing( - _fields( - _fields( - _fields(kwargs.get("litellm_params")).get("litellm_metadata") - or _fields(kwargs.get("litellm_params")).get("metadata") - ).get("model_info") - ) - ), - } - - -def _terminal_facts(write: UsageWrite) -> Mapping[str, JsonValue] | None: - payload: Final = _JSON_OBJECT.validate_json(write.envelope) - if payload.get("phase") != "terminal": - return None - event: Final = _JSON_OBJECT.validate_python(payload["event"]) - return { - **{ - key: value - for key, value in payload.items() - if key not in {"event", "pricing", "finalized_cost", "allocation"} - }, - "event": {key: value for key, value in event.items() if key not in {"started_at", "ended_at"}}, - } - - -def _reuse_terminal_snapshot(candidate: UsageWrite, accepted: UsageWrite) -> UsageWrite: - facts: Final = _terminal_facts(candidate) - if facts is None or facts != _terminal_facts(accepted) or set(candidate.targets) != set(accepted.targets): - raise AccountingConflictError("Repeated terminal job disagrees with its accepted usage or attribution") - return accepted - - -class UsageBatchWriter: - def __init__( - self, - insert: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], - load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]], - ) -> None: - self._insert: Final = insert - self._load: Final = load - - async def _restore(self, writes: tuple[UsageWrite, ...]) -> tuple[UsageWrite, ...]: - terminal: Final = tuple(write for write in writes if _terminal_facts(write) is not None) - if not terminal: - return writes - identities: Final = tuple(dict.fromkeys(write.event.event_id for write in terminal)) - stored: Final = await self._load(identities) - # Re-read the first accepted observation and pricing before the ledger's unchanged id/hash check. - snapshots: Final = {write.event.event_id: write for write in (*reversed(terminal), *stored)} - return tuple( - _reuse_terminal_snapshot(write, snapshots[write.event.event_id]) - if write.event.event_id in snapshots - else write - for write in writes - ) - - async def write(self, writes: tuple[UsageWrite, ...]) -> None: - restored: Final = await self._restore(writes) - try: - await self._insert(restored) - except AccountingConflictError: - if not any(_terminal_facts(write) is not None for write in writes): - raise - await self._insert(await self._restore(writes)) - - -class UsageCaptureRuntime: - def __init__(self) -> None: - self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle - self._stopping = False - self.meter: DurableUsageMeter | None = None - self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None - - def start( - self, - write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], - settings: UsageCaptureSettings, - *, - load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None, - ) -> None: - from litellm.litellm_core_utils.litellm_logging import Logging - - if self.meter is not None: - if self.meter.settings != settings: - raise ValueError("Changing accounting_capture requires a worker restart") - return - meter: Final = DurableUsageMeter(write_batch, settings) - self.meter = meter - self.load = load - self._stopping = False - Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage( - meter, logger, response, start, end, freeze - ) - - def submit_batch( - self, kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime - ) -> CapturedUsage: - meter: Final = self.meter - load: Final = self.load - if meter is None or load is None or self._stopping: - raise UsageCaptureError("Batch capture requires an active ledger writer and snapshot reader") - if len(self._batches) >= meter.settings.max_pending_records: - raise UsageCaptureError("Batch capture producer capacity exhausted") - capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) - retained, chunk_bytes, _ = _batch_capacity( - meter, capture.root, capture.rows, extra_bytes=len(capture.root.envelope.encode("utf-8")) - ) - reservation: Final = meter.reserve_producer(retained, chunk_bytes) - task: Final = asyncio.create_task( - _persist_batch_capture( - meter, load, capture.root, capture.rows, capture.prerequisites, chunk_bytes, reservation - ), - name="accounting batch capture", - ) - task.add_done_callback(lambda _: meter.release_producer(reservation)) - self._batches.add(task) - task.add_done_callback(self._batch_finished) - return CapturedUsage(capture.payload, (capture.root,), task) - - def _batch_finished(self, task: asyncio.Task[None]) -> None: - self._batches.discard(task) - if task.cancelled(): - verbose_proxy_logger.error("Batch usage capture stopped before complete persistence") - elif (error := task.exception()) is not None: - verbose_proxy_logger.error("Batch usage capture failed", exc_info=error) - - async def stop(self, timeout: float) -> None: - from litellm.litellm_core_utils.litellm_logging import Logging - - if self.meter is None: - return - self._stopping = True - deadline: Final = asyncio.get_running_loop().time() + timeout - try: - if self._batches: - await asyncio.wait(tuple(self._batches), timeout=max(0, timeout)) - finally: - remaining: Final = tuple(self._batches) - for task in remaining: - task.cancel() - await asyncio.gather(*remaining, return_exceptions=True) - try: - await self.meter.stop(max(0, deadline - asyncio.get_running_loop().time())) - finally: - Logging.usage_capture_callback = None - self.meter = None - self.load = None diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py deleted file mode 100644 index 431e4c2c3c3..00000000000 --- a/litellm/proxy/spend_tracking/usage_meter.py +++ /dev/null @@ -1,330 +0,0 @@ -from __future__ import annotations - -import asyncio -from collections import deque -from collections.abc import Awaitable, Callable, Mapping -from concurrent.futures import Future -from dataclasses import dataclass -from itertools import accumulate, chain, count, takewhile -from threading import Lock -from typing import Final - -from pydantic import BaseModel, ConfigDict, Field - -from litellm._logging import verbose_proxy_logger -from litellm.accounting import ( - DeliveryTarget, - Dimension, - DimensionName, - InboundUsageEvent, - OutboundUsageEvent, -) -from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite - -_DAILY_TARGETS: Final = ( - (DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND), - (DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND), - (DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND), - (DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND), - (DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND), - (DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND), - (DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND), - (DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE), - (DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND), -) - - -def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]: - names: Final = frozenset(dimension.name for dimension in dimensions) - return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names)) - - -class UsageCaptureSettings(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - batch_size: int = Field(default=1000, ge=2, le=1000) - flush_interval_seconds: float = Field(default=5.0, gt=0, le=10) - max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024) - max_pending_records: int = Field(default=10000, ge=2) - - -class UsageCaptureError(RuntimeError): - pass - - -@dataclass(frozen=True, slots=True) -class _Pending: - writes: tuple[UsageWrite, ...] - size: int - committed: Future[None] - finished: asyncio.Event - prerequisites: tuple[asyncio.Future[None], ...] - - -async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None: - try: - await asyncio.wait_for(wakeup.wait(), timeout) - except asyncio.TimeoutError: - return - - -def _report_failure(receipt: Future[None]) -> None: - if not receipt.cancelled() and receipt.exception() is not None: - verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception()) - - -def _retryable_persistence_failure(error: Exception) -> bool: - from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain - - outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error) - return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any( - isinstance(cause, Exception) - and ( - PrismaDBExceptionHandler.is_deadlock_error(cause) - or PrismaDBExceptionHandler.is_lock_timeout_error(cause) - or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001" - ) - for cause in exception_chain(error) - ) - - -def _isolatable_persistence_failure(error: Exception) -> bool: - from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain - - return isinstance(error, AccountingConflictError) or any( - isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22") - for cause in exception_chain(error) - ) - - -class DurableUsageMeter: - def __init__( - self, - write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], - settings: UsageCaptureSettings, - *, - wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch, - retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep, - ) -> None: - self._write_batch: Final = write_batch - self.settings: Final = settings - self._wait_for_batch: Final = wait_for_batch - self._retry_delay: Final = retry_delay - self._loop: Final = asyncio.get_running_loop() - self._admission: Final = Lock() - self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue - self._wakeup: Final = asyncio.Event() - self._capacity_changed: Final = asyncio.Event() - self._reservations: Mapping[object, tuple[int, int]] = {} - self._reserved_bytes = 0 - self._task: asyncio.Task[None] | None = None - self._closed = False - self.pending_bytes = 0 - self.pending_records = 0 - - def submit( - self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = () - ) -> asyncio.Future[None]: - size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) - with self._admission: - if ( - self._closed - or not writes - or len(writes) > self.settings.batch_size - or self.pending_bytes + size > self.settings.max_pending_bytes - or self.pending_records + len(writes) > self.settings.max_pending_records - ): - raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted") - receipt: Final = Future[None]() - receipt.set_running_or_notify_cancel() - receipt.add_done_callback(_report_failure) - self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites)) - self.pending_bytes += size - self.pending_records += len(writes) - self._loop.call_soon_threadsafe(self._ensure_worker) - wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop) - wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception()) - return wrapped - - def reserve_producer(self, size: int, headroom: int) -> object: - token: Final = object() - self.resize_producer(token, size, headroom) - return token - - def resize_producer(self, token: object, size: int, headroom: int) -> None: - with self._admission: - previous: Final = self._reservations.get(token, (0, 0))[0] - delta: Final = size - previous - others: Final = (value[1] for key, value in self._reservations.items() if key is not token) - required: Final = max((headroom, *others)) - if ( - self._closed - or self.pending_bytes + delta > self.settings.max_pending_bytes - or self._reserved_bytes + delta + required > self.settings.max_pending_bytes - ): - raise UsageCaptureError("Batch capture producer byte capacity exhausted") - self._reservations = {**self._reservations, token: (size, headroom)} - self._reserved_bytes += delta - self.pending_bytes += delta - self._capacity_changed.set() - - def release_producer(self, token: object) -> None: - with self._admission: - size, _ = self._reservations[token] - self._reservations = {key: value for key, value in self._reservations.items() if key is not token} - self._reserved_bytes -= size - self.pending_bytes -= size - self._capacity_changed.set() - - def _ensure_worker(self) -> None: - if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites: - self._wakeup.set() - if self._task is None or self._task.done(): - self._task = asyncio.create_task(self._run(), name="accounting usage capture") - - async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None: - envelope: Final = '{"event":' + event.model_dump_json() + "}" - await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),))) - - async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None: - size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) - capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes - if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or ( - size > capacity and len(writes) > 1 - ): - middle: Final = len(writes) // 2 - await self.record_writes(writes[:middle]) - await self.record_writes(writes[middle:]) - return - if size > capacity: - raise UsageCaptureError("One usage observation exceeds capture capacity") - while (receipt := self._submit_available(writes)) is None: - self._wakeup.set() - await self._capacity_changed.wait() - self._wakeup.set() - await asyncio.shield(receipt) - - def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None: - self._capacity_changed.clear() - try: - receipt: Final = self.submit(writes) - except UsageCaptureError: - if self._closed or not self.pending_records or not writes: - raise - return None - return receipt - - async def flush(self) -> None: - self._ensure_worker() - self._wakeup.set() - if self._task is not None: - await asyncio.shield(self._task) - - async def close(self) -> None: - with self._admission: - self._closed = True - await self.flush() - - async def stop(self, timeout: float) -> None: - with self._admission: - self._closed = True - try: - await asyncio.wait_for(self.close(), timeout) - except asyncio.TimeoutError: - if self._task is not None: - self._task.cancel() - await asyncio.gather(self._task, return_exceptions=True) - while self._queue: - self._finish( - self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") - ) - verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete") - - def _take_batch(self) -> tuple[_Pending, ...]: - with self._admission: - if self._queue[0].prerequisites: - return (self._queue.popleft(),) - independent: Final = takewhile(lambda item: not item.prerequisites, self._queue) - sizes: Final = accumulate(len(item.writes) for item in independent) - count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes)) - return tuple(self._queue.popleft() for _ in range(count)) - - async def _persist(self, writes: tuple[UsageWrite, ...]) -> None: - for attempt in count(): - try: - await self._write_batch(writes) - except Exception as error: # noqa: BLE001 # retry only classified database failures, preserving frozen writes - if not _retryable_persistence_failure(error): - raise - verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry") - await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6))) - else: - return - - async def _commit(self, batch: tuple[_Pending, ...]) -> None: - writes: Final = tuple(chain.from_iterable(item.writes for item in batch)) - try: - if batch[0].prerequisites: - outcomes: Final = await asyncio.gather( - *(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True - ) - for outcome in outcomes: - if isinstance(outcome, BaseException): - raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome - await self._persist(writes) - except asyncio.CancelledError: - self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) - raise - except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers - if len(batch) > 1 and _isolatable_persistence_failure(error): - try: - for item in batch: - await self._commit((item,)) - except asyncio.CancelledError: - self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) - raise - return - self._finish( - batch, - UsageCaptureError("Usage identity conflicts with accepted content") - if isinstance(error, AccountingConflictError) - else error, - ) - else: - self._finish(batch, None) - - def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None: - for item in batch: - if item.finished.is_set(): - continue - item.finished.set() - with self._admission: - self.pending_bytes -= item.size - self.pending_records -= len(item.writes) - self._capacity_changed.set() - if item.committed.done(): - continue - if error is None: - item.committed.set_result(None) - else: - item.committed.set_exception(error) - - async def _run(self) -> None: - try: - while self._queue: - await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds) - self._wakeup.clear() - await self._commit(self._take_batch()) - if ( - self._closed - or self.pending_records >= self.settings.batch_size - or self._queue - and self._queue[0].prerequisites - ): - self._wakeup.set() - except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally - while self._queue: - self._finish( - self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") - ) - raise diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 6d31fa3f226..e68008135aa 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -6,6 +6,7 @@ from collections import Counter from collections.abc import Mapping, MutableMapping, Sequence from types import MappingProxyType from typing import ( + TYPE_CHECKING, Annotated, Final, Literal, @@ -16,7 +17,6 @@ from typing import ( from urllib.parse import urlparse from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile -from prisma.errors import PrismaError from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model, field_validator from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -63,6 +63,9 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( SSOConfig, ) +if TYPE_CHECKING: + from prisma.errors import PrismaError + router: Final = APIRouter() _DECODED_JSON: Final = TypeAdapter(object) @@ -1260,7 +1263,9 @@ def _parse_environment_variables_param(param_value: str | Mapping[str, object] | async def _load_environment_variables_param( repo: ConfigRepository, -) -> Mapping[str, object] | None | PrismaError | ValueError: +) -> "Mapping[str, object] | None | PrismaError | ValueError": + from prisma.errors import PrismaError + try: entry: Final = await _config_param_db(repo).find_unique(where={"param_name": "environment_variables"}) param_value: Final = None if entry is None else _parse_environment_variables_param(entry.param_value) @@ -1314,6 +1319,8 @@ async def update_sso_settings( detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, ) + from prisma.errors import PrismaError + # Read the existing SSO row first so the audit log captures a real # before/after diff. Stored values are encrypted; decrypt them so the # before-snapshot has the same shape as after_value, and rely on @@ -1353,7 +1360,7 @@ async def update_sso_settings( if isinstance(environment_variables_param, Mapping) else MappingProxyType({}) ) - stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None} + stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None and v != ""} fallback_values: Final = {**legacy_env_values, **stored_values} submitted: Final = sso_config.model_dump(include=sso_config.model_fields_set) sso_data: Final = { diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 837433fd8cc..71ecae74058 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -28,7 +28,7 @@ from dataclasses import dataclass, field from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from functools import partial +from functools import partial, reduce from itertools import takewhile from operator import attrgetter from types import MappingProxyType @@ -66,6 +66,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT, SPEND_LOG_QUEUE_MAX_BYTES, + SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS, SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, VLLM_CHOICE_FIELDS, @@ -167,7 +168,6 @@ from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import ( BilledRoundsDispatch, Logging, - UsageObservation, deployment_pricing_model_info, ) from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token @@ -213,6 +213,7 @@ from litellm.proxy.db.db_url_settings import ( from litellm.proxy.db.exception_handler import ( PrismaDBExceptionHandler, call_with_db_reconnect_retry, + db_write_never_sent, ) from litellm.proxy.db.health_check_latest import ( LatestHealthCheckRow, @@ -277,6 +278,7 @@ from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.spend_log_insert import RejectedSpendLogRow from litellm.repositories.table_repositories import ( EndUserRepository, HealthCheckRepository, @@ -1680,14 +1682,17 @@ _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "updat _EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 +def _extended_exception_chain(chain: tuple[BaseException, ...], iteration: int) -> tuple[BaseException, ...]: + if len(chain) != iteration + 1: + return chain + following: Final = chain[-1].__cause__ or chain[-1].__context__ + if following is None or following in chain: + return chain + return (*chain, following) + + def _exception_chain(exception: BaseException) -> tuple[BaseException, ...]: - chain: Final[list[BaseException]] = [exception] # mutable-ok: built once, returned frozen - for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH): - following = chain[-1].__cause__ or chain[-1].__context__ - if following is None or following in chain: - break - chain.append(following) - return tuple(chain) + return reduce(_extended_exception_chain, range(_EXCEPTION_CHAIN_MAX_DEPTH), (exception,)) def _timed_out_reading_from_provider(exception: Exception) -> bool: @@ -1801,39 +1806,6 @@ def _failure_fields_to_lift( if _prompt_estimate is not None else () ) - from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation - - _recorded_observation: Final[object] = _model_call_details.get("_accounting_usage_observed") - _current_usage: Final = ( - _current_failure_usage_to_lift( - _model_call_details, - request_data, - _model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None, - _billed, - _deployment_id, - _base_model, - _estimate_call_type, - ) - if Logging.usage_capture_callback is not None - and (_billed or _model_call_details.get("combined_usage_object") is not None) - else None - ) - _current_observation: Final = ( - _recorded_observation - if isinstance(_recorded_observation, UsageObservation) - else UsageObservation.UNKNOWN - if _current_usage is None - else UsageObservation.KNOWN - if _model_call_details.get("combined_usage_object") is not None - else UsageObservation.ESTIMATED - ) - _capture_observation: Final = UsageCaptureObservation.from_usage( - _current_usage[0] if _current_usage is not None and isinstance(_current_usage[0], Usage) else None, - _current_observation, - float(_current_usage[1]) - if _current_usage is not None and isinstance(_current_usage[1], (int, float)) - else None, - ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), ("first_api_call_start_time", _first_handoff), @@ -1842,16 +1814,12 @@ def _failure_fields_to_lift( ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)), ("standard_logging_object", _model_call_details.get("standard_logging_object")), ("billed_failed_attempts", _earlier_attempts or None), - ("_accounting_capture", _model_call_details.get("_accounting_capture")), ("_accounting_pricing_snapshot", _model_call_details.get("_accounting_pricing_snapshot")), ("_accounting_finalized_cost", _logging_obj.finalized_cost if isinstance(_logging_obj, Logging) else None), ( "_accounting_allocation", _logging_obj.cost_allowance_allocation if isinstance(_logging_obj, Logging) else None, ), - ("_accounting_usage_observed", _current_observation if Logging.usage_capture_callback is not None else None), - ("_accounting_usage_snapshot", _capture_observation if Logging.usage_capture_callback is not None else None), - ("_accounting_prior_captures", _model_call_details.get("_accounting_prior_captures")), ) return MappingProxyType({key: value for key, value in _entries if value is not None}) @@ -5309,6 +5277,39 @@ _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(ma _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60 _PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) _PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2) +_SPEND_LOG_READ_STATEMENT_TIMEOUT: Final = timedelta(seconds=SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS) +_STATEMENT_TIMEOUT_TRANSACTION_GRACE: Final = timedelta(seconds=5) + + +class _ReadTransactionClient(Protocol): + @property + def read_db(self) -> PrismaWrapper: ... + + +async def query_raw_within_statement_timeout( + prisma_client: _ReadTransactionClient, + sql: str, + *params: object, + what: str = "Spend logs", + timeout: timedelta = _SPEND_LOG_READ_STATEMENT_TIMEOUT, +) -> Sequence[Mapping[str, object]]: + """One raw read on the read target that Postgres cancels at ``timeout``. + + Prisma's HTTP client gives up on a statement after 30s anyway, so without + this the Admin UI's error is the same but Postgres keeps running the scan. + A cancelled read surfaces as a 503 asking for a narrower date range. + """ + try: + async with prisma_client.read_db.tx(timeout=timeout + _STATEMENT_TIMEOUT_TRANSACTION_GRACE) as transaction: + await transaction.execute_raw(f"SET LOCAL statement_timeout = {int(timeout / timedelta(milliseconds=1))}") + return await transaction.query_raw(sql, *params) + except Exception as e: + if not PrismaDBExceptionHandler.is_statement_timeout_error(e): + raise + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={"error": f"{what} took longer than {int(timeout.total_seconds())}s. Pick a shorter date range"}, + ) from e async def _lookup_deprecated_key( @@ -8418,14 +8419,17 @@ async def enqueue_spend_logs( """ added: Final = sum(spend_log_row_bytes(row) for row in logs) async with prisma_client._spend_log_transactions_lock: - queued: Final = ( + queued: Final[tuple[Mapping[str, object], ...]] = ( tuple(logs) + tuple(prisma_client.spend_log_transactions) if at_head else tuple(prisma_client.spend_log_transactions) + tuple(logs) ) kept, kept_bytes = spend_log_queue_within_budget(queued, PrismaClient.spend_log_queue_bytes + added, max_bytes) - prisma_client.spend_log_transactions[:] = kept - PrismaClient.spend_log_queue_bytes = kept_bytes + rescued: Final = tuple( + accounting_only_spend_log(row) for row in queued[: len(queued) - len(kept)] if _carries_a_charge(row) + ) + prisma_client.spend_log_transactions[:] = (*rescued, *kept) + PrismaClient.spend_log_queue_bytes = kept_bytes + sum(spend_log_row_bytes(row) for row in rescued) if len(kept) < len(queued): verbose_proxy_logger.error( "Spend tracking - spend log queue is at its %d byte budget; dropped the %d oldest spend logs", @@ -8480,13 +8484,18 @@ class ProxyUpdateSpend: prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, end_user_list_transactions: dict[str, float], + outer_transaction: _EndUserSpendTransaction | None = None, ): for i in range(n_retry_times + 1): start_time = time.time() try: async with ( db_span("update_end_user_spend", "LiteLLM_EndUserTable"), - _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) as transaction, + ( + contextlib.nullcontext(outer_transaction) + if outer_transaction is not None + else _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) + ) as transaction, ): batcher: _EndUserSpendBatch async with transaction.batch_() as batcher: @@ -8508,6 +8517,8 @@ class ProxyUpdateSpend: break except Exception as e: + if outer_transaction is not None: + raise await DBSpendUpdateWriter.handle_spend_update_failure( e=e, attempt=i, @@ -8547,11 +8558,18 @@ class ProxyUpdateSpend: for i in range(n_retry_times + 1): try: base_url = os.getenv("SPEND_LOGS_URL", None) - if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None: + await _commit_export_accounting( + prisma_client, rows_to_write, base_url if db_writer_client is not None else None + ) + if ( + base_url is not None + and db_writer_client is not None + and any(row.get("accounting_log_disabled") is not True for row in rows_to_write) + ): if not base_url.endswith("/"): base_url += "/" verbose_proxy_logger.debug("base_url: %s", base_url) - json_data = json.dumps(rows_to_write) + json_data = json.dumps(_spend_log_export_rows(rows_to_write)) response = await db_writer_client.post( url=base_url + "spend/update", data=json_data, @@ -8629,6 +8647,28 @@ class ProxyUpdateSpend: return False +def _spend_log_export_rows(rows: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + { + key: value + for key, value in row.items() + if key not in {"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"} + } + for row in rows + if row.get("accounting_log_disabled") is not True + ) + + +async def _commit_export_accounting( + prisma_client: PrismaClient, rows: Sequence[Mapping[str, object]], base_url: str | None +) -> None: + if base_url is None: + return + await SpendLogsRepository(prisma_client).insert_with_deliveries( + tuple(row for row in rows if "accounting_payload" in row), write_spend_logs=False + ) + + async def commit_tool_discoveries(prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging) -> None: """Scheduler job writing discovered tools to LiteLLM_ToolTable, on its own lock, next to update_spend.""" writer: Final = proxy_logging_obj.db_spend_update_writer @@ -8804,9 +8844,20 @@ async def update_daily_tag_spend( def _rollup_batch_never_applied(e: Exception) -> bool: """True when a drained rollup batch provably never reached its rows: Postgres had no - connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row - lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued.""" - return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e) + connection for it (53300), was unreachable or refusing connections (P1001, 57P03), or + cancelled it under ``lock_timeout`` before it took the row lock (55P03). In each case + re-sending the batch cannot double-count, so it is requeued.""" + return ( + PrismaDBExceptionHandler.is_database_capacity_error(e) + or db_write_never_sent(e) + or PrismaDBExceptionHandler.is_lock_timeout_error(e) + ) + + +def _rollup_failure_stops_job(e: Exception) -> bool: + """True when the database cannot take any write, so the job stops instead of draining + the queues after this one into the same failure.""" + return PrismaDBExceptionHandler.is_database_capacity_error(e) or db_write_never_sent(e) async def update_spend_logs_job( @@ -8897,12 +8948,12 @@ async def _run_spend_logs_job( *prisma_client.tool_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during tool usage " + "Spend tracking - database unavailable or rollup row lock timed out during tool usage " "flush, requeued %s tool usage transactions for the next flush: %s", len(tool_usage_to_process), tool_tracking_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err): + if _rollup_failure_stops_job(tool_tracking_err): raise else: verbose_proxy_logger.error( @@ -8929,12 +8980,12 @@ async def _run_spend_logs_job( *prisma_client.model_usage_transactions, ] verbose_proxy_logger.warning( - "Spend tracking - database out of connections or rollup row lock timed out during model usage " + "Spend tracking - database unavailable or rollup row lock timed out during model usage " "flush, requeued %s model usage transactions for the next flush: %s", len(model_usage_to_process), model_usage_err, ) - if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err): + if _rollup_failure_stops_job(model_usage_err): raise else: verbose_proxy_logger.error( @@ -9111,10 +9162,45 @@ def _is_transient_spend_log_write_error(e: Exception) -> bool: return ( PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) + or PrismaDBExceptionHandler.is_database_lifecycle_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e) ) +_ACCOUNTING_ONLY_KEYS: Final = frozenset({"request_id", "call_type", "accounting_payload", "accounting_delivery_id"}) + + +def _carries_a_charge(row: Mapping[str, object]) -> bool: + return "accounting_payload" in row + + +def accounting_only_spend_log(row: Mapping[str, object]) -> Mapping[str, object]: + return { + **{key: value for key, value in row.items() if key in _ACCOUNTING_ONLY_KEYS}, + "accounting_log_disabled": True, + } + + +async def _keep_charges_of_dropped_logs(repo: SpendLogsRepository, rows: Sequence[Mapping[str, object]]) -> None: + charged: Final = tuple(accounting_only_spend_log(row) for row in rows if _carries_a_charge(row)) + if not charged: + return + try: + await repo.insert_with_deliveries(charged) + except Exception as e: # noqa: BLE001 # the request log is already lost; keep the flush going for other rows + if not ( + PrismaDBExceptionHandler.is_rejected_row_error(e) + or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow)) + ): + raise + spend_log_error( + "Spend tracking - could not keep the charges of %d dropped spend log rows. error=%s", + len(charged), + str(e), + exc=e, + ) + + async def _create_spend_logs_with_poison_isolation( repo: SpendLogsRepository, rows: Sequence[Mapping[str, object]], @@ -9143,10 +9229,13 @@ async def _create_spend_logs_with_poison_isolation( persist. Returns the budget left after this subtree. """ try: - await repo.table.create_many(data=rows, skip_duplicates=True) + await repo.insert_with_deliveries(rows) return failure_budget except Exception as e: - if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)): + if not ( + PrismaDBExceptionHandler.is_rejected_row_error(e) + or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow)) + ): raise if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): raise @@ -9161,6 +9250,7 @@ async def _create_spend_logs_with_poison_isolation( str(e), exc=e, ) + await _keep_charges_of_dropped_logs(repo, rows) return budget_left if budget_left <= 0: spend_log_error( @@ -9168,6 +9258,7 @@ async def _create_spend_logs_with_poison_isolation( "isolation failure budget exhausted for this flush", len(rows), ) + await _keep_charges_of_dropped_logs(repo, rows) return 0 mid: Final = len(rows) // 2 remaining: Final = await _create_spend_logs_with_poison_isolation(repo, rows[:mid], budget_left) @@ -9177,6 +9268,7 @@ async def _create_spend_logs_with_poison_isolation( "isolation failure budget exhausted for this flush", len(rows) - mid, ) + await _keep_charges_of_dropped_logs(repo, rows[mid:]) return 0 return await _create_spend_logs_with_poison_isolation(repo, rows[mid:], remaining) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e03cb44efda..be1df4aa0f6 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -2,11 +2,13 @@ import asyncio import os +import traceback from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast +import httpx from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never @@ -14,10 +16,14 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, + REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.xai.common_utils import XAIModelInfo @@ -36,18 +42,24 @@ from litellm.utils import ( ModelInfoLookupMiss, ProviderConfigManager, aresolve_api_key_secret_reference, + get_base_model_from_metadata, lookup_model_info_for_registration, ) -from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params +from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from ..litellm_core_utils.realtime_streaming import client_websocket_scope +from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason +from ..litellm_core_utils.realtime_streaming import ( + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, + client_websocket_scope, + translation_guardrails_apply, +) from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed -from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, realtime_ssl_for_url from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import ( @@ -66,8 +78,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig - azure_realtime: Final = AzureOpenAIRealtime() openai_realtime: Final = OpenAIRealtime() openai_live_sessions: Final = OpenAILiveSessions() @@ -77,7 +87,7 @@ vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) -_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: @@ -90,26 +100,145 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> return MappingProxyType({**credential_values, **model_params}) -def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: - if "model" not in session: - return session - return {**session, "model": model_name} +def _transcription_model_overrides(session: Mapping[str, object], model_name: str) -> Mapping[str, object]: + flat_transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + audio: Final = as_str_mapping(session.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested_transcription: Final = as_str_mapping(audio_input.get("transcription")) + flat_part: Final = ( + {} if flat_transcription is None else {"input_audio_transcription": {**flat_transcription, "model": model_name}} + ) + nested_part: Final = ( + {} + if nested_transcription is None and flat_transcription is not None + else { + "audio": { + **audio, + "input": {**audio_input, "transcription": {**(nested_transcription or {}), "model": model_name}}, + } + } + ) + return MappingProxyType({**flat_part, **nested_part}) -def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None: - session: Final = as_str_mapping(request_data.get("session")) - if not isinstance(session, dict): - return None - return _with_resolved_session_model(session, model_name) +def with_resolved_session_model(session: Mapping[str, object], model_name: str) -> dict[str, object]: + top_level_model: Final = {} if "model" not in session else {"model": model_name} + transcription_models: Final = ( + _transcription_model_overrides(session, model_name) if session.get("type") == "transcription" else {} + ) + return {**session, **top_level_model, **transcription_models} -def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: - """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" - metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})} - guardrails: Final = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or [] - if guardrails: - metadata["guardrails"] = guardrails - return metadata +def _without_top_level_nulls(fields: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in fields.items() if value is not None} + + +_POLICY_PIPELINE_METADATA_KEYS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails") + + +def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: + request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {}) + internal_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("litellm_metadata") or {}) + policy_pipeline_state: Final = { + key: request_metadata[key] for key in _POLICY_PIPELINE_METADATA_KEYS if key in request_metadata + } + guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails") + return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} + + +_GUARDRAIL_OPT_OUT_METADATA_KEYS: Final = ("user_api_key_metadata", "user_api_key_team_metadata") + + +def _translation_litellm_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {}) + guardrail_opt_outs: Final = { + key: request_metadata[key] for key in _GUARDRAIL_OPT_OUT_METADATA_KEYS if key in request_metadata + } + return MappingProxyType({**guardrail_opt_outs, **_build_litellm_metadata(kwargs)}) + + +_HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS}) + + +@dataclass(frozen=True, slots=True) +class _TranslationProviderUnsupported: + provider: str + supported_providers: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class _TranslationGuardrailsUnsupported: + pass + + +_TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrailsUnsupported + + +def _translation_rejection( + custom_llm_provider: str, supported_providers: frozenset[str], litellm_metadata: Mapping[str, object] +) -> _TranslationRejection | None: + if custom_llm_provider not in supported_providers: + return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers) + if translation_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}): + return _TranslationGuardrailsUnsupported() + return None + + +def _translation_rejection_message(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(provider=provider, supported_providers=supported_providers): + return ( + f"Realtime translation supports only the {', '.join(sorted(supported_providers))} providers, " + f"got {provider}" + ) + case _TranslationGuardrailsUnsupported(): + return TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + case _: + assert_never(rejection) + + +def _translation_rejection_error_type(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(): + return "invalid_request_error" + case _TranslationGuardrailsUnsupported(): + return "guardrail_error" + case _: + assert_never(rejection) + + +def _translation_rejection_error( + rejection: _TranslationRejection, model: str, custom_llm_provider: str +) -> BadRequestError: + return BadRequestError( + message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider + ) + + +def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None: + rejection: Final = _translation_rejection( + custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, _translation_litellm_metadata(kwargs) + ) + if rejection is None: + return + raise _translation_rejection_error(rejection, model, custom_llm_provider) + + +async def _refuse_translation_session( + websocket: "WebSocket", + rejection: _TranslationRejection, + logging_obj: LiteLLMLogging, + model: str, + custom_llm_provider: str, +) -> None: + message: Final = _translation_rejection_message(rejection) + await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection))) + await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused")) + error: Final = _translation_rejection_error(rejection, model, custom_llm_provider) + await logging_obj.dispatch_failure_handlers( + error, "".join(traceback.format_exception(error)), prefer_async_handlers=True + ) def _get_realtime_http_provider_config( @@ -125,10 +254,6 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import ( - BaseRealtimeHTTPConfig, - ) - provider_config: BaseRealtimeHTTPConfig | None = None if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_realtime_http_config( @@ -163,6 +288,32 @@ async def _realtime_http_bearer_token( return bearer_token or "" +async def _get_realtime_http_extra_headers( + provider_config: BaseRealtimeHTTPConfig | None, + litellm_params: GenericLiteLLMParams, + resolved_api_key: str, + extra_headers: Mapping[str, object] | None, +) -> Mapping[str, object] | None: + if provider_config is None: + return extra_headers + return await asyncio.to_thread( + provider_config.get_extra_headers, + litellm_params=litellm_params, + api_key=resolved_api_key, + extra_headers=extra_headers, + ) + + +def _client_secret_session_body(req: RealtimeClientSecretRequest, model_name: str) -> Mapping[str, object] | None: + if req.session is not None: + return with_resolved_session_model( + _without_top_level_nulls(req.session.model_dump(exclude_unset=True)), model_name + ) + if req.model is not None: + return {"type": "realtime", "model": model_name} + return None + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -176,31 +327,42 @@ async def acreate_realtime_client_secret( session=RealtimeSessionConfig.model_validate(session) if session else None, expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None, ) - model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" + transcription_model: Final = ( + req.session.audio.input.transcription.model + if req.session is not None + and req.session.type == "transcription" + and req.session.audio is not None + and req.session.audio.input is not None + and req.session.audio.input.transcription is not None + else None + ) + requested_model_name: Final = ( + transcription_model + or (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") litellm_params: Final = GenericLiteLLMParams(**kwargs) - ( - model_name, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = get_llm_provider( - model=model_name, + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, custom_llm_provider=litellm_params.custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - ( - provider_config, - resolved_api_base, - resolved_api_key, - ) = _get_realtime_http_provider_config( + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -208,26 +370,124 @@ async def acreate_realtime_client_secret( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - resolved_session: Final = _resolved_request_session(request_data, model_name) - if resolved_session is not None: - request_data["session"] = resolved_session + session_body: Final = _client_secret_session_body(req, model_name) return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params ), - request_data=request_data, + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + **({} if session_body is None else {"session": session_body}), + }, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) +@wrapper_client +async def acreate_realtime_translation_client_secret( + model: str | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options +) -> httpx.Response: + requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" + session_config: Final = RealtimeSessionConfig.model_validate( + { + **(session or {}), + "type": "translation", + "model": requested_model_name, + } + ) + req: Final = RealtimeClientSecretRequest( + model=requested_model_name, + session=session_config, + expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + ) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=as_str_mapping(kwargs.get("extra_headers")), + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ + "expires_after": expires_after, + "session": session, + }, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + translation_session: Final = with_resolved_session_model( + _without_top_level_nulls(session_config.model_dump(exclude_unset=True)), model_name + ) + return await base_llm_http_handler.async_realtime_translation_client_secret_handler( + api_base=resolved_api_base, + api_key=await _realtime_http_bearer_token( + custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params + ), + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + "session": {key: value for key, value in translation_session.items() if key != "type"}, + }, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=resolved_extra_headers, + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, + api_version=litellm_params.api_version, + ) + + +def _with_upstream_transcription_model(request_data: Mapping[str, object], model_name: str) -> dict[str, object]: + flat: Final = as_str_mapping(request_data.get("input_audio_transcription")) + audio: Final = as_str_mapping(request_data.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested: Final = as_str_mapping(audio_input.get("transcription")) + return { + **request_data, + **({} if flat is None else {"input_audio_transcription": {**flat, "model": model_name}}), + **( + {} + if nested is None + else {"audio": {**audio, "input": {**audio_input, "transcription": {**nested, "model": model_name}}}} + ), + } + + +def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None: + base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs}) + return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/") + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, @@ -273,6 +533,12 @@ async def acreate_realtime_transcription_session( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -280,13 +546,11 @@ async def acreate_realtime_transcription_session( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - # Ensure the upstream body's input_audio_transcription.model matches the - # authorized routing model. This prevents a caller from supplying an allowed - # top-level model for auth while sneaking a different model into the nested - # transcription config that gets forwarded to the provider. - if isinstance(request_data.get("input_audio_transcription"), dict): - request_data["input_audio_transcription"]["model"] = model_name + # The upstream transcription model must match the authorized routing model, so a + # caller cannot pass auth with an allowed top-level model and forward a different one. + request_data: Final = _with_upstream_transcription_model( + _without_top_level_nulls(req.model_dump(exclude_unset=True, exclude={"model"})), model_name + ) return await base_llm_http_handler.async_realtime_transcription_session_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( @@ -297,7 +561,7 @@ async def acreate_realtime_transcription_session( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) @@ -334,7 +598,7 @@ async def arealtime_calls( litellm_params=litellm_params, ) if session is not None: - session = _with_resolved_session_model(session, model_name) + session = with_resolved_session_model(session, model_name) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -357,6 +621,68 @@ async def arealtime_calls( ) +@wrapper_client +async def arealtime_translation_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: str | None = None, + session: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options +) -> httpx.Response: + requested_model_name: Final = model or "gpt-realtime-translate" + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + session_config: Final = with_resolved_session_model( + { + **(session or {}), + "type": "translation", + "model": model_name, + }, + model_name, + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ + "realtime_translation_calls": True, + "session": session_config, + }, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session_config, + extra_headers=as_str_mapping(kwargs.get("extra_headers")), + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, + api_version=litellm_params.api_version, + translation=True, + ) + + async def vertex_access_token_resolver( credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, @@ -486,6 +812,7 @@ async def _arealtime( timeout: float | None = None, query_params: RealtimeQueryParams | None = None, live_session_start: Mapping[str, object] | None = None, + realtime_mode: str = "realtime", **kwargs, ) -> None: """ @@ -530,7 +857,24 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) - deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs) + session_litellm_metadata: Final = ( + dict(_translation_litellm_metadata(kwargs)) + if realtime_mode == "translation" + else _build_litellm_metadata(kwargs) + ) + translation_rejection: Final = ( + _translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, session_litellm_metadata) + if realtime_mode == "translation" + else None + ) + if translation_rejection is not None: + await _refuse_translation_session( + websocket, translation_rejection, litellm_logging_obj, model, _custom_llm_provider + ) + litellm_logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True + return + + deployment_params: Final = _OBJECT_MAPPING.validate_python(kwargs) if live_session_start is not None: await _alive_session( model=model, @@ -540,7 +884,7 @@ async def _arealtime( session_start=live_session_start, api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, litellm_params=deployment_params, ) return @@ -566,7 +910,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: @@ -581,9 +925,6 @@ async def _arealtime( or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - realtime_protocol: Final = azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) resolved_azure_ad_token: Final = ( None if api_key @@ -592,6 +933,14 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) + realtime_protocol: Final = azure_realtime_protocol_for_client( + configured_realtime_protocol, + model=model, + base_model=_deployment_base_model(kwargs, _custom_llm_provider), + query_params=query_params, + realtime_mode=realtime_mode, + websocket=websocket, + ) await azure_realtime.async_realtime( model=model, websocket=websocket, @@ -602,10 +951,13 @@ async def _arealtime( client=None, timeout=timeout, logging_obj=litellm_logging_obj, - realtime_protocol=realtime_protocol, + realtime_protocol=realtime_protocol.name, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, + bridges_beta_client_to_ga=realtime_protocol.bridges_beta_client_to_ga, + drop_params=drop_params, ) elif _custom_llm_provider == "openai": api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/" @@ -622,8 +974,10 @@ async def _arealtime( client=None, timeout=timeout, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, + drop_params=drop_params, ) elif _custom_llm_provider == "bedrock": # Extract AWS parameters from kwargs @@ -678,7 +1032,7 @@ async def _arealtime( timeout=timeout, query_params=query_params, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, ) elif _custom_llm_provider == "vertex_ai": vertex_credentials: Final = ( @@ -738,7 +1092,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) else: diff --git a/litellm/repositories/accounting_delivery_repository.py b/litellm/repositories/accounting_delivery_repository.py new file mode 100644 index 00000000000..7dd92aecb2d --- /dev/null +++ b/litellm/repositories/accounting_delivery_repository.py @@ -0,0 +1,193 @@ +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import timedelta +from itertools import chain +from types import TracebackType +from typing import TYPE_CHECKING, Final, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm.repositories.prisma_protocols import TableActions + +if TYPE_CHECKING: + from litellm.proxy.db.db_spend_update_writer import SpendBatchManager + + +class LedgerTransaction(Protocol): + @property + def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + async def execute_raw(self, query: str, *args: object) -> int: ... + + def batch_(self) -> "SpendBatchManager": ... + + +class _TransactionManager(Protocol): + async def __aenter__(self) -> LedgerTransaction: ... + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + + +class LedgerDatabase(Protocol): + def tx(self, *, timeout: timedelta) -> _TransactionManager: ... + + +class _Delivery(BaseModel): + model_config = ConfigDict(frozen=True) + + delivery_id: str + attempts: int + accounting_payload: JsonValue + + +@dataclass(frozen=True, slots=True) +class DeliveryProjection: + row: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class DeliveryBatchResult: + applied: int = 0 + failed: int = 0 + parked: tuple[str, ...] = () + error: str | None = None + + +class DeliveryMetrics(BaseModel): + model_config = ConfigDict(frozen=True) + + pending_count: int + parked_count: int + oldest_pending_seconds: float + batch_count: int + table_bytes: int + + +_DELIVERIES: Final = TypeAdapter(tuple[_Delivery, ...]) +_PAYLOAD: Final = TypeAdapter(dict[str, object]) +_TIMEOUT: Final = timedelta(seconds=60) +_CLAIM: Final = """SELECT delivery_id, attempts, accounting_payload FROM "LiteLLM_SpendLogDelivery" +WHERE state = 'pending' AND next_attempt_at <= CURRENT_TIMESTAMP + AND (attempts > 0) = $2::boolean +ORDER BY next_attempt_at, delivery_id LIMIT $1::integer FOR UPDATE SKIP LOCKED""" +_APPLIED: Final = """UPDATE "LiteLLM_SpendLogDelivery" +SET state = 'applied', applied_at = CURRENT_TIMESTAMP, last_error = NULL +WHERE delivery_id = ANY($1::text[]) AND state = 'pending' """ +_FAILED: Final = """UPDATE "LiteLLM_SpendLogDelivery" +SET attempts = attempts + 1, last_error = $3, + next_attempt_at = CURRENT_TIMESTAMP + make_interval(secs => LEAST(300, power(2, LEAST(attempts, 8)))::integer), + state = $4::"AccountingDeliveryState" +WHERE delivery_id = $1 AND state = 'pending' AND attempts = $2::integer +RETURNING delivery_id, attempts, accounting_payload""" + + +def _is_permanent_failure(error: Exception) -> bool: + """A delivery whose own data failed fails the same way on every retry, while any other error is the database's + and clears once it recovers.""" + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler + + if PrismaDBExceptionHandler.is_database_service_unavailable_error(error): + return False + return isinstance(error, (ValueError, TypeError, KeyError)) or PrismaDBExceptionHandler.is_rejected_row_error(error) + + +class AccountingDeliveryRepository: + def __init__(self, database: LedgerDatabase) -> None: + self._database = database + + async def metrics(self) -> DeliveryMetrics: + async with self._database.tx(timeout=_TIMEOUT) as transaction: + rows: Final = await transaction.query_raw( + "SELECT COUNT(*) FILTER (WHERE state = 'pending') AS pending_count, " + "COUNT(*) FILTER (WHERE state = 'parked') AS parked_count, " + "COALESCE(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - MIN(created_at) FILTER (WHERE state = 'pending')), 0) " + "AS oldest_pending_seconds, COUNT(*) FILTER (WHERE is_batch) AS batch_count, " + "pg_total_relation_size('\"LiteLLM_SpendLogDelivery\"') AS table_bytes " + 'FROM "LiteLLM_SpendLogDelivery"' + ) + return DeliveryMetrics.model_validate(rows[0]) + + async def cleanup_applied(self, *, retention_days: int = 1, batch_size: int = 1000) -> int: + if retention_days < 1 or batch_size < 1: + raise ValueError("Delivery retention and cleanup batch size must be positive") + async with self._database.tx(timeout=_TIMEOUT) as transaction: + return await transaction.execute_raw( + 'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id IN (' + 'SELECT delivery_id FROM "LiteLLM_SpendLogDelivery" ' + "WHERE state = 'applied' AND NOT is_batch AND applied_at < CURRENT_TIMESTAMP - make_interval(days => $1::integer) " + "ORDER BY applied_at, delivery_id LIMIT $2::integer FOR UPDATE SKIP LOCKED)", + retention_days, + batch_size, + ) + + async def apply_pending( + self, + apply: Callable[[LedgerTransaction, tuple[DeliveryProjection, ...]], Awaitable[None]], + *, + batch_size: int = 100, + max_attempts: int = 5, + retry_only: bool | None = None, + ) -> DeliveryBatchResult: + if batch_size < 1 or max_attempts < 1: + raise ValueError("Delivery batch size and maximum attempts must be positive") + claimed: tuple[_Delivery, ...] = () # rebind-ok: preserve claims for retry accounting after rollback + try: + async with self._database.tx(timeout=_TIMEOUT) as transaction: + retries: Final = ( + _DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, 1, True)) + if retry_only is not False + else () + ) + claimed = retries or ( # rebind-ok: retain claimed identities after rollback + _DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, batch_size, False)) + if retry_only is not True + else () + ) + if not claimed: + return DeliveryBatchResult() + await apply( + transaction, + tuple( + DeliveryProjection(_PAYLOAD.validate_python(delivery.accounting_payload)) + for delivery in claimed + ), + ) + await transaction.execute_raw(_APPLIED, tuple(delivery.delivery_id for delivery in claimed)) + return DeliveryBatchResult(applied=len(claimed)) + except Exception as error: # noqa: BLE001 # roll back all destinations before recording a retry + if not claimed: + raise + return await self._record_failure(claimed, error, max_attempts) + + async def _record_failure( + self, claimed: tuple[_Delivery, ...], error: Exception, max_attempts: int + ) -> DeliveryBatchResult: + parks: Final = len(claimed) == 1 and _is_permanent_failure(error) + async with self._database.tx(timeout=_TIMEOUT) as transaction: + updates: Final = tuple( + [ + await self._record_row_failure( + transaction, + delivery, + type(error).__name__, + "parked" if parks and delivery.attempts + 1 >= max_attempts else "pending", + ) + for delivery in claimed + ] + ) + return DeliveryBatchResult( + failed=len(claimed), parked=tuple(chain.from_iterable(updates)), error=type(error).__name__ + ) + + @staticmethod + async def _record_row_failure( + transaction: LedgerTransaction, delivery: _Delivery, error: str, state: Literal["pending", "parked"] + ) -> tuple[str, ...]: + updated: Final = _DELIVERIES.validate_python( + await transaction.query_raw(_FAILED, delivery.delivery_id, delivery.attempts, error, state) + ) + return tuple(row.delivery_id for row in updated) if state == "parked" else () diff --git a/litellm/repositories/plugin_upload_repository.py b/litellm/repositories/plugin_upload_repository.py new file mode 100644 index 00000000000..99f1f76abb5 --- /dev/null +++ b/litellm/repositories/plugin_upload_repository.py @@ -0,0 +1,71 @@ +import json +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma dynamically exposes its typed table actions + +from pydantic import TypeAdapter + +from litellm.models.plugin_upload import PluginArchiveMetadata, PluginUpload +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import ClaudeCodePluginRepository + +if TYPE_CHECKING: + from prisma import models as prisma_models + + from litellm.proxy.utils import PrismaClient + +_MANIFEST: Final = TypeAdapter(dict[str, object]) +_SOURCE: Final = TypeAdapter(dict[str, str]) + + +class PluginUploadRepository: + def __init__(self, prisma_client: "PrismaClient") -> None: + self._plugins = ClaudeCodePluginRepository(prisma_client, use_writer=True).table + self._archives = cast( # cast-ok: Prisma exposes generated table actions through an untyped delegate + TableActions["prisma_models.LiteLLM_ClaudeCodePluginUpload"], + prisma_client.writer_db.litellm_claudecodepluginupload, + ) + + async def create(self, upload: PluginUpload) -> str | None: + from prisma import Base64 + from prisma.errors import UniqueViolationError + + try: + plugin: Final = await self._plugins.create( + data={ + "name": upload.name, + "version": upload.version, + "description": upload.description, + "manifest_json": json.dumps(dict(upload.manifest)), + "created_by": upload.created_by, + "uploaded_archive": {"create": {"content": Base64.encode(upload.content)}}, + } + ) + except UniqueViolationError: + return None + return plugin.id + + async def get_metadata(self, name: str) -> PluginArchiveMetadata | None: + plugin: Final = await self._plugins.find_unique(where={"name": name}) + if plugin is None: + return None + manifest: Final = _MANIFEST.validate_json(plugin.manifest_json or "{}") + return PluginArchiveMetadata( + name=plugin.name, + enabled=plugin.enabled, + source=MappingProxyType(_SOURCE.validate_python(manifest.get("source", {}))), + manifest_json=plugin.manifest_json, + ) + + async def get_content(self, metadata: PluginArchiveMetadata) -> bytes | None: + archive: Final = await self._archives.find_first( + where={ + "plugin_name": metadata.name, + "plugin": { + "is": { + "manifest_json": metadata.manifest_json, + "enabled": metadata.enabled, + } + }, + } + ) + return None if archive is None else archive.content.decode() diff --git a/litellm/repositories/spend_log_insert.py b/litellm/repositories/spend_log_insert.py new file mode 100644 index 00000000000..a958edb9f8a --- /dev/null +++ b/litellm/repositories/spend_log_insert.py @@ -0,0 +1,88 @@ +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError +from typing_extensions import LiteralString + +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + +_COLUMNS: Final[tuple[LiteralString, ...]] = ( + "request_id", + "call_type", + "api_key", + "spend", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "startTime", + "endTime", + "request_duration_ms", + "completionStartTime", + "model", + "model_id", + "model_group", + "custom_llm_provider", + "api_base", + "user", + "metadata", + "cache_hit", + "cache_key", + "request_tags", + "team_id", + "organization_id", + "end_user", + "requester_ip_address", + "messages", + "response", + "session_id", + "status", + "mcp_namespaced_tool_name", + "billing_agent_id", + "agent_id", + "proxy_server_request", + "litellm_call_id", + "created_at", + "updated_at", +) +_JSON_COLUMNS: Final = frozenset({"metadata", "request_tags", "messages", "response", "proxy_server_request"}) + + +class RejectedSpendLogRow(ValueError): + """A row the raw insert cannot encode, so it is isolated like a row Postgres refuses.""" + + +def _json_value(column: str, value: object) -> object: + if isinstance(value, datetime): + return value.isoformat() + if column not in _JSON_COLUMNS or not isinstance(value, str): + return value + try: + return _JSON.validate_json(value) + except ValidationError as error: + raise RejectedSpendLogRow(f"Spend log column {column} is not valid JSON") from error + + +def spend_log_insert_statement(rows: Sequence[Mapping[str, object]]) -> tuple[LiteralString, str]: + if not rows: + raise ValueError("Spend log insert requires at least one row") + fields: Final = frozenset(rows[0]) | {"created_at", "updated_at"} + if not fields.issubset(_COLUMNS) or any(frozenset(row) | {"created_at", "updated_at"} != fields for row in rows): + raise RejectedSpendLogRow("Spend log insert requires matching known columns") + columns: Final = ", ".join('"' + column + '"' for column in _COLUMNS if column in fields) + now: Final = datetime.now(timezone.utc).isoformat() + values: Final = tuple( + {"created_at": now, "updated_at": now, **{column: _json_value(column, value) for column, value in row.items()}} + for row in rows + ) + sql: Final = ( + 'INSERT INTO "LiteLLM_SpendLogs" (' + + columns + + ") SELECT " + + columns + + ' FROM jsonb_populate_recordset(NULL::"LiteLLM_SpendLogs", $1::jsonb) ' + "ON CONFLICT DO NOTHING RETURNING request_id" + ) + return sql, safe_dumps(values, non_finite_as_null=True) diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index 2d3139f7a2c..10db81eca4b 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -7,8 +7,15 @@ These are thin wrappers for tables that do not (yet) need domain-specific query methods; richer repositories live in their own modules. """ +from collections.abc import Mapping, Sequence +from datetime import timedelta +from itertools import groupby from typing import TYPE_CHECKING, Any, Final, Generic +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.repositories.accounting_delivery_repository import LedgerDatabase, LedgerTransaction from litellm.repositories.prisma_protocols import RowT_co, TableActions if TYPE_CHECKING: @@ -80,9 +87,117 @@ class OrganizationMembershipRepository(PrismaTableRepository["prisma_models.Lite table_name = "litellm_organizationmembership" +_SPEND_LOG_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def _accounting_payload(value: object) -> Mapping[str, object]: + return ( + _SPEND_LOG_FIELDS.validate_json(value) if isinstance(value, str) else _SPEND_LOG_FIELDS.validate_python(value) + ) + + +_ACCOUNTING_ONLY_FIELDS: Final = frozenset({"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"}) + + +def _settles_once(row: Mapping[str, object]) -> bool: + """A batch or video cost row: update_database gives every other ledger row its own delivery id.""" + return "accounting_delivery_id" not in row + + +def _delivery_id(row: Mapping[str, object]) -> str: + return str(row["accounting_delivery_id"]) if not _settles_once(row) else str(row["request_id"]) + + class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs"]): table_name = "litellm_spendlogs" + async def insert_with_deliveries( + self, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool = True + ) -> int: + legacy: Final = tuple(row for row in rows if "accounting_payload" not in row) + legacy_count: Final = await self.table.create_many(data=legacy, skip_duplicates=True) if legacy else 0 + ledger: Final = tuple( + sorted( + {_delivery_id(row): row for row in reversed(rows) if "accounting_payload" in row}.values(), + key=_delivery_id, + ) + ) + if not ledger: + return legacy_count + database: Final[LedgerDatabase] = self.prisma_client.writer_db + async with database.tx(timeout=timedelta(seconds=60)) as transaction: + accepted: Final = await self._insert_ledger(transaction, ledger, write_spend_logs=write_spend_logs) + return legacy_count + accepted + + @staticmethod + async def _insert_ledger( + transaction: LedgerTransaction, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool + ) -> int: + from litellm.proxy.db.db_span import db_span + from litellm.repositories.spend_log_insert import spend_log_insert_statement + + payloads: Final = tuple( + { + "delivery_id": _delivery_id(row), + "request_id": row["request_id"], + "accounting_payload": _accounting_payload(row["accounting_payload"]), + "is_batch": _settles_once(row), + } + for row in rows + ) + inserted_deliveries: Final = await transaction.query_raw( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) ' + "SELECT delivery_id, request_id, accounting_payload, is_batch FROM jsonb_to_recordset($1::jsonb) " + "AS inputs(delivery_id text, request_id text, accounting_payload jsonb, is_batch boolean) " + "ON CONFLICT (delivery_id) DO NOTHING RETURNING delivery_id", + safe_dumps(payloads, non_finite_as_null=True), + ) + accepted_ids: Final = frozenset(str(row["delivery_id"]) for row in inserted_deliveries) + charged_by_legacy: Final = await SpendLogsRepository._jobs_charged_by_legacy( + transaction, + tuple(_delivery_id(row) for row in rows if _settles_once(row) and _delivery_id(row) in accepted_ids), + ) + if charged_by_legacy: + await transaction.execute_raw( + 'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = ANY($1::text[])', list(charged_by_legacy) + ) + accepted: Final = accepted_ids - charged_by_legacy + local_rows: Final = tuple( + {key: value for key, value in row.items() if key not in _ACCOUNTING_ONLY_FIELDS} + for row in rows + if _delivery_id(row) in accepted and row.get("accounting_log_disabled") is not True + ) + if write_spend_logs and local_rows: + groups: Final = tuple( + tuple(group) + for _, group in groupby( + sorted(local_rows, key=lambda row: tuple(sorted(row))), key=lambda row: tuple(sorted(row)) + ) + ) + async with db_span("insert_spend_logs", "LiteLLM_SpendLogs"): + for group in groups: + await transaction.query_raw(*spend_log_insert_statement(group)) + return len(accepted) + + @staticmethod + async def _jobs_charged_by_legacy(transaction: LedgerTransaction, cost_row_ids: Sequence[str]) -> frozenset[str]: + """Batches and videos a pre-ledger poll already charged through their SpendLogs cost row.""" + if not cost_row_ids: + return frozenset() + rows: Final = await transaction.query_raw( + 'SELECT DISTINCT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1::text[]) ' + "AND status = 'success' AND spend > 0", + list(cost_row_ids), + ) + return frozenset(str(row["request_id"]) for row in rows) + + async def has_delivery(self, delivery_id: str) -> bool: + database: Final[LedgerTransaction] = self.prisma_client.writer_db + rows: Final = await database.query_raw( + 'SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = $1', delivery_id + ) + return bool(rows) + class BudgetWindowSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_BudgetWindowSpend"]): table_name = "litellm_budgetwindowspend" diff --git a/litellm/repositories/verification_token_repository.py b/litellm/repositories/verification_token_repository.py index b20fb47306e..fb1f1c89e69 100644 --- a/litellm/repositories/verification_token_repository.py +++ b/litellm/repositories/verification_token_repository.py @@ -124,9 +124,7 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"user_id": user_id}) return self._to_model_list(records) - async def find_newest_reusable_llm_api_key( - self, user_id: str, team_id: str | None - ) -> LiteLLM_VerificationToken | None: + async def find_newest_reusable_key(self, user_id: str, team_id: str | None) -> LiteLLM_VerificationToken | None: records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many( where={ "user_id": user_id, @@ -140,12 +138,6 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): {"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}}, ] }, - { - "OR": [ - {"allowed_routes": {"is_empty": True}}, - {"allowed_routes": {"has": "llm_api_routes"}}, - ] - }, ], }, order={"created_at": "desc"}, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 0f4ed0611be..3e605e926f7 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -52,6 +52,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( cache_control_count, dropped_cache_control, newest_marks_within_cache_control_limit, + routed_model_info, whole_messages_within_cache_control_limit, with_cache_control, ) @@ -782,13 +783,15 @@ def _replays_compaction_block(input: str | ResponseInputParam, context_managemen _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: Final = frozenset(("anthropic", "dashscope", "minimax", "sap", "zai")) -def _bedrock_target_keeps_item_marks(model: str, drop_params: bool) -> bool: - declared: Final = bedrock_declared_cache_point_support(model) +def _bedrock_target_keeps_item_marks(model: str, drop_params: bool, model_info: Mapping[str, object] | None) -> bool: + declared: Final = bedrock_declared_cache_point_support(model, model_info) return declared is True or (declared is None and not drop_params) -def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_params: bool) -> bool: - if supports_anthropic_cache_control(model, custom_llm_provider): +def _target_keeps_item_marks( + model: str, custom_llm_provider: str | None, drop_params: bool, model_info: Mapping[str, object] | None +) -> bool: + if supports_anthropic_cache_control(model, custom_llm_provider, model_info): return True if custom_llm_provider in _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: return True @@ -798,7 +801,7 @@ def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_p return is_snowflake_claude_model(model) if custom_llm_provider == "databricks": return "claude" in model.lower() - return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params) + return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params, model_info) def _refuses_marks_on_tool_blocks(model: str, custom_llm_provider: str | None) -> bool: @@ -1234,7 +1237,10 @@ class LiteLLMCompletionResponsesConfig: request_kwargs: Final = _STR_KEY_DICT_ADAPTER.validate_python({**kwargs, "drop_params": drop_params}) context_management: Final = responses_api_request.get("context_management") keeps_item_marks: Final = _target_keeps_item_marks( - model, custom_llm_provider, drops_unsupported_params(request_kwargs.get("drop_params")) + model, + custom_llm_provider, + drops_unsupported_params(request_kwargs.get("drop_params")), + routed_model_info(request_kwargs.get("model_info")), ) request_input: Final = with_compaction_items_as_params(input) native_compaction: Final = _replays_compaction_block( @@ -1456,7 +1462,10 @@ class LiteLLMCompletionResponsesConfig: model: Final = str(request_fields.get("model") or "") provider: Final = requested_provider if isinstance(requested_provider, str) else None keeps_item_marks: Final = _target_keeps_item_marks( - model, provider, drops_unsupported_params(request_kwargs.get("drop_params")) + model, + provider, + drops_unsupported_params(request_kwargs.get("drop_params")), + routed_model_info(request_kwargs.get("model_info")), ) lookup: Final = session_lookup or partial( ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index ad1d35bbe17..e9f9eb85d25 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -27,6 +27,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( CARRY_UNMATCHED_MESSAGE_POINTS, AnthropicCacheControlHook, configured_injection_points, + routed_model_info, ) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind @@ -758,9 +759,7 @@ def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None: return api_base if isinstance(api_base, str) else None -def _dispatched_model_name( - model: str, custom_llm_provider: str, api_base: str | None, litellm_params: GenericLiteLLMParams | None = None -) -> str: +def _dispatched_model_name(model: str, custom_llm_provider: str) -> str: from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) @@ -773,7 +772,6 @@ def _will_bridge_to_chat_completions( use_chat_completions_api: bool, model_info: object, api_base: str | None, - litellm_params: GenericLiteLLMParams | None = None, ) -> bool: """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. @@ -790,7 +788,7 @@ def _will_bridge_to_chat_completions( return True return _bridges_to_chat_completions( _resolve_responses_api_provider_config( - _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), + _dispatched_model_name(normalized_model[0], custom_llm_provider), custom_llm_provider, model_info, api_base, @@ -815,7 +813,6 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), api_base, - litellm_params, ) @@ -882,6 +879,7 @@ def _stamp_injection_points_for_the_hook( custom_llm_provider, kwargs.get("api_base") or kwargs.get("base_url"), kwargs.get("prompt_cache_options"), + routed_model_info(kwargs.get("model_info")), ) @@ -997,7 +995,6 @@ async def aresponses( bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), _api_base_kwarg(kwargs), - GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -1173,7 +1170,6 @@ def _apply_prompt_management_to_responses_call( use_chat_completions_api, kwargs.get("model_info"), _api_base_kwarg(kwargs), - GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index da8ff2cd207..249c6370b86 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1031,9 +1031,7 @@ class BaseResponsesAPIStreamingIterator: ) if estimate is None: return - usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the success path above uses - estimate - ) + usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(estimate) served_tier: Final[object] = getattr(self._created_response, "service_tier", None) assembled: Final = ModelResponse( model=self.model, usage=usage, service_tier=served_tier if isinstance(served_tier, str) else None @@ -1892,7 +1890,7 @@ def stamp_responses_usage_cost( return if isinstance(bill, bool) or not isinstance(bill, (int, float)): return - if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): + if bill > 0 or isinstance(usage_obj.cost, (int, float)): stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill)) diff --git a/litellm/router.py b/litellm/router.py index 4ee138fc9fb..6a60d5d958e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11,7 +11,7 @@ outside code used before the entry points live in ``DeprecatedRouterMembers`` un import inspect import weakref from collections.abc import Mapping -from types import CellType, FunctionType, MappingProxyType +from types import FunctionType, MappingProxyType from typing import TYPE_CHECKING, Final from litellm import router_python_core @@ -35,7 +35,7 @@ from litellm.types.router import ( RetryPolicy, ) -_CORE_SIGNATURE: Final = inspect.signature(PythonCore.__init__) +_CORE_SIGNATURE: Final = inspect.signature(PythonCore.__init__) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # reflection reads only the parameters' names def router_arguments( @@ -65,142 +65,40 @@ else: _replaced_core_members: Final[ # mutable-ok: originals of patched core members, restored on delete dict[tuple[type, str], list[object]] ] = {} - _FACADE_ONLY: Final = frozenset( - { - "__module__", - "__qualname__", - "__doc__", - "__dict__", - "__weakref__", - "__init__", - "__init_subclass__", - "__annotations__", - "__firstlineno__", - "__static_attributes__", - "__classcell__", - "_core_class", - } - ) - - def _with_class_cell(value: object, cell: "CellType") -> object: - """``value`` with its ``__class__`` cell (what zero-argument ``super()`` reads) replaced by ``cell``.""" - if isinstance(value, staticmethod | classmethod): - return type(value)(_function_with_class_cell(value.__func__, cell)) - if isinstance(value, property): - return property( - *( - None if f is None else _function_with_class_cell(f, cell) - for f in (value.fget, value.fset, value.fdel) - ), - value.__doc__, - ) - return _function_with_class_cell(value, cell) - - def _function_with_class_cell(value: object, cell: "CellType") -> object: - """A plain function with its ``__class__`` cell replaced by ``cell``; anything else unchanged.""" - if not isinstance(value, FunctionType) or "__class__" not in value.__code__.co_freevars: - return value - closure: Final = tuple( - cell if name == "__class__" else original - for name, original in zip(value.__code__.co_freevars, value.__closure__ or (), strict=True) - ) - rebound: Final = FunctionType(value.__code__, value.__globals__, value.__name__, value.__defaults__, closure) - rebound.__kwdefaults__ = value.__kwdefaults__ - rebound.__qualname__ = value.__qualname__ - rebound.__dict__.update(value.__dict__) - return rebound - - class _Absent: - """A facade-only member, hidden from the core class: reading it raises the usual AttributeError.""" - - def __set_name__(self, owner: type, name: str) -> None: - self._name = name - - def __get__(self, instance: object, owner: type | None = None) -> object: - if instance is None: - raise AttributeError( - f"type object {getattr(owner, '__name__', owner)!r} has no attribute {self._name!r}" - ) - raise AttributeError(f"{type(instance).__name__!r} object has no attribute {self._name!r}") - - def _core_class_for(facade: type, parent: type) -> type: - """ - A core class carrying a Router subclass's own members, so the core runs its overrides. It - subclasses the facade too, so an override's ``super(Sub, self)`` finds the core it runs on. - """ - cell: Final = CellType() - members: Final = { - name: _with_class_cell(value, cell) for name, value in vars(facade).items() if name not in _FACADE_ONLY - } - namespace: Final = { - # The facade's own plumbing sits in the core class's MRO ahead of the parent core: put - # the parent core's versions back, and hide the facade-only members. - "__init__": parent.__init__, - "__getattribute__": object.__getattribute__, - "__setattr__": parent.__setattr__, - "__delattr__": parent.__delattr__, - "__dir__": parent.__dir__, - **{name: _Absent() for name in ("_attach", "_core", "from_config", "_core_class")}, - **members, - "__module__": facade.__module__, - } - core_class: Final = _CoreType(f"{facade.__name__}Core", (facade, parent), namespace) - cell.cell_contents = core_class - for name in members: - type.__delattr__(facade, name) - return core_class class _RouterType(type): """ - Class-level reads of core members (``Router._helper(...)``) go to the router's core class, and - so do class-level patches, which put the original back when the patch is removed. - ``isinstance`` and ``dir`` see the core class too. + Class-level reads of core members (``Router._helper(...)``) go to ``PythonCore``, and so do + class-level patches, which put the original back when the patch is removed. + ``isinstance`` and ``dir`` see ``PythonCore`` too. """ def __getattr__(cls, name: str) -> object: - if name == "_core_class": - raise AttributeError(name) - return getattr(cls._core_class, name) + return getattr(PythonCore, name) def __setattr__(cls, name: str, value: object) -> None: - core_class: Final = vars(cls).get("_core_class") or getattr(cls, "_core_class", None) - if core_class is None or any(name in vars(base) for base in cls.__mro__) or not hasattr(core_class, name): + if any(name in vars(base) for base in cls.__mro__) or not hasattr(PythonCore, name): super().__setattr__(name, value) return - _replaced_core_members.setdefault((core_class, name), []).append(vars(core_class).get(name, _MISSING)) - setattr(core_class, name, value) + _replaced_core_members.setdefault((PythonCore, name), []).append(vars(PythonCore).get(name, _MISSING)) + setattr(PythonCore, name, value) def __delattr__(cls, name: str) -> None: - core_class: Final = getattr(cls, "_core_class", None) - replaced: Final = _replaced_core_members.get((core_class, name)) if core_class is not None else None + replaced: Final = _replaced_core_members.get((PythonCore, name)) if not replaced or any(name in vars(base) for base in cls.__mro__): super().__delattr__(name) return original: Final = replaced.pop() if original is _MISSING: - delattr(core_class, name) + delattr(PythonCore, name) return - setattr(core_class, name, original) + setattr(PythonCore, name, original) def __instancecheck__(cls, instance: object) -> bool: - return type.__instancecheck__(cls, instance) or isinstance(instance, cls._core_class) + return type.__instancecheck__(cls, instance) or isinstance(instance, PythonCore) def __dir__(cls) -> list[str]: # mutable-ok: dir() returns a list - return sorted({*type.__dir__(cls), *dir(cls._core_class)}) - - class _CoreType(_RouterType): - """The metaclass of a Router subclass's core class: a plain class, none of the facade's redirects.""" - - def __getattr__(cls, name: str) -> object: - raise AttributeError(f"type object {cls.__name__!r} has no attribute {name!r}") - - __setattr__ = type.__setattr__ - __delattr__ = type.__delattr__ - __instancecheck__ = type.__instancecheck__ - __dir__ = type.__dir__ - - def _is_core(instance: object) -> bool: - return isinstance(type(instance), _CoreType) + return sorted({*type.__dir__(cls), *dir(PythonCore)}) def _patchable_method(core: object, name: str) -> bool: """A method of the core: an instance-level replacement of it is also kept on the facade, so @@ -212,27 +110,20 @@ else: return hasattr(type(core), "__getattr__") and inspect.ismethod(getattr(core, name, None)) class Router(DeprecatedRouterMembers, metaclass=_RouterType): - _core_class: type[PythonCore] = PythonCore - - def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() - if isinstance(cls, _CoreType): - # A core class built below: the parent core's hooks wire it, as for any core class. - super().__init_subclass__(**kwargs) - return - # DeprecatedRouterMembers wires a subclass's overrides of old members; here that runs - # on the core class built from it, which is where the router's own calls land. - super(DeprecatedRouterMembers, cls).__init_subclass__(**kwargs) - type.__setattr__(cls, "_core_class", _core_class_for(cls, cls._core_class)) + def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, refused with the class + raise TypeError( + "litellm.Router cannot be subclassed. Use the router's entry points instead: " + "register_plugin() for custom routing, apply() for settings, observe() for outcomes. " + "See https://docs.litellm.ai/docs/routing_plugins." + ) def __init__(self, *args: object, **kwargs: object) -> None: # kwargs-ok: Router's constructor arguments - self._attach( - make_core(build_router_config(router_arguments(*args, **kwargs)), python_host, type(self)._core_class) - ) + self._attach(make_core(build_router_config(router_arguments(*args, **kwargs)), python_host)) @classmethod def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": router: Final = cls.__new__(cls) - router._attach(make_core(config, host, cls._core_class)) + router._attach(make_core(config, host)) return router def _attach(self, core: RouterCore) -> None: @@ -253,8 +144,7 @@ else: try: return object.__getattribute__(self, name) except AttributeError: - if _is_core(self): - raise + pass facade: Final = object.__getattribute__(self, "__dict__") core: Final = facade.get("_core") if core is None: @@ -273,9 +163,6 @@ else: ) from None def __setattr__(self, name: str, value: object) -> None: - if _is_core(self): - super().__setattr__(name, value) - return if _class_descriptor(type(self), name): object.__setattr__(self, name, value) return @@ -289,9 +176,6 @@ else: self.__dict__[name] = value def __delattr__(self, name: str) -> None: - if _is_core(self): - super().__delattr__(name) - return if _class_descriptor(type(self), name): object.__delattr__(self, name) return @@ -299,9 +183,7 @@ else: delattr(self._core, name) def __dir__(self) -> list[str]: # mutable-ok: dir() returns a list - if _is_core(self): - return super().__dir__() - return sorted({*super().__dir__(), *dir(self.__dict__.get("_core", type(self)._core_class))}) + return sorted({*super().__dir__(), *dir(self.__dict__.get("_core", PythonCore))}) Router.__init__.__signature__ = _CORE_SIGNATURE.replace( # pyright: ignore[reportFunctionMemberAccess] # inspect reads it parameters=[parameter for parameter in _CORE_SIGNATURE.parameters.values() if parameter.name != "_host"] diff --git a/litellm/router_api.py b/litellm/router_api.py index 3007b5345c7..82660439ae9 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -32,6 +32,7 @@ as they were. # members are still untyped (bare ``dict`` / ``list``) or mutable (the live objects); the overloads # repeat them unchanged, and typing the members themselves is the core's job. +import copy import hashlib import inspect import json @@ -45,6 +46,7 @@ from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload from litellm.router_config import detached if TYPE_CHECKING: + from functools import _lru_cache_wrapper from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type from opentelemetry.trace import Span @@ -179,9 +181,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id`` set) only by that team, any other deployment by everyone. Needs no router instance. """ - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.registry import HostRegistry - return PythonCore._core_deployment_usable_by_team(deployment, team_id) + return HostRegistry._core_deployment_usable_by_team(deployment, team_id) def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str: @@ -201,11 +203,77 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str return hashlib.sha256("".join(parts).encode()).hexdigest() +# The arguments ``Router.__init__`` accepts, in its signature's order. Every core reads this list +# (the Python core's ``get_valid_args`` reads its constructor's signature, and the core suite checks +# that the two agree). +ROUTER_INIT_ARGS: Final[tuple[str, ...]] = ( + "model_list", + "assistants_config", + "search_tools", + "guardrail_list", + "redis_url", + "redis_host", + "redis_port", + "redis_password", + "redis_db", + "cache_responses", + "cache_kwargs", + "caching_groups", + "client_ttl", + "polling_interval", + "default_priority", + "num_retries", + "max_fallbacks", + "timeout", + "stream_timeout", + "default_litellm_params", + "default_max_parallel_requests", + "set_verbose", + "debug_level", + "default_fallbacks", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "model_group_alias", + "enable_pre_call_checks", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "plugins", + "retry_after", + "retry_policy", + "model_group_retry_policy", + "allowed_fails", + "allowed_fails_policy", + "cooldown_time", + "disable_cooldowns", + "fallback_on_unsupported_params", + "routing_strategy", + "optional_pre_call_checks", + "routing_strategy_args", + "routing_groups", + "provider_budget_config", + "alerting_config", + "router_general_settings", + "deployment_affinity_ttl_seconds", + "model_group_affinity_config", + "ignore_invalid_deployments", + "enable_health_check_routing", + "health_check_staleness_threshold", + "health_check_ignore_transient_errors", + "background_health_check_model_groups", + "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", + "fallback_access_check", + "fallback_budget_check", + "auto_router_capability_limit", + "session_affinity_repin_on_failover", +) + + def router_init_args() -> tuple[str, ...]: """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance.""" - from litellm.router_python_core import PythonCore - - return tuple(PythonCore.get_valid_args()) + return ROUTER_INIT_ARGS def mark_advisor_orchestration_failure(exception: BaseException) -> None: @@ -269,9 +337,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob def _json_default_stable_id(obj: object) -> str: - from litellm.router_python_core import PythonCore + from litellm.router_rust.steps.prerouting import _json_default_stable_id as stable_id - return PythonCore._json_default_stable_id(obj) + return stable_id(obj) class RouterRegistry: @@ -316,6 +384,13 @@ class RouterRegistry: return self._router._core_cached_deployment_model_info(id, model_name) return self._router._core_get_deployment_model_info(model_id=id, model_name=model_name) + def deployment_model_info_cache(self) -> "_lru_cache_wrapper[ModelMapInfo | None]": + """ + The lru cache ``deployment_model_info(..., cached=True)`` reads, called as ``(id, model_name)`` + and cleared with ``cache_clear()`` (was the ``cached_deployment_model_info`` attribute). + """ + return self._router._core_cached_deployment_model_info + def router_model_info( self, deployment: "dict[str, object] | Deployment | None", # mutable-ok: the core takes a dict @@ -532,10 +607,8 @@ class RouterRegistry: ``should_include_deployment``), or, without ``model_name``, whether the team may use it at all (was ``deployment_usable_by_team``). """ - from litellm.router_python_core import PythonCore - if model_name is None: - return PythonCore._core_deployment_usable_by_team(deployment, team_id) + return deployment_usable_by_team(deployment, team_id) return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) @overload @@ -1074,13 +1147,22 @@ PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"}) _MISSING: Final = object() +# The settings fields ``RouterSettings`` copies, so an in-place edit of what a read returns never +# reaches the router (only ``apply()`` changes settings). +_COPIED_SETTING_FIELDS: Final = ( + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "default_litellm_params", +) @dataclass(frozen=True) class RouterSettings: """ The router's configuration as one frozen snapshot (``settings()``). Fields hold the router's - own objects, not copies: mutating a list in here mutates the router's. + own objects, except the fallback lists and ``default_litellm_params``, which are copies: settings + change only through ``apply()``, so editing one of those in place changes nothing on any core. A field the router never set reads as ``None``. Each field has the type of the Router attribute it reads (``_core_``), the type the old @@ -1119,6 +1201,12 @@ class RouterSettings: _router: "Router | None" = field(default=None, repr=False, compare=False) _unset: frozenset[str] = field(default=frozenset(), repr=False, compare=False) + def __post_init__(self) -> None: + for name in _COPIED_SETTING_FIELDS: + value: object = getattr(self, name) # pyright: ignore[reportAny] # a settings field + if isinstance(value, list | dict): + object.__setattr__(self, name, copy.deepcopy(value)) # pyright: ignore[reportUnknownArgumentType] # a settings list or dict + @classmethod def of(cls, router: "Router") -> "RouterSettings": read: Final = {name: getattr(router, "_core_" + name, _MISSING) for name in SETTING_FIELDS} @@ -1188,16 +1276,16 @@ class RouterHealth: @staticmethod def is_blocked(deployment: "Deployment") -> bool: """Whether an admin paused this deployment (was ``is_deployment_blocked``).""" - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.health_routing import is_blocked as deployment_is_blocked - return PythonCore._core_is_deployment_blocked(deployment) + return deployment_is_blocked(deployment) @staticmethod def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``).""" - from litellm.router_python_core import PythonCore + from litellm.router_rust.surface.health_routing import all_blocked as deployments_all_blocked - return PythonCore._core_are_all_deployments_blocked(deployments) + return deployments_all_blocked(deployments) def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``).""" @@ -1402,6 +1490,9 @@ class RoutingContext: def deployment_params(self, request: Mapping[str, object], deployment: "Deployment | None") -> Mapping[str, object]: return self._router._params_sent_to_deployment(request, deployment) + def default_litellm_params(self) -> Mapping[str, object]: + return self._router._active_default_litellm_params() + def settings(self) -> RouterSettings: return self._router.settings() diff --git a/litellm/router_attempt.py b/litellm/router_attempt.py new file mode 100644 index 00000000000..33e0a19c530 --- /dev/null +++ b/litellm/router_attempt.py @@ -0,0 +1,215 @@ +""" +One router attempt, as the host runs it: the request for the deployment the router core picked. + +The Rust router core picks a deployment and hands the attempt to its host; ``PythonHost`` builds +the keyword arguments for that one deployment here and calls ``litellm.`` with them. Nothing +here selects, retries, falls back or records cooldowns: the core does all of that. Nothing here +reads a Python router either, so this module outlives ``router_python_core.py``. +""" + +import re +import uuid +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter +from litellm.litellm_core_utils.routing_common_utils import without_router_only_kwargs + +# Request keys the router reads to route the call; the deployment call never receives them. +ROUTING_KEYS: Final = frozenset( + { + "model", + "num_retries", + "disable_fallbacks", + "specific_deployment", + "_router_weights", + "_retry_skipped_deployment_ids", + "_target_order", + "model_group_retry_policy", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + } +) + +# The positional parameters of each endpoint, in litellm's order: a call that passes them +# positionally is read back into keywords before it is routed. +POSITIONAL: Final = MappingProxyType( + { + "acompletion": ("model", "messages"), + "completion": ("model", "messages"), + "atext_completion": ("model", "prompt"), + "text_completion": ("model", "prompt"), + "aembedding": ("model", "input"), + "embedding": ("model", "input"), + "aimage_generation": ("prompt", "model"), + "image_generation": ("prompt", "model"), + "atranscription": ("model", "file"), + "aspeech": ("model", "input", "voice"), + "arerank": ("model", "query", "documents"), + "amoderation": ("input", "model"), + "aresponses": ("input", "model"), + "aanthropic_messages": (), + } +) + + +def request_from_call(op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The call's arguments as one keyword request. Raises ``TypeError`` like the endpoint would.""" + names: Final = POSITIONAL.get(op, ()) + if len(args) > len(names): + raise TypeError(f"{op}() takes {len(names)} positional arguments but {len(args)} were given") + request: Final = dict(kwargs) + for name, value in zip(names, args, strict=False): + if name in request: + raise TypeError(f"{op}() got multiple values for argument {name!r}") + request[name] = value + return request + + +# Endpoints the Python router serves through ``_ageneric_api_call_with_fallbacks``, which writes +# the routing metadata under ``litellm_metadata``. +_GENERIC_API_OPS: Final = frozenset({"aanthropic_messages", "aresponses"}) + + +def metadata_field(op: str) -> str: + """``metadata`` or ``litellm_metadata``: where the router writes what it routed the call as.""" + return "litellm_metadata" if op in _GENERIC_API_OPS else get_router_metadata_variable_name(function_name=op) + + +def prepare_call(op: str, request: Mapping[str, object], model_group_alias: str | None) -> Mapping[str, object]: + """What the router adds to a request once, before its first attempt + (``_update_kwargs_before_fallbacks``): a trace id and the model group it was asked for.""" + field: Final = metadata_field(op) + metadata: Final = dict(_as_mapping(request.get(field))) + metadata.update({"model_group": request.get("model"), "model_group_alias": model_group_alias}) + return {**request, "litellm_trace_id": request.get("litellm_trace_id") or str(uuid.uuid4()), field: metadata} + + +def _wildcard_match(pattern: object, requested: str) -> "re.Match[str] | None": + if not isinstance(pattern, str) or "*" not in pattern: + return None + regex: Final = PatternMatchRouter().pattern_to_regex(pattern) + return re.match(regex, requested) or re.match(regex, f"{inferred_provider(requested)}/{requested}") + + +def for_requested_model(deployment: Mapping[str, object], requested: str | None) -> Mapping[str, object]: + """A wildcard deployment as the router serves ``requested`` through it: its ``model`` with + the wildcards filled in from the requested name (``PatternMatchRouter.route``), by the team's + public pattern (``_core_team_pattern_routers``) or its own. Any other deployment comes back + unchanged.""" + params: Final = _as_mapping(deployment.get("litellm_params")) + model: Final = params.get("model") + if requested is None or not isinstance(model, str): + return deployment + public_name: Final = _as_mapping(deployment.get("model_info")).get("team_public_model_name") + match: Final = _wildcard_match(public_name, requested) or _wildcard_match(deployment.get("model_name"), requested) + if match is None: + return deployment + return { + **deployment, + "litellm_params": {**params, "model": PatternMatchRouter.set_deployment_model_name(match, model)}, + } + + +def attempt_kwargs( + op: str, + request: Mapping[str, object], + deployment: Mapping[str, object], + *, + model_group: str, + default_litellm_params: Mapping[str, object], + timeout: float | None, +) -> Mapping[str, object]: + """The keyword arguments for ``litellm.`` against one deployment + (``_update_kwargs_with_deployment`` and the call sites that merge ``litellm_params``).""" + params: Final = _as_mapping(deployment.get("litellm_params")) + model_info: Final = _model_info_for(request, deployment, model_group) + field: Final = metadata_field(op) + metadata: Final = dict(_as_mapping(request.get(field))) + metadata.update( + { + "model_group": model_group, + "deployment": params.get("model"), + "model_info": model_info, + "api_base": params.get("api_base"), + "deployment_model_name": deployment.get("model_name"), + } + ) + tags: Final = _as_list(params.get("tags")) + if tags: + merged: Final = list(_as_list(metadata.get("tags"))) + merged.extend(tag for tag in tags if tag not in merged) + metadata["tags"] = merged + call: Final = {key: value for key, value in request.items() if key not in ROUTING_KEYS} + for key, value in default_litellm_params.items(): + call.setdefault(key, value) + call[field] = metadata + call["model_info"] = model_info + if "timeout" not in call: + deployment_timeout: Final = params.get("timeout") + chosen: Final = deployment_timeout if deployment_timeout is not None else timeout + if chosen is not None: + call["timeout"] = chosen + return without_router_only_kwargs({**params, **call, "model": params.get("model")}) + + +def _model_info_for( + request: Mapping[str, object], deployment: Mapping[str, object], model_group: str +) -> dict[str, object]: # mutable-ok: a fresh dict for the call + """The deployment's ``model_info``, or, for a request carrying its own credentials, the + request-scoped deployment's (``_handle_clientside_credential``): its own stable id, and the + configured deployment's id as ``original_model_id``.""" + from litellm.litellm_core_utils import clientside_credential_handler as clientside + from litellm.router_api import deployment_id + + model_info: Final = dict(_as_mapping(deployment.get("model_info"))) + if not clientside.is_clientside_credential(request_kwargs=dict(request)): # pyright: ignore[reportUnknownMemberType] # an untyped helper + return model_info + dynamic: Final = _as_mapping( + clientside.get_dynamic_litellm_params( # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an untyped helper + litellm_params=dict(_as_mapping(deployment.get("litellm_params"))), request_kwargs=dict(request) + ) + ) + original: Final = model_info.get("id") + model_info["id"] = deployment_id(model_group, dynamic) + model_info["original_model_id"] = original + return model_info + + +def stamp_attempt_error(error: BaseException, deployment: Mapping[str, object]) -> None: + """What the router stamps on a failed attempt's error: the deployment's own ``num_retries`` + (``_set_deployment_num_retries_on_exception``) and its id (``failed_deployment_id``).""" + params: Final = _as_mapping(deployment.get("litellm_params")) + if getattr(error, "num_retries", None) is None and params.get("num_retries") is not None: + try: + error.num_retries = int(str(params["num_retries"])) # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the router stamps the error in place + except ValueError: + pass + deployment_id: Final = _as_mapping(deployment.get("model_info")).get("id") + if deployment_id and not getattr(error, "failed_deployment_id", None): + try: + error.failed_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: see above + except AttributeError: + pass + + +def _as_mapping(value: object) -> Mapping[str, object]: + if isinstance(value, Mapping): + entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # a request's untyped mapping + return {str(key): item for key, item in entries.items()} + return MappingProxyType({}) + + +def _as_list(value: object) -> list[object]: # mutable-ok: a fresh list + if isinstance(value, list): + items: Final[list[object]] = value # pyright: ignore[reportUnknownVariableType] # a request's untyped list # mutable-ok: read, then copied + return list(items) + return [] diff --git a/litellm/router_config.py b/litellm/router_config.py index 75c79068191..36154af296c 100644 --- a/litellm/router_config.py +++ b/litellm/router_config.py @@ -79,7 +79,12 @@ def detached(value: object) -> object: def build_router_config(arguments: Mapping[str, object]) -> RouterConfig: - """The constructor arguments, detached from the caller's objects: settings are static.""" + """The constructor arguments, detached from the caller's objects: settings are static. + + A bool `cooldown_time` is refused: `True` is not a number of seconds, and + reading it as 1 s or as the default would both be a guess.""" + if isinstance(arguments.get("cooldown_time"), bool): + raise TypeError("Router cooldown_time must be a number of seconds, not a bool") return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()})) diff --git a/litellm/router_core.py b/litellm/router_core.py index acbe82a0643..c077b3f0015 100644 --- a/litellm/router_core.py +++ b/litellm/router_core.py @@ -1,14 +1,16 @@ """ ``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one. -``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core -(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when -the router is built, from its ``RouterConfig``. +``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. ``RustCore`` +(``litellm/router_rust/core.py`` over litellm-rust/crates/router-core) is a separate implementation +that reaches outside itself only through its host; it does not use ``PythonCore``. The core is +picked once, when the router is built, from its ``RouterConfig``. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable +from litellm._logging import verbose_router_logger from litellm.router_config import RouterConfig if TYPE_CHECKING: @@ -20,7 +22,6 @@ if TYPE_CHECKING: RouterSettings, RoutingObservers, ) - from litellm.router_python_core import PythonCore from litellm.router_python_host import PythonHost from litellm.types.router import CustomRoutingStrategyBase @@ -50,7 +51,26 @@ class RouterCore(Protocol): def retire(self) -> None: ... -def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore: +def make_core(config: RouterConfig, host: "PythonHost") -> RouterCore: + """ + The Rust core when the route catalog's router rule allows it and the Rust core serves this + config, else the Python core. Decided once, for the router's life, from the config alone: the + Rust core never builds, wraps or falls back to a Python core. + """ from litellm.router_python_core import PythonCore + from litellm.rust_bridge.catalog import Route, RouteContext, decision + from litellm.rust_bridge.configuration import Decision - return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature + chosen: Final = decision(RouteContext(Route.ROUTER)) + if chosen is not Decision.PYTHON: + from litellm.router_rust.core import RustCore + from litellm.rust_bridge._native import RustBridgeDeclined + + try: + return RustCore.build(config, host) # pyright: ignore[reportReturnType] # RustCore serves the RouterCore entry points; other members raise RustCoreUnsupported + except RustBridgeDeclined as declined: + # A config the Rust core cannot take keeps the router on Python while both cores exist. + if chosen is Decision.RUST_REQUIRED: + raise + verbose_router_logger.info("Router served by the Python core: %s", declined) + return PythonCore(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 83a1251d3c0..9d915fe0982 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -5,8 +5,10 @@ DeprecatedRouterMembers: the Router members that outside code used before the en entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``, ``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here returns what it returned before, with the type it had before, and names its replacement in a -``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_``) the -entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves. +``DeprecationWarning``. Reads go through that entry point, so they work on either router core +(Python or Rust) and, on the Python core, return the same ``_core_`` object as before. +Settings writes go through ``apply`` / ``aapply`` so the registry version moves; any other old +attribute write sets the core's ``_core_``, which each core keeps where its reads find it. Access from the router itself and from its own modules (``litellm.router*``) is not counted: those are the router's internals and move behind the core interface separately. Every other access @@ -17,21 +19,21 @@ deleted (planned about a month after the core swap). # pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false # pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false # pyright: reportUnknownLambdaType=false -# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do) +# This shim is part of the router (it writes Router's ``_core_*`` members, as Router's own methods do) # and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so # callers keep the types they had. Typing those parts is the core's job, not the shim's. -import contextvars import functools import inspect import sys import warnings from collections import Counter -from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload +from typing import TYPE_CHECKING, Final, Generic, ParamSpec, TypeVar, overload -from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet +from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet, RoutingContext +from litellm.router_strategy.least_busy import picks_reserve if TYPE_CHECKING: from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged @@ -39,6 +41,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span from litellm.caching.caching import RedisCache + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -58,6 +61,7 @@ if TYPE_CHECKING: from litellm.types.utils import ModelInfo as ModelMapInfo _T: Final = TypeVar("_T") +_P: Final = ParamSpec("_P") DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments @@ -129,6 +133,9 @@ class _DeprecatedAttribute(Generic[_T]): _note(self.name, self.replacement) router: Final = _as_router(obj) try: + if obj.__dict__.get("_core") is None: + # No core yet (a Router made without __init__): the value written through this name. + return getattr(obj, "_core_" + self.name) # pyright: ignore[reportAny] # the pending write, as given return self.read(router) except AttributeError as e: raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None @@ -164,243 +171,186 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib return _DeprecatedAttribute(replacement, read) -# (id(router), member name) pairs whose subclass override is running in this context. -_OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar( - "router_overrides_running", default=frozenset() -) +def _without_reserving(lookup: Callable[_P, Awaitable[_T]]) -> Callable[_P, Coroutine[object, object, _T]]: + @functools.wraps(lookup) + async def looked_up( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: forwards the wrapped lookup's ParamSpec unchanged + ) -> _T: + with picks_reserve(False): + return await lookup(*args, **kwargs) + + return looked_up -def _override_key(router: object, name: str) -> tuple[int, str]: - """The running-override key for ``router``: a Router and its core share one, so an old member - entered on the Router is seen by the override routing that runs on the core.""" - core: Final = getattr(router, "__dict__", {}).get("_core") - return (id(router if core is None else core), name) - - -def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]: +def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]: """ - A direct call of a subclass override (``router.()``) marks it running, so its - ``super().()`` reaches the parent once instead of re-entering the override through - ``_core_``. + The router's async pick (the old ``async_get_available_deployment`` attribute). A core that + keeps its pick as a member (the Python core's, or one assigned through this attribute) returns + that member, so a patched pick reads back as patched; any other core picks through ``select``. """ - if inspect.iscoroutinefunction(override): - async_override: Final[Callable[..., Awaitable[object]]] = override + member: Final[object] = getattr(router, "_core_async_get_available_deployment", None) + if callable(member): + return member # pyright: ignore[reportReturnType] # the core's own async pick - @functools.wraps(override) - async def entered_async( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) - try: - return await async_override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return entered_async - - @functools.wraps(override) - def entered( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + async def async_get_available_deployment( + model: str, + request_kwargs: dict, # mutable-ok: the old parameter type, unchanged + messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged + input: str | list | None = None, # mutable-ok: the old parameter type, unchanged + specific_deployment: bool | None = False, ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) - try: - return override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) + return await router.select( + model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment + ) - return entered + return async_get_available_deployment -def _private_alias(name: str, member: Callable[..., object]) -> Callable[..., object]: +def _unreserved_picker(router: "Router") -> Callable[..., Awaitable[object]]: + """``_async_picker``, picking without holding a least-busy reservation (a lookup makes no + call). A pick assigned through the attribute reads back as the object assigned.""" + picker: Final = _async_picker(router) + if inspect.ismethod(picker) or picker.__name__ == "async_get_available_deployment": + return _without_reserving(picker) + return picker + + +def _slack_alerting_logger(router: "Router") -> "SlackAlerting": + """``observe().slack_alerting_logger``; a router without one has no such attribute, as before.""" + logger: Final = router.observe().slack_alerting_logger + if logger is None: + raise AttributeError( + f"'{type(router).__name__}' object has no attribute 'slack_alerting_logger'", + name="slack_alerting_logger", + obj=router, + ) + return logger + + +def _run_ready(step: Coroutine[object, object, _T]) -> _T: """ - A private alias of an old member (``_get_all_deployments``). It was bound to Router's own - method, so it never reached a subclass override of ``name``, and still doesn't. + The result of a coroutine that finishes without suspending: ``select(kind='pass_through')`` + picks synchronously on every core, so the old synchronous member can return its result. """ - return _enter_override(name, member) - - -def _route_override(cls: type, name: str, override: Callable[..., object]) -> Callable[..., object]: - """ - Make the router's own calls to ``_core_`` reach a subclass override of ````. - - When the override calls ``super().()``, the call comes back to ``_core_``; it is - then sent on to the parent's implementation instead of back into the override. - """ - core: Final = "_core_" + name - - def parent(router: object) -> Callable[..., object]: - return getattr(super(cls, router), core) - - if inspect.iscoroutinefunction(override): - async_override: Final[Callable[..., Awaitable[object]]] = override - - @functools.wraps(override) - async def routed_async( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - key: Final = _override_key(router, name) - running: Final = _OVERRIDES_RUNNING.get() - if key in running: - parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core) - return await parent_async(*args, **kwargs) - token: Final = _OVERRIDES_RUNNING.set(running | {key}) - try: - return await async_override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return routed_async - - @functools.wraps(override) - def routed( - router: object, - *args: object, - **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged - ) -> object: - key: Final = _override_key(router, name) - running: Final = _OVERRIDES_RUNNING.get() - if key in running: - return parent(router)(*args, **kwargs) - token: Final = _OVERRIDES_RUNNING.set(running | {key}) - try: - return override(router, *args, **kwargs) - finally: - _OVERRIDES_RUNNING.reset(token) - - return routed + try: + step.send(None) + except StopIteration as done: + return done.value # pyright: ignore[reportAny] # the coroutine's own result + step.close() + raise RuntimeError("select(kind='pass_through') suspended; it must pick without awaiting") class DeprecatedRouterMembers: """Old Router members, kept for about a month. Each names its replacement.""" - def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() - # A Router subclass that overrides an old member (say async_pre_routing_hook) keeps - # having its override called by the router, which now calls _core_ internally. - super().__init_subclass__(**kwargs) - for name, value in list(vars(cls).items()): - if ( - name.startswith("_core_") - or name not in vars(DeprecatedRouterMembers) - or not hasattr(cls, "_core_" + name) - ): - continue - if isinstance(value, staticmethod): - # Its super() call reaches the old member here, which calls Router's own implementation. - setattr(cls, "_core_" + name, value) - elif inspect.isfunction(value): - setattr(cls, "_core_" + name, _route_override(cls, name, value)) - setattr(cls, name, _enter_override(name, value)) - # -- registry: attributes -------------------------------------------------------------------- - model_list = _attr("registry().all()", lambda r: r._core_model_list) - model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names) - deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names) - team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names) + model_list = _attr("registry().all()", lambda r: r.registry().all()) + model_names = _attr("registry().names(kind='group_set')", lambda r: r.registry().names(kind="group_set")) + deployment_names = _attr( + "registry().names(kind='deployment_names')", lambda r: r.registry().names(kind="deployment_names") + ) + team_public_model_names = _attr( + "registry().names(kind='team_public')", lambda r: r.registry().names(kind="team_public") + ) model_name_to_deployment_indices = _attr( - "registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices + "registry().index('model_name')", lambda r: r.registry().index("model_name") ) team_model_to_deployment_indices = _attr( - "registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices + "registry().index('team_model')", lambda r: r.registry().index("team_model") ) cached_deployment_model_info = _attr( "registry().deployment_model_info(id, model_name, cached=True)", - lambda r: r._core_cached_deployment_model_info, + lambda r: r.registry().deployment_model_info_cache(), ) - model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias) - pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router) - team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers) - routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups) + model_group_alias = _attr("registry().aliases()", lambda r: r.registry().aliases()) + pattern_router = _attr("registry().patterns()", lambda r: r.registry().patterns()) + team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r.registry().patterns(team=True)) + routing_groups = _attr("registry().routing_groups()", lambda r: r.registry().routing_groups()) _routing_groups = routing_groups - auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers) - complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers) - quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers) - adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers) - routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins) + auto_routers = _attr("registry().plugins('auto')", lambda r: r.registry().plugins("auto")) + complexity_routers = _attr("registry().plugins('complexity')", lambda r: r.registry().plugins("complexity")) + quality_routers = _attr("registry().plugins('quality')", lambda r: r.registry().plugins("quality")) + adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r.registry().plugins("adaptive")) + routing_plugins = _attr("registry().plugins('routing')", lambda r: r.registry().plugins("routing")) prompt_caching_pin_cache = _attr( - "registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache + "registry().plugins('prompt_caching_pins')", lambda r: r.registry().plugins("prompt_caching_pins") ) # -- health and selection: attributes -------------------------------------------------------- - cache = _attr("health().store", lambda r: r._core_cache) - cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache) - health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache) - deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map) - async_get_available_deployment = _attr( - "select(model, request_kwargs)", - lambda r: r._core_async_get_available_deployment, - ) + cache = _attr("health().store", lambda r: r.health().store) + cooldown_cache = _attr("health().cooldowns", lambda r: r.health().cooldowns) + health_state_cache = _attr("health().health_checks", lambda r: r.health().health_checks) + deployment_latency_map = _attr("health().latency", lambda r: r.health().latency) + async_get_available_deployment = _attr("select(model, request_kwargs)", lambda r: _unreserved_picker(r)) # -- observers ------------------------------------------------------------------------------ - slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger) + slack_alerting_logger = _attr("observe().slack_alerting_logger", _slack_alerting_logger) # -- settings ------------------------------------------------------------------------------- - routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy) - context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks) - content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks) - default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment) - default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params) + routing_strategy = _setting("routing_strategy", lambda r: r.settings().routing_strategy) + context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r.settings().context_window_fallbacks) + content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r.settings().content_policy_fallbacks) + default_deployment = _setting("default_deployment", lambda r: r.settings().default_deployment) + default_litellm_params = _setting("default_litellm_params", lambda r: r.settings().default_litellm_params) default_max_parallel_requests = _setting( - "default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests + "default_max_parallel_requests", lambda r: r.settings().default_max_parallel_requests + ) + cooldown_time = _setting("cooldown_time", lambda r: r.settings().cooldown_time) + allowed_fails = _setting("allowed_fails", lambda r: r.settings().allowed_fails) + allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r.settings().allowed_fails_policy) + disable_cooldowns = _setting("disable_cooldowns", lambda r: r.settings().disable_cooldowns) + enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r.settings().enable_pre_call_checks) + enable_health_check_routing = _setting( + "enable_health_check_routing", lambda r: r.settings().enable_health_check_routing ) - cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time) - allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails) - allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy) - disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns) - enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks) - enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing) health_check_ignore_transient_errors = _setting( - "health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors + "health_check_ignore_transient_errors", lambda r: r.settings().health_check_ignore_transient_errors ) background_health_check_model_groups = _setting( - "background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups + "background_health_check_model_groups", lambda r: r.settings().background_health_check_model_groups ) - enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering) - tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any) - tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix) - router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings) - provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config) + enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r.settings().enable_tag_filtering) + tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r.settings().tag_filtering_match_any) + tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r.settings().tag_routing_prefix) + router_general_settings = _setting("router_general_settings", lambda r: r.settings().router_general_settings) + provider_budget_config = _setting("provider_budget_config", lambda r: r.settings().provider_budget_config) provider_default_deployment_ids = _setting( - "provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids + "provider_default_deployment_ids", lambda r: r.settings().provider_default_deployment_ids ) - fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check) - fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check) + fallback_access_check = _setting("fallback_access_check", lambda r: r.settings().fallback_access_check) + fallback_budget_check = _setting("fallback_budget_check", lambda r: r.settings().fallback_budget_check) # These three are declared with the attribute type Router gave them rather than the descriptor's: # protocols in router_utils and the proxy match Router on them, and protocol matching does not # see through a descriptor. Reads and writes still go through _setting at runtime. fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type - "fallbacks", lambda r: r._core_fallbacks + "fallbacks", lambda r: r.settings().fallbacks ) kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting( # pyright: ignore[reportAssignmentType] # see above - "kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery + "kubernetes_pod_discovery", lambda r: r.settings().kubernetes_pod_discovery ) search_tools: "list[SearchToolTypedDict]" = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type - "search_tools", lambda r: r._core_search_tools + "search_tools", lambda r: r.settings().search_tools ) - guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list) - cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses) + guardrail_list = _setting("guardrail_list", lambda r: r.settings().guardrail_list) + cache_responses = _setting("cache_responses", lambda r: r.settings().cache_responses) # -- registry: one deployment --------------------------------------------------------------- def get_deployment(self, model_id: str) -> "Deployment | None": _note("get_deployment", "registry().deployment(id)") - return _as_router(self)._core_get_deployment(model_id=model_id) + return _as_router(self).registry().deployment(model_id) def has_model_id(self, candidate_id: str) -> bool: _note("has_model_id", "registry().has_deployment(id)") - return _as_router(self)._core_has_model_id(candidate_id) + return _as_router(self).registry().has_deployment(candidate_id) def get_model_info(self, id: str) -> dict | None: # mutable-ok: the old return type, unchanged _note("get_model_info", "registry().deployment_dict(id)") - return _as_router(self)._core_get_model_info(id=id) + return _as_router(self).registry().deployment_dict(id) def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None": _note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)") - return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name) + return _as_router(self).registry().deployment_model_info(model_id, model_name) @overload def get_router_model_info( @@ -420,62 +370,56 @@ class DeprecatedRouterMembers: id: str | None = None, ) -> "ModelMapInfo": _note("get_router_model_info", "registry().router_model_info(...)") - router: Final = _as_router(self) - # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged. - return router._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above - deployment=deployment, # pyright: ignore[reportArgumentType] # see above - received_model_name=received_model_name, - id=id, # pyright: ignore[reportArgumentType] # see above - ) + return _as_router(self).registry().router_model_info(deployment, received_model_name, id) def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: _note("get_discovered_model_info", "registry().discovered_model_info(id)") - return _as_router(self)._core_get_discovered_model_info(model_id) + return _as_router(self).registry().discovered_model_info(model_id) # -- registry: many deployments ------------------------------------------------------------- def get_model_list( self, model_name: str | None = None, team_id: str | None = None ) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged _note("get_model_list", "registry().deployments(model_name, team_id=...)") - return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id) + return _as_router(self).registry().deployments(model_name, team_id=team_id) def get_all_deployments( self, model_name: str, model_alias: str | None = None, team_id: str | None = None ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_all_deployments", "registry().deployments(model_name, source='all')") - return _as_router(self)._core_get_all_deployments( - model_name=model_name, model_alias=model_alias, team_id=team_id + return ( + _as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id) ) - _get_all_deployments = _private_alias("get_all_deployments", get_all_deployments) + _get_all_deployments = get_all_deployments def config_deployments(self) -> Iterator[Mapping[str, object]]: _note("config_deployments", "registry().deployments(source='config')") - return _as_router(self)._core_config_deployments() + return _as_router(self).registry().deployments(source="config") def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None": _note("deployments_named", "registry().deployments(model_name, source='named')") - return _as_router(self)._core_deployments_named(model_name) + return _as_router(self).registry().deployments(model_name, source="named") def get_model_list_from_model_alias( self, model_name: str | None = None ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')") - return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name) + return _as_router(self).registry().deployments(model_name, source="alias") def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]": _note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')") - return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name) + return _as_router(self).registry().deployments(model_name, source="routing_groups") def get_model_list_of_routed_group( self, model_group: str ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')") - return _as_router(self)._core_get_model_list_of_routed_group(model_group) + return _as_router(self).registry().deployments(model_group, source="routed_group") def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: _note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')") - return _as_router(self)._core_wildcard_route_deployments(model, team_id) + return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id) def get_routable_deployments( self, @@ -486,17 +430,21 @@ class DeprecatedRouterMembers: _note( "get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)" ) - return _as_router(self)._core_get_routable_deployments(model, request_kwargs, call_type) + return ( + _as_router(self) + .registry() + .deployments(model, source="routable", request=request_kwargs, call_type=call_type) + ) def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None": _note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')") - return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name) + return _as_router(self).registry().deployments(model_group_name, source="first_in_group") def deployments_for_request( self, model: str, request_kwargs: Mapping[str, object] ) -> "Sequence[DeploymentTypedDict]": _note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)") - return _as_router(self)._core_deployments_for_request(model, request_kwargs) + return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs) def should_include_deployment( self, @@ -505,135 +453,137 @@ class DeprecatedRouterMembers: team_id: str | None = None, ) -> bool: _note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)") - return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id) + return _as_router(self).registry().includes(model, model_name=model_name, team_id=team_id) @staticmethod def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool: _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)") - from litellm.router_python_core import PythonCore + from litellm.router_api import deployment_usable_by_team as usable_by_team - return PythonCore._core_deployment_usable_by_team(model, team_id) + return usable_by_team(model, team_id) _deployment_usable_by_team = deployment_usable_by_team # -- registry: names ------------------------------------------------------------------------- def get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: the old return type, unchanged _note("get_model_names", "registry().names(team_id=...)") - return _as_router(self)._core_get_model_names(team_id=team_id) + return _as_router(self).registry().names(team_id=team_id) def get_model_ids( self, model_name: str | None = None, exclude_team_models: bool = False ) -> list[str]: # mutable-ok: the old return type, unchanged _note("get_model_ids", "registry().names(model_name, kind='ids')") - return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models) + return _as_router(self).registry().names(model_name, kind="ids", exclude_team_models=exclude_team_models) def resolved_litellm_models( self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False ) -> tuple[str, ...]: _note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')") - return _as_router(self)._core_resolved_litellm_models( - model_name, team_id=team_id, use_base_model=use_base_model + return ( + _as_router(self) + .registry() + .names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id) ) def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: _note("wildcard_routed_models", "registry().names(model, kind='wildcard')") - return _as_router(self)._core_wildcard_routed_models(model, team_id) + return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id) def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: _note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')") - return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id) + return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id) # -- registry: resolving names --------------------------------------------------------------- def routable_model_group(self, model_name: str) -> str: _note("routable_model_group", "registry().resolve(name)") - return _as_router(self)._core_routable_model_group(model_name) + return _as_router(self).registry().resolve(model_name) def get_model_from_alias(self, model: str) -> str | None: _note("get_model_from_alias", "registry().resolve(name, kind='alias')") - return _as_router(self)._core_get_model_from_alias(model=model) + return _as_router(self).registry().resolve(model, kind="alias") - _get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias) + _get_model_from_alias = get_model_from_alias def resolve_served_model_name(self, model_name: str, served_model: str) -> str: _note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)") - return _as_router(self)._core_resolve_served_model_name(model_name, served_model) + return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model) def resolve_model_name_from_model_id( self, model_id: str | None, custom_llm_provider: str | None = None ) -> str | None: _note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')") - return _as_router(self)._core_resolve_model_name_from_model_id( - model_id, custom_llm_provider=custom_llm_provider - ) + return _as_router(self).registry().resolve(model_id, kind="from_id", custom_llm_provider=custom_llm_provider) def map_team_model( self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None ) -> str | None: _note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)") - return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider) + return ( + _as_router(self) + .registry() + .resolve(team_model_name, kind="team_model", team_id=team_id, custom_llm_provider=custom_llm_provider) + ) def is_recognized_model(self, model: str) -> bool: _note("is_recognized_model", "registry().resolve(name, kind='recognized')") - return _as_router(self)._core_is_recognized_model(model) + return _as_router(self).registry().resolve(model, kind="recognized") def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: _note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')") - return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id) + return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id) def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: _note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')") - return _as_router(self)._core_get_search_tool_support(model_name, team_id=team_id) + return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id) def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged _note("get_model_group", "registry().resolve(id, kind='group_of_id')") - return _as_router(self)._core_get_model_group(id=id) + return _as_router(self).registry().resolve(id, kind="group_of_id") def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: _note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')") - return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group) + return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group") - _is_model_access_group_for_wildcard_route = _private_alias( - "is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route - ) + _is_model_access_group_for_wildcard_route = is_model_access_group_for_wildcard_route def team_model_has_alternatives(self, deployment_id: str) -> bool: _note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')") - return _as_router(self)._core_team_model_has_alternatives(deployment_id) + return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives") def routing_group_has_alternatives(self, model_group: str | None) -> bool: _note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')") - return _as_router(self)._core_routing_group_has_alternatives(model_group) + return _as_router(self).registry().resolve(model_group, kind="routing_group_alternatives") # -- registry: groups, access groups, credentials, plugins ----------------------------------- def get_model_group_info( self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None ) -> "ModelGroupInfo | None": _note("get_model_group_info", "registry().group(name)") - return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id) + return _as_router(self).registry().group(model_group, include_hidden=include_hidden, team_id=team_id) def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None": _note("cached_model_group_info", "registry().group(name, field='cached_info')") - return _as_router(self)._core_cached_model_group_info(model_group) + return _as_router(self).registry().group(model_group, field="cached_info") def get_configured_display_name(self, model_name: str) -> str | None: _note("get_configured_display_name", "registry().group(name, field='display_name')") - return _as_router(self)._core_get_configured_display_name(model_name) + return _as_router(self).registry().group(model_name, field="display_name") def get_configured_mode(self, model_name: str) -> str | None: _note("get_configured_mode", "registry().group(name, field='mode')") - return _as_router(self)._core_get_configured_mode(model_name) + return _as_router(self).registry().group(model_name, field="mode") def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: _note("get_configured_service_tiers", "registry().group(name, field='service_tiers')") - return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id) + return _as_router(self).registry().group(model_name, field="service_tiers", team_id=team_id) def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]: _note("get_configured_token_limits", "registry().group(name, field='token_limits')") - return _as_router(self)._core_get_configured_token_limits(model_name) + return _as_router(self).registry().group(model_name, field="token_limits") def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None": _note("get_model_listing_info", "registry().group(name, field='listing_info')") - return _as_router(self)._core_get_model_listing_info(model_name) + return _as_router(self).registry().group(model_name, field="listing_info") def get_model_access_groups( self, @@ -642,31 +592,31 @@ class DeprecatedRouterMembers: team_id: str | None = None, ) -> dict[str, list[str]]: # mutable-ok: the old return type, unchanged _note("get_model_access_groups", "registry().access_groups(...)") - return _as_router(self)._core_get_model_access_groups( - model_name=model_name, model_access_group=model_access_group, team_id=team_id + return ( + _as_router(self) + .registry() + .access_groups(model_name=model_name, model_access_group=model_access_group, team_id=team_id) ) def get_deployment_credentials(self, model_id: str) -> dict | None: # mutable-ok: the old return type, unchanged _note("get_deployment_credentials", "registry().credentials(id)") - return _as_router(self)._core_get_deployment_credentials(model_id=model_id) + return _as_router(self).registry().credentials(model_id) def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> "dict[str, Any] | None": # pyright: ignore[reportExplicitAny] # the old return type, unchanged # mutable-ok: same _note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')") - return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id) + return _as_router(self).registry().credentials(model_id, form="with_provider", team_id=team_id) def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None": _note("get_credential_deployment", "registry().credentials(id, form='deployment')") - return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id) + return _as_router(self).registry().credentials(model_id, form="deployment", team_id=team_id) def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": _note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')") - return _as_router(self)._core_get_router_deployment_budget_limiter() + return _as_router(self).registry().plugins("budget_limiter") - _get_router_deployment_budget_limiter = _private_alias( - "get_router_deployment_budget_limiter", get_router_deployment_budget_limiter - ) + _get_router_deployment_budget_limiter = get_router_deployment_budget_limiter @staticmethod def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: the old parameter type @@ -678,15 +628,15 @@ class DeprecatedRouterMembers: # -- settings: methods ----------------------------------------------------------------------- def get_settings(self) -> dict[str, object]: # mutable-ok: Router._core_get_settings's type, a fresh dict _note("get_settings", "settings().as_dict()") - return _as_router(self)._core_get_settings() + return dict(_as_router(self).settings().as_dict()) def get_allowed_fails_from_policy(self, exception: Exception) -> int | None: _note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)") - return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception) + return _as_router(self).settings().allowed_fails_for(exception) def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict": _note("get_available_guardrail", "settings().guardrail(name)") - return _as_router(self)._core_get_available_guardrail(guardrail_name) + return _as_router(self).settings().guardrail(guardrail_name) # -- health ---------------------------------------------------------------------------------- @staticmethod @@ -709,31 +659,31 @@ class DeprecatedRouterMembers: def get_fully_blocked_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged _note("get_fully_blocked_model_names", "health().fully_blocked_model_names()") - return _as_router(self)._core_get_fully_blocked_model_names() + return _as_router(self).health().fully_blocked_model_names() async def async_get_fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged _note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()") - return await _as_router(self)._core_async_get_fully_unhealthy_model_names() + return await _as_router(self).health().fully_unhealthy_model_names() async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: _note("get_model_group_usage", "health().usage(group)") - return await _as_router(self)._core_get_model_group_usage(model_group=model_group) + return await _as_router(self).health().usage(model_group) async def get_remaining_model_group_usage( self, model_group: str ) -> dict[str, int]: # mutable-ok: the old return type, unchanged _note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')") - return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group) + return await _as_router(self).health().usage(model_group, kind="remaining") def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": _note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)") - return _as_router(self)._core_team_ptu_ceiling(team_id, model_group) + return _as_router(self).health().team_ptu_ceiling(team_id, model_group) def split_request_priority( self, model: str, kwargs: Mapping[str, object] ) -> tuple[int | None, Mapping[str, object]]: _note("split_request_priority", "health().split_priority(model, kwargs)") - return _as_router(self)._core_split_request_priority(model, kwargs) + return _as_router(self).health().split_priority(model, kwargs) # -- selection ------------------------------------------------------------------------------- async def async_get_healthy_deployments( @@ -747,9 +697,10 @@ class DeprecatedRouterMembers: health_check_probe: bool = False, ) -> list[dict] | dict: # mutable-ok: the old return type, unchanged _note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')") - return await _as_router(self)._core_async_get_healthy_deployments( - model=model, - request_kwargs=request_kwargs, + return await _as_router(self).select( + model, + request_kwargs, + kind="healthy", messages=messages, input=input, specific_deployment=specific_deployment, @@ -757,7 +708,7 @@ class DeprecatedRouterMembers: health_check_probe=health_check_probe, ) - def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from the core + def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from select self, model: str, messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged @@ -766,12 +717,15 @@ class DeprecatedRouterMembers: request_kwargs: dict | None = None, # mutable-ok: the old parameter type, unchanged ): _note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')") - return _as_router(self)._core_get_available_deployment_for_pass_through( - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - request_kwargs=request_kwargs, + return _run_ready( + _as_router(self).select( + model, + request_kwargs, + kind="pass_through", + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) ) # -- writes ---------------------------------------------------------------------------------- @@ -791,7 +745,7 @@ class DeprecatedRouterMembers: _note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))") _as_router(self).apply(ChangeSet(redis_cache=cache)) - _update_redis_cache = _private_alias("update_redis_cache", update_redis_cache) + _update_redis_cache = update_redis_cache async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None: _note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))") @@ -808,8 +762,15 @@ class DeprecatedRouterMembers: def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None: _note("bind_to_request_logging", "observe(logging_obj)") - # Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not. - _as_router(self)._core_bind_to_request_logging(logging_obj) + router: Final = _as_router(self) + if logging_obj is None: # pyright: ignore[reportUnnecessaryComparison] # callers pass None despite the type + # observe() skips binding when logging_obj is None; the old member bound it, and failed on None. + raise AttributeError( + "'NoneType' object has no attribute 'add_dynamic_function_callbacks'", + name="add_dynamic_function_callbacks", + obj=None, + ) + router.observe(logging_obj) def discard(self) -> None: _note("discard", "retire()") @@ -822,7 +783,7 @@ class DeprecatedRouterMembers: request_kwargs: dict[str, object] | None = None, # mutable-ok: the old parameter type, unchanged ) -> None: _note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch") - _as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs) + RoutingContext(_as_router(self)).arm_read_prefetch(model, request_kwargs) async def async_pre_routing_hook( self, @@ -833,7 +794,7 @@ class DeprecatedRouterMembers: specific_deployment: bool | None = False, ) -> "PreRoutingHookResponse | None": _note("async_pre_routing_hook", "RoutingContext.pre_routing_hook") - return await _as_router(self)._core_async_pre_routing_hook( + return await RoutingContext(_as_router(self)).pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -843,7 +804,7 @@ class DeprecatedRouterMembers: def log_retry(self, kwargs: dict, e: Exception) -> dict: # mutable-ok: the old signature, unchanged _note("log_retry", "RoutingContext.log_retry") - return _as_router(self)._core_log_retry(kwargs=kwargs, e=e) + return RoutingContext(_as_router(self)).log_retry(kwargs=kwargs, e=e) # Router-family functions and classes that took the Router (``litellm_router_instance=``, diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 2235abc9897..8ca2afb9a99 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -84,6 +84,8 @@ from litellm.constants import ( MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, @@ -108,6 +110,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( replace_complexity_router_headers, response_total_token_count, ) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.batch_utils import ( batch_create_routing, @@ -124,6 +127,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -133,7 +137,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.dd_tracing import tracer -from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models, deployment_pricing_models from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, @@ -279,7 +283,15 @@ from litellm.router_strategy.complexity_router.context_compaction import ( reject_recursive_compactor, surface_for_call, ) -from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_strategy.least_busy import ( + CALL_IDENTITY, + SELECTION_RESERVATION_MAX_KEYS, + LeastBusyLoggingHandler, + call_id_of, + owning_picks, + pick_reserves, + picks_reserve, +) from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler @@ -303,6 +315,7 @@ from litellm.router_utils.cooldown_handlers import ( get_cooldown_deployment_info, get_cooldown_deployments, is_background_response_poll_not_found, + is_caller_entra_credential_failure, is_caller_scoped_auth_failure, is_caller_timeout_408, is_failure_outside_selected_deployment, @@ -310,6 +323,7 @@ from litellm.router_utils.cooldown_handlers import ( mark_deployment_selection_failure, set_cooldown_deployments, ) +from litellm.router_utils.deployment_numbers import refuse_unservable_numbers from litellm.router_utils.embedding_batch_size_check import ( async_log_rejection_before_provider_call, clear_rejected_before_provider_call, @@ -354,6 +368,11 @@ from litellm.router_utils.fallback_event_handlers import ( routed_deployment_id, run_async_fallback, ) +from litellm.router_utils.global_callback_holders import ( + live_check_owners, + routers_holding_global_callbacks, + routers_holding_global_callbacks_lock, +) from litellm.router_utils.handle_error import ( async_raise_no_deployment_exception, send_llm_exception_alert, @@ -477,11 +496,13 @@ from litellm.types.services import ServiceTypes from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, COST_MAP_LOOKUP_KEY, + DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS, PROMPT_QUOTING_ROUTING_DECISION_FIELDS, CustomPricingLiteLLMParams, GenericBudgetConfigType, LiteLLMBatch, LlmProviders, + LlmProvidersSet, ModelInfo, ModelResponseStream, StandardLoggingPayload, @@ -504,9 +525,11 @@ from litellm.utils import ( TextCompletionResponse, _restore_correlation_context_if_supported, # pyright: ignore[reportPrivateUsage] # same task-local cleanup as the SDK wrapper aresolve_api_key_secret_reference, + catalog_with_runtime_registrations, function_setup, get_llm_provider, get_non_default_completion_params, + get_potential_model_names, get_secret, get_utc_datetime, is_generalized_model_info, @@ -672,6 +695,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) @@ -847,9 +871,11 @@ def _with_router_resolved_session_model(session: object, model_name: str) -> Map typed_session: Final = _SESSION_ADAPTER.validate_python(session) except ValidationError: return _NO_SESSION_KWARGS - if "model" not in typed_session: + if "model" not in typed_session and typed_session.get("type") != "transcription": return _NO_SESSION_KWARGS - return MappingProxyType({"session": {**typed_session, "model": model_name}}) + from litellm.realtime_api.main import with_resolved_session_model + + return MappingProxyType({"session": with_resolved_session_model(dict(typed_session), model_name)}) # Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks @@ -1214,8 +1240,9 @@ class RoutingArgs(enum.Enum): # as the per-request one built from a caller-supplied user_config, drops out on its # own rather than leaving entries behind that nothing can withdraw. _live_routers: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() -_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() -_routers_holding_global_callbacks_lock: Final = threading.Lock() +# Shared with the other router cores of the process (litellm/router_utils/global_callback_holders.py). +_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = routers_holding_global_callbacks # pyright: ignore[reportAssignmentType] # every member serves the holder members +_routers_holding_global_callbacks_lock: Final = routers_holding_global_callbacks_lock def _replay_live_router_model_cost() -> None: @@ -1451,13 +1478,14 @@ class PythonCore(DeprecatedRouterMembers): (``pass_through``). """ if kind == "pick": - return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - request_kwargs=request_kwargs if request_kwargs is not None else {}, - messages=messages, - input=input, - specific_deployment=specific_deployment, - ) + with picks_reserve(False): + return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + request_kwargs=request_kwargs if request_kwargs is not None else {}, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) if kind == "healthy": return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts model=model, @@ -1469,13 +1497,14 @@ class PythonCore(DeprecatedRouterMembers): health_check_probe=health_check_probe, ) if kind == "pass_through": - return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts - model=model, - messages=messages, - input=input, - specific_deployment=specific_deployment, - request_kwargs=request_kwargs, - ) + with picks_reserve(False): + return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + messages=messages, + input=input, + specific_deployment=specific_deployment, + request_kwargs=request_kwargs, + ) raise ValueError(f"unknown select kind {kind!r}") def apply(self, changes: ChangeSet) -> ApplyResult: @@ -1757,6 +1786,9 @@ class PythonCore(DeprecatedRouterMembers): self._core_cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. + self._core_least_busy_reservations: InMemoryCache = InMemoryCache( + max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS + ) self._claude_code_session_router_cache: DualCache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache(), @@ -1854,6 +1886,7 @@ class PythonCore(DeprecatedRouterMembers): # cost-map entries from the list it is serving at that moment, whether # those deployments came from model_list or were added later from the DB. _live_routers.add(self) + live_check_owners.add(self) if allowed_fails is not None: self._core_allowed_fails = allowed_fails @@ -2079,6 +2112,7 @@ class PythonCore(DeprecatedRouterMembers): # Stop contributing to cost-map rebuilds straight away rather than waiting # for this router to be collected. _live_routers.discard(self) + live_check_owners.discard(self) self.retire_routing_state() self._release_global_callbacks() @@ -2260,7 +2294,9 @@ class PythonCore(DeprecatedRouterMembers): selector: RouterStrategySelector | None = None match self._normalize_strategy(strategy): case RoutingStrategy.LEAST_BUSY.value: - selector = LeastBusyLoggingHandler(router_cache=self._core_cache) + selector = LeastBusyLoggingHandler( + router_cache=self._core_cache, reservation_store=self._core_least_busy_reservations + ) case RoutingStrategy.USAGE_BASED_ROUTING.value: selector = LowestTPMLoggingHandler( router_cache=self._core_cache, @@ -2725,6 +2761,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Asks the strategy selector for a deployment. Caller handles @@ -2738,6 +2775,8 @@ class PythonCore(DeprecatedRouterMembers): return await selector.async_get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing": # `LowestTPMLoggingHandler` (v1) only exposes the sync @@ -2778,6 +2817,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, request_kwargs: dict | None, + reserve: bool, ) -> Any | None: """ Sync sibling of `_select_deployment_async`. Caller handles @@ -2794,6 +2834,8 @@ class PythonCore(DeprecatedRouterMembers): return selector.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), ) case "usage-based-routing" | "usage-based-routing-v2": return selector.get_available_deployments( @@ -2865,6 +2907,14 @@ class PythonCore(DeprecatedRouterMembers): self._host.invoke("acreate_realtime_transcription_session"), call_type="acreate_realtime_transcription_session", ) + self.acreate_realtime_translation_client_secret = self.factory_function( + self._host.invoke("acreate_realtime_translation_client_secret"), + call_type="acreate_realtime_translation_client_secret", + ) + self.arealtime_translation_calls = self.factory_function( + self._host.invoke("arealtime_translation_calls"), + call_type="arealtime_translation_calls", + ) self._aresponses_websocket = self.factory_function( self._host.invoke("_aresponses_websocket"), call_type="_aresponses_websocket" ) @@ -3374,7 +3424,7 @@ class PythonCore(DeprecatedRouterMembers): self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] if any( router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) - for router in tuple(_live_routers) + for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))) ): return for cb in tuple(self._host.callback_list("callbacks")): @@ -3505,12 +3555,13 @@ class PythonCore(DeprecatedRouterMembers): ) # pick the one that is available (lowest TPM/RPM) - deployment = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._drop_unsupported_classifier_reasoning_effort( deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment model=model, @@ -3824,11 +3875,6 @@ class PythonCore(DeprecatedRouterMembers): priced_partial._hidden_params = { # pyright: ignore[reportPrivateUsage] # no public accessor key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost" } - if LiteLLMLogging.usage_capture_callback is not None: - from litellm._internal_context import internal_sub_call - - with internal_sub_call(cut_logging.litellm_call_id): - cut_logging.capture_usage_before_logging(priced_partial) return ContinuedPartial( usage=BaseTokenUsageProcessor.combine_usage_objects( usage_objects=[partial_usage, inherited_usage] @@ -4027,6 +4073,8 @@ class PythonCore(DeprecatedRouterMembers): if continuation_output is not None else None ) + if cut_partial is not None: + model_response.logging_obj.mark_stash_billed_by_continuation() if continue_after_content: emitted_tokens: Final = int( getattr(complete_response_object_usage, "completion_tokens", 0) or 0 @@ -5196,8 +5244,19 @@ class PythonCore(DeprecatedRouterMembers): # _update_kwargs_with_deployment and stamped into kwargs["model_info"]; stamping # the static shared deployment's id instead would let one tenant's bad credentials # cool down the deployment every other tenant sharing this config relies on. + if getattr(exception, "failed_deployment_id", None): + return effective_model_info: Final = kwargs.get("model_info") or deployment.get("model_info") or MappingProxyType({}) self._set_failed_deployment_id_on_exception(exception, MappingProxyType({"model_info": effective_model_info})) + configured_deployment_id: Final = _MODEL_INFO_ADAPTER.validate_python(effective_model_info).get( + "original_model_id" + ) + if isinstance(configured_deployment_id, str) and configured_deployment_id: + exception.failed_configured_deployment_id = configured_deployment_id # pyright: ignore[reportAttributeAccessIssue] # rebind-ok: the exception carries the failure to _maybe_run_weighted_failover + + def _stamp_failed_attempt(self, exception: Exception, kwargs: Mapping[str, object], *, picked: bool) -> None: + if picked: + self._stamp_failed_deployment_id_with_effective_model_info(exception, MappingProxyType({}), kwargs) @staticmethod def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, object]) -> None: @@ -6074,14 +6133,16 @@ class PythonCore(DeprecatedRouterMembers): def _image_generation(self, prompt: str, model: str, **kwargs): model_name: Final = "" + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) - deployment: Final = self.get_available_deployment( - model=model, - messages=[{"role": "user", "content": "prompt"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(True): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=[{"role": "user", "content": "prompt"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() @@ -6112,6 +6173,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def aimage_generation(self, prompt: str, model: str, **kwargs): @@ -6137,11 +6201,12 @@ class PythonCore(DeprecatedRouterMembers): async def _aimage_generation(self, prompt: str, model: str, **kwargs): model_name = model start_time: Final = datetime.now() # noqa: DTZ005 # SDK logging uses naive local timestamps + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) try: - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6202,6 +6267,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def atranscription(self, file: FileTypes, model: str, **kwargs): @@ -6248,10 +6316,11 @@ class PythonCore(DeprecatedRouterMembers): async def _atranscription(self, file: FileTypes, model: str, **kwargs): model_name: Final = model + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _atranscription()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6285,6 +6354,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def aspeech(self, model: str, input: str, voice: str | None = None, **kwargs): @@ -6341,10 +6413,11 @@ class PythonCore(DeprecatedRouterMembers): async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs): model_name: Final = model + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _aspeech()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6378,6 +6451,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def arerank(self, model: str, **kwargs): @@ -6403,9 +6479,10 @@ class PythonCore(DeprecatedRouterMembers): async def _arerank(self, model: str, **kwargs): model_name = None + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _rerank()- model: %s", model) - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, @@ -6437,6 +6514,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e def text_completion( @@ -6449,37 +6529,38 @@ class PythonCore(DeprecatedRouterMembers): **kwargs, ): messages: Final = [{"role": "user", "content": prompt}] - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + kwargs["model"] = model + kwargs["prompt"] = prompt + kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) + kwargs.setdefault("metadata", {}).update({"model_group": model}) - # pick the one that is available (lowest TPM/RPM) - with endpoint_selection("text_completion", model=model): - deployment: Final = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + # pick the one that is available (lowest TPM/RPM) + with endpoint_selection("text_completion", model=model), picks_reserve(True): + deployment: Final = self.get_available_deployment( + model=model, + messages=messages, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + + data: Final = deployment["litellm_params"].copy() + for k, v in self._core_default_litellm_params.items(): + if k not in kwargs: # prioritize model-specific params > default router params + kwargs[k] = v + elif k == "metadata": + kwargs[k].update(v) + pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults + + # call via litellm.completion() + return self._host.invoke("text_completion")( + **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} ) - - data: Final = deployment["litellm_params"].copy() - for k, v in self._core_default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) - pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) - if pinned_model_id is not None: - kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults - - # call via litellm.completion() - return self._host.invoke("text_completion")( - **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} - ) - except Exception as e: - raise e + except Exception as e: + raise e async def atext_completion( self, @@ -6518,10 +6599,11 @@ class PythonCore(DeprecatedRouterMembers): raise e async def _atext_completion(self, model: str, prompt: str, **kwargs): + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, messages=[{"role": "user", "content": prompt}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6557,6 +6639,9 @@ class PythonCore(DeprecatedRouterMembers): if model is not None: self.fail_calls[model] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def aadapter_completion( @@ -6588,10 +6673,11 @@ class PythonCore(DeprecatedRouterMembers): raise e async def _aadapter_completion(self, adapter_id: str, model: str, **kwargs): + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, messages=[{"role": "user", "content": "default text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6627,6 +6713,9 @@ class PythonCore(DeprecatedRouterMembers): if model is not None: self.fail_calls[model] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def _asearch_with_fallbacks(self, original_function: Callable, **kwargs) -> object: @@ -6892,6 +6981,22 @@ class PythonCore(DeprecatedRouterMembers): raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_logging: Final = ( + _CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if generic_function_name == "_arealtime" + else None + ) + realtime_session_deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") + if isinstance(realtime_session_logging, LiteLLMLogging) + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is not True + else None + ) + if realtime_session_deployment_id is not None: + increment_deployment_successes_for_current_minute( + ctx=self._core_ctx, deployment_id=str(realtime_session_deployment_id) + ) verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response @@ -7055,6 +7160,8 @@ class PythonCore(DeprecatedRouterMembers): or self._stream_can_fall_back(model, initial_kwargs) ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline + # An error this loop raises off a frame, not the provider's. + raised_here: BaseException | None = None # rebind-ok: set when the loop raises off a frame try: async for chunk in source_iterator: if _anthropic_stream_forwards_ping_live(chunk, has_generated_content): @@ -7085,25 +7192,27 @@ class PythonCore(DeprecatedRouterMembers): and self._refusal_fallback_available(model, initial_kwargs) ): refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) - raise MidStreamFallbackError( + raised_here = MidStreamFallbackError( message=refusal_error.message, model=model, llm_provider="anthropic", original_exception=refusal_error, is_pre_first_chunk=True, ) + raise raised_here if not has_generated_content and error_event is None: buffered_lifecycle_chunks = (*buffered_lifecycle_chunks, chunk) continue if recoverable_frame_error is not None: assert error_event is not None - raise MidStreamFallbackError( + raised_here = MidStreamFallbackError( message=error_event[1], model=model, llm_provider="anthropic", original_exception=recoverable_frame_error, is_pre_first_chunk=True, ) + raise raised_here for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk buffered_lifecycle_chunks = () @@ -7111,6 +7220,8 @@ class PythonCore(DeprecatedRouterMembers): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + if stream_error is not raised_here: + await self._areport_raised_messages_stream_failure(stream_error, initial_kwargs, source_iterator) async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, @@ -7130,6 +7241,52 @@ class PythonCore(DeprecatedRouterMembers): wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator) return wrapper + async def _areport_raised_messages_stream_failure( + self, stream_error: Exception, attempt_kwargs: Mapping[str, object], source: object + ) -> None: + """A /v1/messages stream whose provider iterator raised, counted against the deployment + that served it and cooled down by the rules, the same as a non-stream call's failure, so a + retry picks another deployment. A stream that logs itself (litellm's own stream logging + reaches the router's failure callback), or a failure the request's logging object already + logged, is not counted twice.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.litellm_core_utils.litellm_logging import Logging + + if isinstance(getattr(source, "logging_obj", None), Logging): + return + logging_obj: Final = attempt_kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, Logging) and logging_obj.model_call_details.get("has_logged_async_failure") is True: + return + bucket: Final = as_str_mapping(attempt_kwargs.get(get_metadata_variable_name_from_kwargs(attempt_kwargs))) or {} + model_info: Final = as_str_mapping(bucket.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + if deployment_id is None: + return + deployment: Final = self.get_deployment(model_id=str(deployment_id)) + deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + provider_error: Final = ( + stream_error.original_exception + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None + else stream_error + ) + now: Final = datetime.now() # noqa: DTZ005 # naive, as the logging pipeline's times + logged: Final = { + "call_type": "anthropic_messages", + "exception": provider_error, + "litellm_params": { + **{ + name: deployment_params[name] for name in ("api_base", "cooldown_time") if name in deployment_params + }, + "metadata": dict(bucket), + "model_info": dict(model_info), + }, + } + try: + _ = self.deployment_callback_on_failure(logged, None, now, now) # pyright: ignore[reportUnknownMemberType] # an untyped callback + await self.async_deployment_callback_on_failure(logged, None, now, now) # pyright: ignore[reportUnknownMemberType] # see above + except Exception as error: # noqa: BLE001 # a recording never fails the stream + verbose_router_logger.warning("a stream's failure was not recorded: %s", error) + async def _aanthropic_messages_recover_stream_error( self, stream_error: Exception, @@ -7340,6 +7497,8 @@ class PythonCore(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) + if is_agentic_followup_failure(trigger): + return _StreamRetriesExhausted(last_error) if not policy_applies and not self._stream_should_retry( trigger, healthy_deployments, all_deployments, initial_kwargs ): @@ -7597,61 +7756,70 @@ class PythonCore(DeprecatedRouterMembers): """ handler_name: Final = original_function.__name__ metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") - try: - verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) - self._update_kwargs_before_fallbacks( - model=model, - kwargs=kwargs, - metadata_variable_name=metadata_variable_name, - ) - with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): - deployment: Final = self.get_available_deployment( + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) + self._update_kwargs_before_fallbacks( model=model, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, + kwargs=kwargs, + metadata_variable_name=metadata_variable_name, + ) + with ( + self._video_call_pin(handler_name, kwargs), + self._route_capability_pin(handler_name), + picks_reserve(True), + ): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="generic_api_call", + splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="generic_api_call", - splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] - self.total_calls[model_name] += 1 + self.total_calls[model_name] += 1 - # For passthrough routes, use the actual model from deployment - # and swap model name in endpoint if present - if "endpoint" in kwargs and kwargs["endpoint"]: - kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) - kwargs["model"] = model_name + # For passthrough routes, use the actual model from deployment + # and swap model name in endpoint if present + if "endpoint" in kwargs and kwargs["endpoint"]: + kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + kwargs["model"] = model_name - # Perform pre-call checks for routing strategy - self.routing_strategy_pre_call_checks(deployment=deployment) + # Perform pre-call checks for routing strategy + self.routing_strategy_pre_call_checks(deployment=deployment) - custom_llm_provider: Final = provider_for_generic_call(data) + custom_llm_provider: Final = provider_for_generic_call(data) - response: Final = original_function( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - **kwargs, - } - ) + response: Final = original_function( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + **kwargs, + } + ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) - return response - except Exception as e: - verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) - if model is not None: - self.fail_calls[model] += 1 - raise e + self.success_calls[model_name] += 1 + verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", handler_name, model_name) + return response + except Exception as e: + verbose_router_logger.info("%s(model=%s)\x1b[31m Exception %s\x1b[0m", handler_name, model, e) + if model is not None: + self.fail_calls[model] += 1 + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) + raise e def embedding( self, @@ -7672,10 +7840,11 @@ class PythonCore(DeprecatedRouterMembers): def _embedding(self, input: str | list[object], model: str, **kwargs): model_name = None + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside embedding()- model: %s", model) - with embedding_deployment_selection(): - deployment: Final = self.get_available_deployment( + with embedding_deployment_selection(), picks_reserve(True): + deployment = self.get_available_deployment( # rebind-ok: set on success, see pre-init model=model, input=input, specific_deployment=kwargs.pop("specific_deployment", None), @@ -7732,6 +7901,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e async def aembedding( @@ -7761,11 +7933,12 @@ class PythonCore(DeprecatedRouterMembers): async def _aembedding(self, input: str | list[object], model: str, **kwargs): model_name = None + deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: verbose_router_logger.debug("Inside _aembedding()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with embedding_deployment_selection(): - deployment: Final = await self._core_async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( # rebind-ok: set on success, see pre-init model=model, input=input, specific_deployment=kwargs.pop("specific_deployment", None), @@ -7812,6 +7985,9 @@ class PythonCore(DeprecatedRouterMembers): if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) + self._stamp_failed_attempt( + e, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), picked=deployment is not None + ) raise e #### FILES API #### @@ -7976,68 +8152,71 @@ class PythonCore(DeprecatedRouterMembers): Returns: VectorStoreCreateResponse """ - try: - # If model is None, use the factory function approach (direct SDK call) - if model is None: - from litellm.vector_stores.main import acreate + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + # If model is None, use the factory function approach (direct SDK call) + if model is None: + from litellm.vector_stores.main import acreate - # Use the factory function to handle the call - factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") - return await factory_fn(**kwargs) + # Use the factory function to handle the call + factory_fn: Final = self.factory_function(acreate, call_type="avector_store_create") + return await factory_fn(**kwargs) - from litellm.vector_stores import acreate as avector_store_create_sdk + from litellm.vector_stores import acreate as avector_store_create_sdk - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "vector-store-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) - data: Final = deployment["litellm_params"].copy() - model_name: Final = data["model"] - self._update_kwargs_with_deployment( - deployment=deployment, - kwargs=kwargs, - function_name="avector_store_create", - ) - - model_client: Final = self._get_async_openai_model_client( - deployment=deployment, - kwargs=kwargs, - ) - self.total_calls[model_name] += 1 - - # Get custom provider from deployment params - custom_llm_provider = data.get("custom_llm_provider") - _, inferred_custom_llm_provider, _, _ = get_llm_provider( - model=data["model"], - custom_llm_provider=custom_llm_provider, - ) - custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider - - async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await avector_store_create_sdk( - **{ - **data, - "custom_llm_provider": custom_llm_provider, - "caching": self._core_cache_responses, - "client": model_client, - **kwargs, - } + parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "vector-store-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + data: Final = deployment["litellm_params"].copy() + model_name: Final = data["model"] + self._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="avector_store_create", ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + model_client: Final = self._get_async_openai_model_client( + deployment=deployment, + kwargs=kwargs, + ) + self.total_calls[model_name] += 1 - return response - except Exception as e: - verbose_router_logger.exception( - "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e - ) - if model is not None: - self.fail_calls[model] += 1 - raise e + # Get custom provider from deployment params + custom_llm_provider = data.get("custom_llm_provider") + _, inferred_custom_llm_provider, _, _ = get_llm_provider( + model=data["model"], + custom_llm_provider=custom_llm_provider, + ) + custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider + + async with self._deployment_slot( + deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span + ): + response = await avector_store_create_sdk( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self._core_cache_responses, + "client": model_client, + **kwargs, + } + ) + + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.avector_store_create(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + + return response + except Exception as e: + verbose_router_logger.exception( + "litellm.avector_store_create(model=%s)\x1b[31m Exception %s\x1b[0m", model, e + ) + if model is not None: + self.fail_calls[model] += 1 + raise e def _override_vector_store_methods_for_router(self): """ @@ -8285,12 +8464,13 @@ class PythonCore(DeprecatedRouterMembers): try: verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self._core_async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "batch-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with picks_reserve(False): + deployment: Final = await self._core_async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "batch-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") data: Final = deployment["litellm_params"].copy() @@ -8433,6 +8613,8 @@ class PythonCore(DeprecatedRouterMembers): "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", "_aresponses_websocket", "acreate_fine_tuning_job", "acancel_fine_tuning_job", @@ -8703,6 +8885,8 @@ class PythonCore(DeprecatedRouterMembers): "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, @@ -9046,8 +9230,9 @@ class PythonCore(DeprecatedRouterMembers): kwargs[metadata_variable_name] = meta if not isinstance(meta, dict): return None + configured_failed_id: Final[str | None] = getattr(exception, "failed_configured_deployment_id", None) prev_excluded: Final = set(meta.get("_failover_excluded_ids") or []) - excluded: Final = prev_excluded | {failed_id} + excluded: Final = prev_excluded | {configured_failed_id or failed_id} all_ids: Final = { (d.get("model_info") or {}).get("id") @@ -9152,11 +9337,17 @@ class PythonCore(DeprecatedRouterMembers): if include_fallback_errors: input_kwargs["include_fallback_errors"] = True - # ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list - # Skip for error types that have their own dedicated fallback handlers - _skip_order_fallback: Final = isinstance( - e, - (litellm_exceptions.ContextWindowExceededError, litellm_exceptions.ContentPolicyViolationError), + context_window_fallback_model_group: Final = ( + self._get_fallback_model_group_for_lookup_groups( + fallbacks=context_window_fallbacks, + lookup_groups=lookup_groups, + ) + if isinstance(e, litellm_exceptions.ContextWindowExceededError) and context_window_fallbacks is not None + else None + ) + _skip_order_fallback: Final = ( + isinstance(e, litellm_exceptions.ContentPolicyViolationError) + or context_window_fallback_model_group is not None ) _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") # Use wildcard-aware lookup so order-based fallback also works for model @@ -9202,14 +9393,23 @@ class PythonCore(DeprecatedRouterMembers): "original_model_group": original_model_group, } ) - response = await run_async_fallback( - *args, - **input_kwargs, - ) - return response + try: + return await run_async_fallback( + *args, + **input_kwargs, + ) + except Exception: + if isinstance(original_exception, litellm_exceptions.ContextWindowExceededError): + raise original_exception + raise # Weighted intra-group failover (simple-shuffle only); see _maybe_run_weighted_failover. - if self.enable_weighted_failover and not _skip_order_fallback and original_model_group is not None: + if ( + self.enable_weighted_failover + and not _skip_order_fallback + and not isinstance(e, litellm_exceptions.ContextWindowExceededError) + and original_model_group is not None + ): response = await self._maybe_run_weighted_failover( exception=e, original_model_group=original_model_group, @@ -9227,7 +9427,7 @@ class PythonCore(DeprecatedRouterMembers): # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks=[{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey, how's it going?"}]}] is_non_standard_fallback_format: Final = check_non_standard_fallback_format(fallbacks=fallbacks) - if is_non_standard_fallback_format: + if is_non_standard_fallback_format and context_window_fallback_model_group is None: input_kwargs.update( { "fallback_model_group": fallbacks, @@ -9243,16 +9443,7 @@ class PythonCore(DeprecatedRouterMembers): return response if isinstance(e, litellm_exceptions.ContextWindowExceededError): - if context_window_fallbacks is not None: - context_window_fallback_model_group: Final[list[str] | None] = ( - self._get_fallback_model_group_for_lookup_groups( - fallbacks=context_window_fallbacks, - lookup_groups=lookup_groups, - ) - ) - if context_window_fallback_model_group is None: - raise original_exception - + if context_window_fallback_model_group is not None: input_kwargs.update( { "fallback_model_group": context_window_fallback_model_group, @@ -9266,7 +9457,7 @@ class PythonCore(DeprecatedRouterMembers): ) return response - else: + elif context_window_fallbacks is None: error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" verbose_router_logger.info( msg=f"Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ @@ -9424,38 +9615,39 @@ class PythonCore(DeprecatedRouterMembers): mock_timeout: Final = kwargs.pop("mock_timeout", None) - try: - self._handle_mock_testing_fallbacks( - kwargs=kwargs, - model_group=model_group, - fallbacks=fallbacks, - context_window_fallbacks=context_window_fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - ) + with owning_picks(self._core_least_busy_reservations, call_id_of(CALL_IDENTITY.validate_python(kwargs))): + try: + self._handle_mock_testing_fallbacks( + kwargs=kwargs, + model_group=model_group, + fallbacks=fallbacks, + context_window_fallbacks=context_window_fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + ) - if mock_timeout is not None: - response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) - else: - response = await self.async_function_with_retries(*args, **kwargs) - if verbose_router_logger.isEnabledFor(logging.DEBUG): - verbose_router_logger.debug("Async Response: %s", response) - response = add_fallback_headers_to_response( - response=response, - attempted_fallbacks=0, - ) - return response - except Exception as e: - return await self.async_function_with_fallbacks_common_utils( - e, - disable_fallbacks, - fallbacks, - context_window_fallbacks, - content_policy_fallbacks, - model_group, - args, - kwargs, - include_fallback_errors=include_fallback_errors, - ) + if mock_timeout is not None: + response = await self.async_function_with_retries(*args, **kwargs, mock_timeout=mock_timeout) + else: + response = await self.async_function_with_retries(*args, **kwargs) + if verbose_router_logger.isEnabledFor(logging.DEBUG): + verbose_router_logger.debug("Async Response: %s", response) + response = add_fallback_headers_to_response( + response=response, + attempted_fallbacks=0, + ) + return response + except Exception as e: + return await self.async_function_with_fallbacks_common_utils( + e, + disable_fallbacks, + fallbacks, + context_window_fallbacks, + content_policy_fallbacks, + model_group, + args, + kwargs, + include_fallback_errors=include_fallback_errors, + ) def _handle_mock_testing_fallbacks( self, @@ -9559,7 +9751,7 @@ class PythonCore(DeprecatedRouterMembers): response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None) return response except Exception as e: - if is_guardrail_intervention(e): + if is_guardrail_intervention(e) or is_agentic_followup_failure(e): raise current_attempt = None original_exception = e @@ -9654,6 +9846,7 @@ class PythonCore(DeprecatedRouterMembers): # Update retry tracking metadata before each retry attempt _metadata["attempted_retries"] = current_attempt + 1 _metadata["max_retries"] = num_retries + carry_superseded_attempt_usage(kwargs) # if the function call is successful, no exception will be raised and we'll break out of the loop response = await self.make_call(original_function, *args, **kwargs) if coroutine_checker.is_async_callable(response): # async errors are often returned as coroutines @@ -9667,6 +9860,8 @@ class PythonCore(DeprecatedRouterMembers): return response except Exception as e: + if is_agentic_followup_failure(e): + raise # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e @@ -10211,8 +10406,9 @@ class PythonCore(DeprecatedRouterMembers): return False exception_status: Final = getattr(exception, "status_code", "") + failure_call_details: Final = _CALL_KWARGS_ADAPTER.validate_python(kwargs) - if is_caller_timeout_408(kwargs, exception_status): + if is_caller_timeout_408(failure_call_details, exception_status): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " "A timeout the caller set caused this 408, not the deployment's health." @@ -10236,6 +10432,18 @@ class PythonCore(DeprecatedRouterMembers): "Caller-scoped OAuth authentication failed, not the deployment." ) return False + if is_caller_entra_credential_failure( + caller_failure_deployment, + self._active_default_litellm_params(), + failure_call_details, + exception_status, + exception, + ): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Entra ID rejected a credential the caller sent, not the deployment's own." + ) + return False exception_headers: Final = exception_mapping_utils._get_response_headers(original_exception=exception) @@ -10744,6 +10952,7 @@ class PythonCore(DeprecatedRouterMembers): messages=None, input=None, request_kwargs=routing_kwargs, + reserve=False, ) if selected is None: no_deployment: Final = await async_raise_no_deployment_exception( @@ -11088,6 +11297,7 @@ class PythonCore(DeprecatedRouterMembers): model_info=_model_info, ) ) + refuse_unservable_numbers(deployment.to_json(exclude_none=True, warnings=False)) # pyright: ignore[reportUnknownMemberType] # pydantic's to_json PythonCore._register_deployment_pricing(deployment) ## Check if LLM Deployment is allowed for this deployment @@ -11591,6 +11801,16 @@ class PythonCore(DeprecatedRouterMembers): return model_info_is_active_for_environment(model_info=deployment.model_info) def set_model_list(self, model_list: list): + """Replace every deployment. All or nothing: when an entry raises, the deployments served + before the call are put back and the error is raised.""" + previous: Final = copy.deepcopy(list(self.__dict__.get("_core_model_list") or ())) + try: + self._replace_model_list(model_list) # pyright: ignore[reportUnknownMemberType] # the legacy untyped model list + except Exception: + self._replace_model_list(previous) # pyright: ignore[reportUnknownMemberType] # see above + raise + + def _replace_model_list(self, model_list: list) -> None: # pyright: ignore[reportUnknownParameterType, reportMissingTypeArgument] # the legacy untyped model list original_model_list: Final = copy.deepcopy(model_list) self._discovered_model_info_cache.flush_cache() with self._deployment_index_lock: @@ -11884,6 +12104,7 @@ class PythonCore(DeprecatedRouterMembers): # add to model list _deployment: Final = served.to_json(exclude_none=True) + refuse_unservable_numbers(_deployment) # initialize client self._add_deployment(deployment=served) @@ -12027,6 +12248,9 @@ class PythonCore(DeprecatedRouterMembers): _deployment_model_id: Final = deployment.model_info.id or "" _deployment_on_router: Final[Deployment | None] = self._core_get_deployment(model_id=_deployment_model_id) served: Final = without_model_capability_flags(deployment) + # `_create_deployment` lists the model name before it can still fail; a dropped deployment + # takes its name back off the list. + names_before: Final = len(self._core_deployment_names) try: if _deployment_on_router is not None: # deployment with this model_id exists on the router @@ -12076,6 +12300,7 @@ class PythonCore(DeprecatedRouterMembers): deployment.model_info.id, e, ) + del self._core_deployment_names[names_before:] self._restore_deployment_after_failed_upsert( previous_deployment=_deployment_on_router, model_id=_deployment_model_id ) @@ -12119,6 +12344,25 @@ class PythonCore(DeprecatedRouterMembers): return (backend_key, backend_key.replace("responses/", "")) return (backend_key,) + @staticmethod + def _catalog_entry_has_token_limit(catalog_entry: Mapping[str, object], field: str) -> bool: + return catalog_entry.get(field) is not None or ( + field == "max_tokens" and catalog_entry.get("max_output_tokens") is not None + ) + + @staticmethod + def _shared_backend_model_info_for_keys( + catalog: Mapping[str, Mapping[str, object]], + backend_keys: tuple[str, ...], + shared_model_info: Mapping[str, object], + ) -> Mapping[str, object]: + catalog_token_limit_fields: Final = frozenset( + field + for field in DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS + if any(PythonCore._catalog_entry_has_token_limit(catalog.get(key) or {}, field) for key in backend_keys) + ) + return {field: value for field, value in shared_model_info.items() if field not in catalog_token_limit_fields} + @staticmethod def _deployment_model_cost_payload(deployment: Deployment) -> dict[str, object]: # mutable-ok: cost-map entry """The ``model_info`` a deployment contributes to ``litellm.model_cost``. @@ -12231,6 +12475,31 @@ class PythonCore(DeprecatedRouterMembers): ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes backend_keys: Final = PythonCore._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) + model_provider_prefix: Final = model.partition("/")[0] + catalog_provider: Final = ( + custom_llm_provider + if custom_llm_provider is not None + else model_provider_prefix + if model_provider_prefix in LlmProvidersSet + else None + ) + potential_names: Final = ( + get_potential_model_names(model, catalog_provider) if catalog_provider is not None else None + ) + catalog_backend_keys: Final = ( + ( + *backend_keys, + model, + potential_names["split_model"], + potential_names["stripped_model_name"], + potential_names["combined_model_name"], + potential_names["combined_stripped_model_name"], + potential_names["provider_prefixed_model_name"], + potential_names["region_free_combined_model_name"], + ) + if potential_names is not None + else (*backend_keys, model) + ) # For the shared backend key, keep only cost-map schema fields # (minus custom pricing) so that one deployment's pricing overrides @@ -12238,7 +12507,12 @@ class PythonCore(DeprecatedRouterMembers): # don't pollute another deployment sharing the same backend model # name. Each deployment's full model_info is already stored under # its unique model_id above. - shared_model_info: Final = shared_backend_model_info(model_info) + catalog: Final = catalog_with_runtime_registrations() + shared_model_info: Final = PythonCore._shared_backend_model_info_for_keys( + catalog=catalog, + backend_keys=catalog_backend_keys, + shared_model_info=shared_backend_model_info(model_info), + ) python_host.invoke("register_model")( model_cost={_key: keep_registered_mode(_key, shared_model_info) for _key in backend_keys}, persist_across_reloads=False, @@ -14517,9 +14791,7 @@ class PythonCore(DeprecatedRouterMembers): """ if team_id is not None and self._team_deployments_named(team_id, model_group): return self._core_get_all_deployments(model_name=model_group, team_id=team_id) - own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns - list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group) - ) + own: Final = self._get_all_deployments(model_name=model_group) named: Final = [*own, *self._core_get_model_list_from_routing_groups(model_name=model_group)] return named or self._get_wildcard_deployments(model_name=model_group) @@ -14827,6 +15099,7 @@ class PythonCore(DeprecatedRouterMembers): input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, tokenizer_model: str = "", + deployments: Sequence[Mapping[str, object]] = (), ) -> int: """ Count input tokens for context-window pre-call checks. @@ -14861,11 +15134,15 @@ class PythonCore(DeprecatedRouterMembers): if input is not None: from openai.types.responses.response_create_params import ResponseInputParam + from litellm.litellm_core_utils.token_counter import split_embedding_input_tokens from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) - typed_input: Final = cast(str | ResponseInputParam, input) # cast-ok: str | list matches transform input + text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) + typed_input: Final = cast( # cast-ok: str | list matches transform input + str | ResponseInputParam, text_input + ) input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=typed_input, responses_api_request={"instructions": instructions} if instructions is not None else {}, @@ -14933,6 +15210,7 @@ class PythonCore(DeprecatedRouterMembers): self.served_openai_models(model) or served_openai_models(healthy_deployments) ) or "", + deployments=healthy_deployments, ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( @@ -14963,6 +15241,7 @@ class PythonCore(DeprecatedRouterMembers): self.served_openai_models(model) or served_openai_models(deployments) ) or "", + deployments=deployments, ) except Exception as e: verbose_router_logger.error( @@ -15720,10 +15999,15 @@ class PythonCore(DeprecatedRouterMembers): # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN THE TEAM ## THIS PREVENTS WRITING FILES OF OTHER TEAMS TO MODELS THAT ARE TEAM-ONLY MODELS + by_id: Final = healthy_deployments if isinstance(healthy_deployments, dict) else None # pyright: ignore[reportUnknownVariableType] # the legacy deployment dict healthy_deployments = filter_team_based_models( healthy_deployments=healthy_deployments, request_kwargs=request_kwargs, ) + if by_id is not None and not healthy_deployments: + # Another team's deployment asked for by id: refused under its group's name. + group_name: Final = str(by_id.get("model_name")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + model = group_name # rebind-ok: the error names the group if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug("healthy_deployments after team filter: %s", healthy_deployments) @@ -16046,6 +16330,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, specific_deployment=specific_deployment, + reserve=pick_reserves(default=True), ) async def _async_get_available_deployment( @@ -16055,6 +16340,7 @@ class PythonCore(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, specific_deployment: bool | None, + reserve: bool, ): if ( self._core_routing_strategy != "usage-based-routing-v2" @@ -16151,6 +16437,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=reserve and request_kwargs not in (None, {}), ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16320,6 +16607,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17029,6 +17317,16 @@ class PythonCore(DeprecatedRouterMembers): model, ) ) + # As the async pick: a team-owned deployment serves only its own team's callers, also when + # asked for by id (then refused under its group's name). + by_id: Final = healthy_deployments if isinstance(healthy_deployments, dict) else None # pyright: ignore[reportUnknownVariableType] # the legacy deployment dict + healthy_deployments = filter_team_based_models( # rebind-ok: the team filter narrows # pyright: ignore[reportUnknownVariableType] # legacy selector type + healthy_deployments=healthy_deployments, # pyright: ignore[reportArgumentType] # legacy selector type + request_kwargs=request_kwargs, + ) + if by_id is not None and not healthy_deployments: + group_name: Final = str(by_id.get("model_name")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + model = group_name # rebind-ok: the error names the group strategy, strategy_selector = self._get_routing_context(model, request_kwargs) routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) @@ -17197,6 +17495,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: @@ -17377,6 +17676,7 @@ class PythonCore(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, + reserve=pick_reserves(default=False) and request_kwargs not in (None, {}), ) if deployment is None: diff --git a/litellm/router_python_host.py b/litellm/router_python_host.py index 8156740144e..8ab2c06b82f 100644 --- a/litellm/router_python_host.py +++ b/litellm/router_python_host.py @@ -7,7 +7,10 @@ come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the sam """ import asyncio -from collections.abc import Awaitable, Callable, Sequence +import importlib +import time +import types +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import ( TYPE_CHECKING, Final, @@ -19,12 +22,13 @@ from typing import ( from typing_extensions import overload import litellm -from litellm.router_config import RouterDynamicConfig, live_dynamic_config +from litellm.router_config import RouterConfig, RouterDynamicConfig, live_dynamic_config if TYPE_CHECKING: from pydantic import BaseModel - from litellm.caching.caching import Cache + from litellm.caching.caching import Cache, RedisCache + from litellm.caching.dual_cache import DualCache from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper @@ -97,7 +101,7 @@ class PythonHost: def invoke(self, op: str) -> Endpoint: ... def invoke(self, op: str) -> Endpoint: """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time.""" - endpoint: Final[object] = getattr(litellm, op, None) + endpoint: Final[object] = getattr(litellm, op, None) if "." not in op else _dotted(op) if endpoint is None: raise AttributeError(f"litellm has no endpoint {op!r}") if not callable(endpoint): @@ -107,6 +111,10 @@ class PythonHost: def sleep(self, seconds: float) -> Awaitable[None]: return asyncio.sleep(seconds) + def sleep_sync(self, seconds: float) -> None: + """The backoff of a sync endpoint's retry.""" + time.sleep(seconds) + def dynamic(self) -> RouterDynamicConfig: """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot).""" return cast(RouterDynamicConfig, live_dynamic_config) # cast-ok: the live view reads the same fields @@ -116,7 +124,7 @@ class PythonHost: return litellm.logging_callback_manager def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the process callback list itself - value: Final = getattr(litellm, name) + value: Final[object] = getattr(litellm, name) # pyright: ignore[reportAny] # an untyped litellm global return cast(list[object], value) # cast-ok: the process callback list is an untyped litellm global def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given @@ -131,5 +139,78 @@ class PythonHost: def suppress_debug_info(self) -> None: litellm.suppress_debug_info = True + def routing_cache(self, config: RouterConfig) -> "DualCache": + """The store a router keeps its routing state in (cooldowns, fail counts, usage): in memory, + plus Redis when the router's arguments name one. The host builds and owns it; a Rust core + reads and writes it through cache steps.""" + from litellm.caching.caching import DualCache, InMemoryCache + + return DualCache(redis_cache=redis_cache_for(config), in_memory_cache=InMemoryCache()) + + +def redis_cache_for(config: RouterConfig) -> "RedisCache | None": + """The Redis cache the router's ``redis_*`` and ``cache_kwargs`` arguments describe, or None.""" + import json + + from litellm.caching.caching import RedisCache, RedisClusterCache + from litellm.utils import get_secret + + url: Final = config.get("redis_url") + host: Final = config.get("redis_host") + port: Final = config.get("redis_port") + if url is None and (host is None or port is None): + return None + settings: Final[dict[str, object]] = {} # mutable-ok: built here, then passed as keywords + if url is not None: + settings["url"] = url + if host is not None: + settings["host"] = host + if port is not None: + settings["port"] = str(port) + if config.get("redis_password") is not None: + settings["password"] = config.get("redis_password") + if config.get("redis_db") is not None: + settings["db"] = str(config.get("redis_db")) + extra: Final = config.get("cache_kwargs") + if isinstance(extra, dict): + settings.update({str(key): value for key, value in extra.items()}) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # the cache_kwargs argument is untyped + nodes: object = settings.get("startup_nodes") # rebind-ok: the env cluster nodes when the arguments name none + if not nodes: + env_nodes: Final[object] = get_secret("REDIS_CLUSTER_NODES") # pyright: ignore[reportUnknownVariableType] # get_secret is untyped + if isinstance(env_nodes, str): + parsed: Final[object] = json.loads(env_nodes) # pyright: ignore[reportAny] # plain JSON + nodes = parsed # rebind-ok: see above + if nodes: + return RedisClusterCache(**{**settings, "startup_nodes": nodes}) # pyright: ignore[reportArgumentType] # RedisCache's keyword arguments + return RedisCache(**settings) # pyright: ignore[reportArgumentType] # RedisCache's keyword arguments + python_host: Final = PythonHost() + + +def _dotted(op: str) -> object: + """``op`` named by its path under ``litellm`` (``vector_stores.main.asearch``), read at call time. + + Each segment is a module attribute, a submodule, or a key of a mapping such as a module's + table of generated endpoints. + """ + node: object = litellm # rebind-ok: walks the path one segment at a time + path: str = "litellm" # rebind-ok: the module path walked so far + for segment in op.split("."): + path = f"{path}.{segment}" + if isinstance(node, Mapping): + node = cast(Mapping[str, object], node).get(segment) # cast-ok: endpoint tables are keyed by name + elif isinstance(node, types.ModuleType) and (submodule := _submodule(path)) is not None: + node = submodule + else: + node = cast(object, getattr(node, segment, None)) # cast-ok: any attribute + if node is None: + return None + return node + + +def _submodule(path: str) -> types.ModuleType | None: + try: + return importlib.import_module(path) + except ModuleNotFoundError: + return None diff --git a/litellm/router_rust/__init__.py b/litellm/router_rust/__init__.py new file mode 100644 index 00000000000..42cfc56aca4 --- /dev/null +++ b/litellm/router_rust/__init__.py @@ -0,0 +1,13 @@ +""" +The Rust router core's Python side: ``RustCore`` (``core``) and the host-side modules router-core +(litellm-rust/crates/router-core) reaches through its steps. + +- ``steps``: answers to host steps that need live Python objects (strategies, pre-routing, + pre-call facts, plugins, callbacks, caller access) +- ``invoke``: the host side of ``Host::invoke``, one attempt against one deployment (request + shaping, endpoints, streams, slots, retry and fallback metadata, prompt management, websockets) +- ``surface``: the Python ``Router`` API kept over router-core's state (registry views, model + info, settings, health, retries, fallbacks, alerting); a Rust host needs none of it + +Nothing is imported here, so importing one module does not import the core. +""" diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py new file mode 100644 index 00000000000..395b580d9b2 --- /dev/null +++ b/litellm/router_rust/core.py @@ -0,0 +1,4943 @@ +""" +``RustCore``: the router core served by litellm-rust/crates/router-core through ``_native``. + +The Rust core owns the registry, selection, retries, fallbacks and cooldowns. It reaches outside +itself only through its host, and the host never calls back into a Python router: + +- every attempt runs through the host: ``router_attempt`` builds the request for the one + deployment Rust picked and ``PythonHost.invoke(op)`` calls ``litellm.`` with it +- the core records every outcome (cooldowns, allowed-fails counters, usage) itself, in the routing + cache the host builds and owns (``PythonHost.routing_cache``), as far as the outcome callbacks the + Python router would have running for the request reach (``OptionalChecks.outcome_callbacks``) +- sleeps, fallback access and budget checks, and cache reads and writes are host steps + +``RustCore`` does not build, wrap or read a ``PythonCore``. Settings are resolved from the +``RouterConfig`` and the host's process settings; the views (``registry()``, ``settings()``, +``health()``) are read from the Rust handle. A member the Rust core does not serve raises +``RustCoreUnsupported`` (an ``AttributeError``) instead of reaching a Python core, so deleting +``router_python_core.py`` cannot change what this core does. +""" + +import contextlib +import dataclasses +import inspect +import itertools +import json +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Mapping, Sequence +from datetime import date, datetime, time +from functools import partial +from time import perf_counter +from time import time as unix_time +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, final + +from litellm._internal_context import service_target +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, HEALTH_CHECKS_TARGET +from litellm.integrations.custom_guardrail import is_guardrail_intervention +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, + add_retry_headers_to_response, + apply_quality_router_decision_headers, + apply_remaining_usage_headers, + apply_response_model_id, + complexity_router_decision_headers, + ensure_response_additional_headers, + prepare_response_for_header_attachment, +) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, without +from litellm.litellm_core_utils.routing_common_utils import ( + format_content_policy_fallback_hint, + format_fallback_outcome_message, + format_no_fallback_group_message, + get_request_team_id, + is_web_search_request, + truncate_fallback_error_detail, +) +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.router_api import ( + ROUTER_INIT_ARGS, + SETTING_FIELDS, + ApplyResult, + ChangeSet, + RouterHealth, + RouterRegistry, + RouterSettings, + RoutingContext, + RoutingObservers, +) +from litellm.router_attempt import for_requested_model, request_from_call +from litellm.router_config import RouterConfig, RouterDynamicConfig, detached +from litellm.router_python_host import PythonHost +from litellm.router_rust.invoke.endpoints import ( + AttemptCall, + Plan, + RustEndpoints, + arm_compaction_state, + attempt_call, + clean_realtime_session_deployment, + compacted_attempt, + failover_fields, + generic_plan, + log_refused_call, + log_refused_call_sync, + pinned_video_deployment, + plan_for, + reports_own_success, + scoped_fallback_fields, + selection_endpoint, + stamp_error, + unrouted_fields, +) +from litellm.router_rust.invoke.shaping import Shaping +from litellm.router_rust.invoke.slots import enter_deployment_slot, max_parallel_requests_limit +from litellm.router_rust.invoke.streaming import ( + StreamAttempt, + continuation_flag, + continuation_routing, + open_attempt_stream, + open_attempt_stream_sync, + register_continuation_prefill_check, + shape_attempt_kwargs, +) +from litellm.router_rust.invoke.walk import FallbackOutcomes, WalkMetadata, await_all, log_retry_into +from litellm.router_rust.steps.access import caller_fields, selection_refusal +from litellm.router_rust.steps.callbacks import OptionalChecks, RustOptionalChecks +from litellm.router_rust.steps.plugins import SYNC_PATH_MESSAGE, candidate_models, run_plugins +from litellm.router_rust.steps.precall import ( + DECISIONS_MEDIA_CALL_TYPES, + PreCallFacts, + cannot_serve_route, + declares_no_vision, + refuses_decisions_media, + request_carries_image, + serves_route_capability, +) +from litellm.router_rust.steps.prerouting import PreRouter, PreRoutingStrategies, picks_async +from litellm.router_rust.steps.selection import ( + SELECTOR_ATTRIBUTES, + StrategySelectors, + group_documents, + group_settings, + member_map, + normalize_strategy, + parse_groups, + pick_async, + pick_sync, + picked_id, + routing_pre_call_checks, +) +from litellm.router_rust.surface.alerting import initialize_alerting +from litellm.router_rust.surface.fallbacks import ( + RustFallbacks, + mock_error, + mock_failure, + refuse_malformed_fallbacks, + rejected_fallback_target, +) +from litellm.router_rust.surface.health import ( + RustHealthCallbacks, + failure_already_logged, + failure_status, + first_cooldown, + logged_call, +) +from litellm.router_rust.surface.health_routing import ( + HEALTH_STATE_KEY, + HealthRouting, + failover_document, + fully_unhealthy_model_names, + resolved_allowed_fails_policy, +) +from litellm.router_rust.surface.health_routing import all_blocked as deployments_all_blocked +from litellm.router_rust.surface.health_routing import fully_blocked_model_names as blocked_model_names +from litellm.router_rust.surface.health_routing import is_blocked as deployment_is_blocked +from litellm.router_rust.surface.legacy_members import RustLegacyMembers +from litellm.router_rust.surface.members import RustRegistryMembers +from litellm.router_rust.surface.model_list import typed_deployment +from litellm.router_rust.surface.registry import ( + DEFAULT_DEPLOYMENT_COPIED_ID, + DEFAULT_DEPLOYMENT_FAILING_ID, + INDEX_ATTRIBUTES, + BuildInstall, + HostRegistry, + RustRegistry, + default_deployment_call_error, + default_deployment_copy, + default_deployment_copy_error, + hosted_registry_view, + index_document, + served_default_deployment, +) +from litellm.router_rust.surface.replay import track_live_registry, untrack_live_registry +from litellm.router_rust.surface.retries import ( + GroupPolicies, + RustRetries, + group_policies, + is_policy_per_group, + policy_lookup_error, + raising_groups, + readable_retry_policy, + reads_as_policy, + request_retry_fields, + resolved_retry_policy, + retry_count_error, + static_group_policies, +) +from litellm.router_rust.surface.settings import ( + MISSING, + PLAIN_DEFAULTS, + SettingsUpdate, + apply_construction_effects, + argument_attribute, + default_litellm_params, + initial_plain, + read_settings_update, + request_timeout, + settings_view, +) +from litellm.router_strategy.least_busy import CALL_IDENTITY, call_id_of, owning_picks, pick_reserves, picks_reserve +from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET +from litellm.router_utils.cooldown_handlers import get_cooldown_deployment_info, mark_deployment_selection_failure +from litellm.router_utils.endpoint_eligibility import endpoint_for_call, selected_endpoint +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after_bound, + resolve_pods_after_bound, +) +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingReadBatch +from litellm.types.router import ( + Deployment, + RouterErrors, + RouterGeneralSettings, + RouterRateLimitError, + RoutingGroup, + RoutingStrategy, +) +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.caching.dual_cache import DualCache + from litellm.caching.redis_cache import RedisCache + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router_api import UntypedDict + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.router_utils.health_state_cache import DeploymentHealthCache + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _CallSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + _FailureReportSteps, # pyright: ignore[reportPrivateUsage] # see above + _SuccessReportSteps, # pyright: ignore[reportPrivateUsage] # see above + ) + from litellm.types.router import ( + CustomRoutingStrategyBase, + DeploymentTypedDict, + GuardrailTypedDict, + PreRoutingHookResponse, + RoutingPlugin, + ) + +_FAILURE_KINDS: Final = ( + "RateLimitError", + "ContextWindowExceededError", + "ContentPolicyViolationError", + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "BadRequestError", + "Timeout", + "APIConnectionError", + "InternalServerError", + "ServiceUnavailableError", + "BadGatewayError", + "GuardrailIntervention", + "CallerCredentialAuthenticationError", + "CallerCredentialRateLimitError", +) +_DECLINED: Final = frozenset({"MalformedRequestFieldError", "UnsupportedEndpointError"}) +_PROVIDERS: Final = tuple(provider.value for provider in LlmProviders) +_DEFAULT_COOLDOWN_SECONDS: Final = 5.0 +# Past this a retry-after no longer fits the Rust core's durations; it cools down as good as forever. +_LONGEST_RETRY_AFTER_SECONDS: Final = 2**63 + +# Constructor arguments this core reads. Any other argument set to something other than its +# unset value keeps the router on the Python core while both exist (make_core decides that from +# the config alone, before any core is built). +_SERVED_ARGUMENTS: Final = frozenset( + { + "model_list", + "routing_strategy", + "routing_strategy_args", + "routing_groups", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "model_group_alias", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_after", + "timeout", + "default_litellm_params", + "fallback_access_check", + "fallback_budget_check", + "enable_mid_stream_fallback_continuation", + "redis_url", + "redis_host", + "redis_port", + "redis_password", + "redis_db", + "cache_kwargs", + "set_verbose", + "debug_level", + # Read host-side by the endpoint plans (litellm/router_rust/invoke/endpoints.py). + "stream_timeout", + "cache_responses", + "assistants_config", + "search_tools", + "guardrail_list", + "default_priority", + "polling_interval", + "ignore_invalid_deployments", + # Read host-side by the pre-routing strategies (litellm/router_rust/steps/prerouting.py). + "auto_router_capability_limit", + # Decided in router-core (precall.rs); its facts come from litellm/router_rust/steps/precall.py. + "enable_pre_call_checks", + # Run host-side (litellm/router_rust/steps/plugins.py); router-core filters on what they leave. + "plugins", + "retry_policy", + "model_group_retry_policy", + # Health-check routing and the failover flags (litellm/router_rust/surface/health_routing.py). + "enable_health_check_routing", + "health_check_staleness_threshold", + "health_check_ignore_transient_errors", + "background_health_check_model_groups", + "enable_weighted_failover", + "fallback_on_unsupported_params", + # Built host-side as CustomLogger checks (litellm/router_rust/steps/callbacks.py); router-core runs + # their deployment filters through its callback_filters stage. + "optional_pre_call_checks", + "provider_budget_config", + "deployment_affinity_ttl_seconds", + "session_affinity_repin_on_failover", + "model_group_affinity_config", + # Kept as given (litellm/router_rust/surface/settings.py) and read host-side by the endpoint plans + # (default_litellm_params' caching groups) and the slot limits. + "client_ttl", + "caching_groups", + "default_max_parallel_requests", + # Built host-side into a Slack alerting logger (litellm/router_rust/surface/alerting.py). + "alerting_config", + # Kept as given and reported by settings(); nothing in the router reads its fields. + "router_general_settings", + } +) +# Arguments that may be passed at their default without asking for anything. +_UNSET_DEFAULTS: Final[Mapping[str, object]] = MappingProxyType( + { + "cache_responses": False, + "enable_pre_call_checks": False, + "enable_tag_filtering": False, + "retry_policy": None, + "model_group_retry_policy": {}, + } +) +# Settings a changeset may change. Static ones rebuild the Rust core (settings are fixed for a +# core's life); dynamic ones are handed to it as a new dynamic document. +_STATIC_SETTINGS: Final = frozenset( + { + "routing_strategy", + "routing_groups", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_after", + "timeout", + "enable_pre_call_checks", + } +) +_DYNAMIC_SETTINGS: Final = frozenset( + { + "default_deployment", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "model_group_alias", + "default_litellm_params", + "fallback_access_check", + "fallback_budget_check", + "routing_strategy_args", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + } +) +# Settings only the host reads (the endpoint plans, the slot limits, the budget limiter, the +# settings view). A changeset assigns them as given, the Rust core never sees them, and +# `update_settings` ignores them (none is runtime-updatable). +_HOST_SETTINGS: Final = frozenset( + { + "cache_responses", + "search_tools", + "guardrail_list", + "default_max_parallel_requests", + "provider_budget_config", + "health_check_ignore_transient_errors", + "router_general_settings", + "provider_default_deployment_ids", + "kubernetes_pod_discovery", + # Read by router-core's health-check filter: assigning one rebuilds the Rust core. + "enable_health_check_routing", + "background_health_check_model_groups", + } +) +# The constructor's default `router_general_settings`: one object every router built without the +# argument shares, as the Python core's default argument is. +_DEFAULT_GENERAL_SETTINGS: Final = RouterGeneralSettings() + + +# Plain settings `update_settings` sets that the Rust core's retry loop reads. +_RETRY_POLICY_SETTINGS: Final = ("retry_policy", "model_group_retry_policy") +# Attributes a `del` unsets beyond the setting fields, whose readers then raise AttributeError. +_DELETABLE_ATTRIBUTES: Final = frozenset({"model_group_alias", "num_retries", "scheduler"}) + + +# Calls whose pick least-busy does not hold (`picks_reserve(False)`): a batch cancel makes no +# call that logs a start. +_UNRESERVED_OPS: Final = frozenset({"acancel_batch"}) + + +# Settings converted by `Resolved._selection_setting`. +_SELECTION_SETTINGS: Final = frozenset( + { + "routing_strategy", + "routing_strategy_args", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + } +) +_NUMBER_SETTINGS: Final = frozenset( + {"num_retries", "max_fallbacks", "allowed_fails", "cooldown_time", "retry_after", "timeout"} +) +# The settings `_core_update_settings` casts with `int(value)`. +_UPDATE_INT_SETTINGS: Final = frozenset({"timeout", "num_retries", "retry_after", "allowed_fails", "cooldown_time"}) + + +class RustCoreUnsupported(AttributeError): + """A member or request the Rust core does not serve. It is never forwarded to a Python core.""" + + +def _cache_target(key: str) -> str: + """The service target a routing cache read or write is attributed to, as the Python core's.""" + if key == HEALTH_STATE_KEY: + return HEALTH_CHECKS_TARGET + return ROUTER_COOLDOWNS_TARGET if key.endswith(":cooldown") else ROUTER_USAGE_TARGET + + +# --- failures ----------------------------------------------------------------------------------- + + +def failure_kind(error: BaseException) -> str: + names: Final = tuple(cls.__name__ for cls in type(error).__mro__) + if not _DECLINED.isdisjoint(names): + return "Declined" + if is_guardrail_intervention(error): # pyright: ignore[reportArgumentType] # every attempt error is an Exception + return "GuardrailIntervention" + return next((name for name in names if name in _FAILURE_KINDS), "APIError") + + +def retry_after_seconds(error: BaseException) -> float | None: + """The header cooldown of ``deployment_callback_on_failure``: ``retry-after`` alone, as a whole + number or a date, with a negative or unreadable one read as none.""" + from litellm.litellm_core_utils.exception_mapping_utils import ( + _get_response_headers, # pyright: ignore[reportPrivateUsage] # the failure callback's own read + ) + from litellm.utils import _get_retry_after_from_exception_header # pyright: ignore[reportPrivateUsage] # see above + + headers: Final = _get_response_headers(original_exception=error) # pyright: ignore[reportArgumentType] # every attempt error is an Exception + if headers is None: + return None + seconds: Final = _get_retry_after_from_exception_header(response_headers=headers) # pyright: ignore[reportUnknownVariableType] # an untyped helper's int or None + return float(min(seconds, _LONGEST_RETRY_AFTER_SECONDS)) if seconds is not None and seconds >= 0 else None + + +def attempt_failure(error: BaseException) -> Mapping[str, object]: + status: Final = getattr(error, "status_code", None) + num_retries: Final = getattr(error, "num_retries", None) + # `_deployment_ids_to_skip_on_retry` reads the id off the exception. + named_deployment: Final = getattr(error, "retry_skip_deployment_id", None) or getattr( + error, "failed_deployment_id", None + ) + return { + "ok": False, + "kind": failure_kind(error), + "status": status if isinstance(status, int) and 100 <= status <= 599 else None, + "status_absent": getattr(error, "status_code", "") == "", + "retry_after": retry_after_seconds(error), + "num_retries": num_retries if isinstance(num_retries, int) and num_retries >= 0 else None, + "failed_deployment_id": named_deployment if isinstance(named_deployment, str) else None, + "detail": str(error), + "classes": [cls.__name__ for cls in type(error).__mro__], + "llm_provider": bool(getattr(error, "llm_provider", "")), + "agentic_followup": is_agentic_followup_failure(error), + } + + +def total_tokens(response: object) -> int: + usage: Final = getattr(response, "usage", None) + tokens: Final = getattr(usage, "total_tokens", None) + return tokens if isinstance(tokens, int) else 0 + + +# --- settings ----------------------------------------------------------------------------------- + + +def _json_default(value: object) -> object: + # ISO 8601, which ``typed_deployment`` reads back as the value it was. + if isinstance(value, date | time): + return value.isoformat() + dump: Final = getattr(value, "model_dump", None) + return dump(exclude_none=True) if callable(dump) else str(value) + + +def _plain(value: object) -> object: + """``value`` as plain JSON values, the shape the Rust core reads.""" + loaded: Final[object] = json.loads(json.dumps(value, default=_json_default)) # pyright: ignore[reportAny] # plain JSON + return loaded + + +def _document(value: object) -> Mapping[str, object]: + """A mapping as the plain JSON document the Rust core reads.""" + return as_str_mapping(_plain(value)) or {} + + +def _as_list(value: object) -> Sequence[object]: + return value if isinstance(value, list) else () # pyright: ignore[reportUnknownVariableType] # a list's items are read as objects + + +def _strategy_name(value: object) -> str | None: + """A strategy as a setting stores it: ``None`` stays no strategy (``_normalize_strategy``).""" + return None if value is None else normalize_strategy(value) or "simple-shuffle" + + +def _lenient_int(value: object, default: int) -> int: + """A count setting kept as given, as the crate reads it: an int, or a number spelled another way + (``1.5``, ``"3"``) truncated to one; ``default`` when it is no number. The Python core stores the + value as given and never checks it at construction.""" + if isinstance(value, bool | int): + return int(value) + try: + return int(float(str(value))) + except (ValueError, OverflowError): + return default + + +def _lenient_float(value: object, default: float | None) -> float | None: + """A seconds setting kept as given, as the crate reads it (see ``_lenient_int``).""" + if isinstance(value, bool | int | float): + return float(value) + try: + return float(str(value)) + except (ValueError, OverflowError): + return default + + +# Settings the Python core stores as given, so that a read returns the value passed (``"1"``, +# ``None``, ``[1]``); ``Resolved`` keeps them in ``given`` and the typed fields hold what the crate reads. +_KEPT_AS_GIVEN: Final = frozenset( + { + "num_retries", + "max_fallbacks", + "allowed_fails", + "retry_after", + "timeout", + "disable_cooldowns", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "routing_strategy_args", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + } +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Resolved: + """The router's settings, resolved from its arguments and the process settings the way the + Python router's constructor resolves them. Replaced, never changed, by ``apply``.""" + + routing_strategy: str | None + routing_strategy_args: Mapping[str, object] + num_retries: int + max_fallbacks: int + allowed_fails: int + explicit_allowed_fails: int | None + cooldown_time: float + disable_cooldowns: bool | None + allowed_fails_policy: object + retry_after: float + timeout: float | None + fallbacks: list[object] | None # mutable-ok: the setting as Python spells it + context_window_fallbacks: list[object] | None # mutable-ok: same + content_policy_fallbacks: list[object] | None # mutable-ok: same + model_group_alias: Mapping[str, object] + default_litellm_params: Mapping[str, object] + fallback_access_check: object + fallback_budget_check: object + enable_mid_stream_fallback_continuation: bool = False + routing_groups: "tuple[RoutingGroup, ...]" = () + enable_tag_filtering: bool = False + tag_filtering_match_any: bool = True + tag_routing_prefix: str = "" + enable_pre_call_checks: bool = False + routing_plugins: "tuple[RoutingPlugin, ...]" = () + retry_policy: object = None + model_group_retry_policy: object = None + health_routing: HealthRouting = dataclasses.field(default_factory=HealthRouting) + # `_core_default_deployment`, as assigned (detached). + default_deployment: object = None + # `_model_to_group` as the Python core last built it, kept once a raw `routing_groups` write + # replaced the groups without rebuilding it. + routing_group_members: Mapping[str, str] | None = None + # `_routing_groups_input`: the groups last passed to the constructor or `update_settings`, which + # a strategy change through `update_settings` re-inits the routing groups from. + routing_groups_input: object = None + assigned: frozenset[str] = frozenset() + # The `_KEPT_AS_GIVEN` settings as the Python core stores them, for reads. + given: Mapping[str, object] = dataclasses.field(default_factory=lambda: MappingProxyType({})) + + def read(self, name: str) -> object: + """A setting as the Python core reads it back: as given when it keeps it so.""" + return self.given[name] if name in self.given else getattr(self, name) # pyright: ignore[reportAny] # a setting field + + @classmethod + def of(cls, config: RouterConfig, dynamic: RouterDynamicConfig) -> "Resolved": + get: Final = config.get + # The process list is the router's own, as in the Python router: default_fallbacks are + # appended to it in place. + configured: Final[list[object] | None] = ( # mutable-ok: the router's live list, as Python keeps it + list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given + ) + refuse_malformed_fallbacks(configured) + default_fallbacks: Final = get("default_fallbacks") or dynamic.default_fallbacks + if isinstance(default_fallbacks, str): + raise TypeError( + f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']" + ) + if default_fallbacks is not None and configured is not None: + configured.append({"*": default_fallbacks}) + fallbacks: Final[list[object] | None] = ( # mutable-ok: see above + [{"*": default_fallbacks}] if default_fallbacks is not None and configured is None else configured + ) + content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or ( + list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None + ) + refuse_malformed_fallbacks(content_policy) + num_retries: Final = get("num_retries") + max_fallbacks: Final = get("max_fallbacks") + allowed_fails: Final = get("allowed_fails") + timeout: Final = get("timeout") + strategy: Final = get("routing_strategy") + validate_routing_strategy(strategy) # pyright: ignore[reportArgumentType] # validated as given + # The values the Python core's constructor stores (`self.num_retries = num_retries`, ...). + given_num_retries: Final[object] = ( + num_retries if num_retries is not None else dynamic.num_retries if dynamic.num_retries is not None else 2 + ) + given_max_fallbacks: Final[object] = ( + max_fallbacks + if max_fallbacks is not None + else dynamic.max_fallbacks + if dynamic.max_fallbacks is not None + else dynamic.router_max_fallbacks + ) + given_allowed_fails: Final[object] = allowed_fails if allowed_fails is not None else dynamic.allowed_fails + given_timeout: Final[object] = timeout or dynamic.request_timeout + given_retry_after: Final[object] = get("retry_after") if "retry_after" in config.arguments else 0 + kept: Final = { + "num_retries": given_num_retries, + "max_fallbacks": given_max_fallbacks, + "allowed_fails": given_allowed_fails, + "timeout": given_timeout, + "retry_after": given_retry_after, + "disable_cooldowns": get("disable_cooldowns"), + **{ + name: get(name) + for name in ("enable_tag_filtering", "tag_filtering_match_any", "tag_routing_prefix") + if name in config.arguments + }, + **( + {"routing_strategy_args": get("routing_strategy_args")} + if "routing_strategy_args" in config.arguments + else {} + ), + } + return cls( + # An explicit `routing_strategy=None` is stored as None (no strategy picks). + routing_strategy=None + if strategy is None and "routing_strategy" in config.arguments + else normalize_strategy(strategy) or "simple-shuffle", + # A plain dict, as `self.routing_strategy_args`: `/router/settings` serializes it as read. + routing_strategy_args=dict(as_str_mapping(get("routing_strategy_args")) or {}), + num_retries=_lenient_int(given_num_retries, 2), + max_fallbacks=_lenient_int(given_max_fallbacks, dynamic.router_max_fallbacks), + allowed_fails=_lenient_int(given_allowed_fails, dynamic.allowed_fails), + explicit_allowed_fails=_lenient_int(allowed_fails, dynamic.allowed_fails) + if allowed_fails is not None + else None, + cooldown_time=_lenient_float(get("cooldown_time") or _DEFAULT_COOLDOWN_SECONDS, _DEFAULT_COOLDOWN_SECONDS) + or 0.0, + disable_cooldowns=_optional_bool(get("disable_cooldowns")), + allowed_fails_policy=resolved_allowed_fails_policy(get("allowed_fails_policy")), + retry_after=_lenient_float(given_retry_after or 0, 0.0) or 0.0, + timeout=_lenient_float(given_timeout, None) if given_timeout else None, + given=MappingProxyType(kept), + fallbacks=fallbacks, + context_window_fallbacks=list(_as_list(get("context_window_fallbacks"))) + or (list(dynamic.context_window_fallbacks) if dynamic.context_window_fallbacks else None), + content_policy_fallbacks=content_policy, + model_group_alias=MappingProxyType(dict(as_str_mapping(get("model_group_alias")) or {})), + default_litellm_params=MappingProxyType(dict(as_str_mapping(get("default_litellm_params")) or {})), + fallback_access_check=get("fallback_access_check"), + fallback_budget_check=get("fallback_budget_check"), + enable_mid_stream_fallback_continuation=continuation_flag(get("enable_mid_stream_fallback_continuation")), + enable_tag_filtering=bool(get("enable_tag_filtering")), + tag_filtering_match_any=_true_unless_given(get("tag_filtering_match_any")), + tag_routing_prefix=str(get("tag_routing_prefix") or ""), + enable_pre_call_checks=bool(get("enable_pre_call_checks")), + routing_plugins=tuple(_as_list(get("plugins"))), # pyright: ignore[reportArgumentType] # the plugins as passed + routing_groups_input=get("routing_groups"), + routing_groups=parse_groups( + get("routing_groups"), + frozenset( + str(name) + for entry in _as_list(get("model_list")) + if (name := (as_str_mapping(entry) or {}).get("model_name")) + ), + frozenset(as_str_mapping(get("model_group_alias")) or {}), + ), + retry_policy=resolved_retry_policy(get("retry_policy")), + model_group_retry_policy=get("model_group_retry_policy"), + health_routing=HealthRouting.of(config), + ) + + def changed(self, values: Mapping[str, object], updated: frozenset[str] = frozenset()) -> "Resolved": + """``values`` applied; ``updated`` names those passed as ``ChangeSet.settings`` (which go + through ``update_settings``: only they may name ``routing_strategy_args``).""" + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + unknown: Final = sorted( + set(values) + - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES} + - (updated & {"routing_strategy_args", "routing_groups", "enable_tag_filtering", "tag_routing_prefix"}) + ) + if unknown: + raise ValueError(f"not assignable router settings: {unknown}") + unserved: Final = sorted(set(values) - _STATIC_SETTINGS - _DYNAMIC_SETTINGS) + if unserved: + raise RustCoreUnsupported(f"the Rust router core does not serve the setting(s) {', '.join(unserved)} yet") + updates: Final[dict[str, object]] = {} # mutable-ok: built here, then applied once + for name, value in values.items(): + if name in {"model_group_alias", "default_litellm_params"}: + updates[name] = MappingProxyType(dict(as_str_mapping(value) or {})) + elif name in _NUMBER_SETTINGS: + updates[name] = self._number_setting(name, value) + if name == "allowed_fails": + updates["explicit_allowed_fails"] = updates[name] + elif name in _SELECTION_SETTINGS: + updates[name] = self._selection_setting(name, value) + elif name == "disable_cooldowns": + updates[name] = _optional_bool(value) + elif name == "enable_pre_call_checks": + updates[name] = bool(value) + elif name in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks"): + updates[name] = list(_as_list(value)) or None + elif name == "default_deployment": + updates[name] = detached(value) + else: + updates[name] = value + if "routing_groups" in values: + # `_init_routing_groups` rebuilds the member map from the new groups. + updates["routing_group_members"] = None + # `_core_update_settings` casts its int settings (`int(value)`, which raises as Python's + # does); every other write is stored as given. + kept: Final = { + name: int(value) if name in updated and name in _UPDATE_INT_SETTINGS else value # pyright: ignore[reportArgumentType] # int() of the value as given, as Python casts it + for name, value in values.items() + if name in _KEPT_AS_GIVEN + } + return dataclasses.replace( + self, + **updates, + assigned=self.assigned | frozenset(values), + given=MappingProxyType({**self.given, **kept}), + ) # pyright: ignore[reportArgumentType] # each value converted to its field's type above + + def _number_setting(self, name: str, value: object) -> object: + """A numeric setting as the crate reads it; the value as given is kept in ``given``.""" + if name in ("num_retries", "max_fallbacks", "allowed_fails"): + return _lenient_int(value, getattr(self, name)) # pyright: ignore[reportAny] # the field's current int + if name == "cooldown_time": + # Stored as given (`_core_cooldown_time`), so a failure cools down for that long: + # 0 for no time at all; None leaves the cooldown cache's own default, set by the caller. + return _lenient_float(value, self.cooldown_time) if value is not None else self.cooldown_time + if name == "retry_after": + return _lenient_float(value or 0, self.retry_after) + return _lenient_float(value, self.timeout) if value is not None else None + + def _selection_setting(self, name: str, value: object) -> object: + """A selection setting converted to its field's type.""" + if name == "routing_strategy": + return _strategy_name(value) + if name == "routing_strategy_args": + return dict(as_str_mapping(value) or {}) + if name == "tag_routing_prefix": + return str(value or "") + return bool(value) + + def static_document(self, model_list: Sequence[object]) -> Mapping[str, object]: + values: Final = { + "model_list": _plain(list(model_list)), + "routing_strategy": self.routing_strategy, + "cooldown_time": self.cooldown_time, + "disable_cooldowns": self.disable_cooldowns, + "allowed_fails_policy": _plain(self.allowed_fails_policy), + "retry_after": self.retry_after, + "timeout": self.timeout, + "routing_groups": group_documents(self.routing_groups) if self.routing_groups else None, + "enable_pre_call_checks": True if self.enable_pre_call_checks else None, + "routing_plugins": True if self.routing_plugins else None, + # `async_callback_filter_deployments` runs on every async candidate list. + "callback_filters": True, + "retry_policy": _plain(self.retry_policy), + "model_group_retry_policy": _plain(static_group_policies(self.model_group_retry_policy)) or None, + "routing_group_members": dict(self.routing_group_members) + if self.routing_group_members is not None + else None, + **self.health_routing.static_document(), + } + kept: Final = {name: value for name, value in values.items() if value is not None} + # An explicit `routing_strategy=None` reaches the Rust core as "None": no strategy picks. + return kept if self.routing_strategy is not None else {**kept, "routing_strategy": "None"} + + def dynamic_document(self, version: int, model_names: Sequence[str]) -> Mapping[str, object]: + values: Final = { + "version": version, + "model_group_alias": _plain(dict(self.model_group_alias)), + "fallbacks": _plain(self.fallbacks), + "context_window_fallbacks": _plain(self.context_window_fallbacks), + "content_policy_fallbacks": _plain(self.content_policy_fallbacks), + "num_retries": self.num_retries, + "max_fallbacks": self.max_fallbacks, + "allowed_fails": self.allowed_fails, + "allowed_fails_is_default": True if self.allowed_fails_is_default() else None, + "fallback_keys": self._fallback_keys(model_names), + "default_deployment": _plain(served_default_deployment(self.default_deployment)[0]), + "default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None, + } + return {name: value for name, value in values.items() if value is not None} + + def allowed_fails_is_default(self) -> bool: + """``allowed_fails`` does not count as set on the router (``_is_allowed_fails_set_on_router``): + not passed explicitly, and the value copied at init still equals the live process default.""" + import litellm + + return self.explicit_allowed_fails is None and self.allowed_fails == litellm.allowed_fails + + def _fallback_keys(self, model_names: Sequence[str]) -> Mapping[str, object]: + """The spellings a fallback key matches besides the group name (`get_fallback_model_group`).""" + from litellm.router_utils.fallback_event_handlers import ( + _provider_prefixed_model_group, # pyright: ignore[reportPrivateUsage] # the fallback walk's own spelling rule + ) + + keys: Final = tuple( # comprehension-ok: the first group of each fallback entry, across the three lists + str(next(iter(mapping))) + for entries in (self.fallbacks, self.context_window_fallbacks, self.content_policy_fallbacks) + for entry in _as_list(entries) + if (mapping := as_str_mapping(entry)) + ) + names: Final = {*model_names, *self.model_group_alias} + prefixed: Final = { + name: spelled for name in names if (spelled := _provider_prefixed_model_group(name, keys)) is not None + } + from litellm.types.router import LiteLLMParamsTypedDict + + return { + "providers": _PROVIDERS, + "provider_prefixed": prefixed, + "param_keys": list(LiteLLMParamsTypedDict.__annotations__), + } + + +def _alias_target(target: object) -> str | None: + mapping: Final = as_str_mapping(target) + if mapping is not None: + model: Final = mapping.get("model") + return model if isinstance(model, str) else None + return target if isinstance(target, str) else None + + +def _true_unless_given(value: object) -> bool: + return True if value is None else bool(value) + + +def _group_selectors(groups: "Sequence[RoutingGroup]") -> Sequence[tuple[str, str, Mapping[str, object]]]: + return [ + ( + group.group_name, + group.routing_strategy, + as_str_mapping(group.routing_strategy_args) or {}, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # RoutingGroup's untyped dict + ) + for group in groups + ] + + +def _optional_bool(value: object) -> bool | None: + return None if value is None else bool(value) + + +def unsupported_arguments(config: RouterConfig) -> frozenset[str]: + """Arguments the Rust core does not serve.""" + unserved: Final = frozenset( + name + for name, value in config.arguments.items() + if name not in _SERVED_ARGUMENTS and value not in (None, _UNSET_DEFAULTS.get(name)) + ) + return unserved + + +def _listing_team_id(request: Mapping[str, object]) -> str | None: + """``(kwargs.get("metadata", {}) or {}).get("user_api_key_team_id")``: the team + ``async_function_with_fallbacks_common_utils`` lists a group for reads this bucket alone.""" + metadata: Final = as_str_mapping(request.get("metadata")) + team_id: Final = metadata.get("user_api_key_team_id") if metadata is not None else None + return team_id if isinstance(team_id, str) else None + + +# Request fallback settings, which replace the router's for the call. +_REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", "max_fallbacks") + + +def _strings(value: object) -> "tuple[str, ...]": + """Every string in ``value``, a name or a fallback list of names and dict entries.""" + if isinstance(value, str): + return (value,) + if isinstance(value, Mapping): + entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # read as objects + return tuple(itertools.chain.from_iterable(map(_strings, itertools.chain.from_iterable(entries.items())))) + if isinstance(value, list | tuple): + items: Final[Sequence[object]] = value # pyright: ignore[reportUnknownVariableType] # see above + return tuple(itertools.chain.from_iterable(map(_strings, items))) # pyright: ignore[reportUnknownArgumentType] # read as objects + return () + + +def _routed_model(request: Mapping[str, object]) -> object: + """The request's ``model`` as a recovery of its stream names it (``RustCore.routed_name``).""" + model: Final = request.get("model") + return model if model is None or isinstance(model, str) else str(model) + + +def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """The request fields the Rust core routes on.""" + fields: Final = { + **request_retry_fields(request.get("num_retries")), + **({"cooldown_time": request["cooldown_time"]} if request.get("cooldown_time") is not None else {}), + } + # `_common_checks_available_deployment`: only True names a litellm model. + specific: Final = {"specific_deployment": True} if request.get("specific_deployment") is True else {} + # `record_disable_fallbacks(kwargs, disable_fallbacks is True)`: only True turns them off. + disabled: Final = {"disable_fallbacks": True} if request.get("disable_fallbacks") is True else {} + lists: Final = {key: _plain(request[key]) for key in _REQUEST_FALLBACK_KEYS if key in request} + weights: Final = request.get("_router_weights") + team_id: Final = get_request_team_id(request) + listing_team_id: Final = _listing_team_id(request) + strategy: Final = request.get("routing_strategy") + override: Final = normalize_strategy(strategy) if isinstance(strategy, str | RoutingStrategy) else None + return { + **fields, + **specific, + **({"routing_strategy": override} if override is not None else {}), + **disabled, + **({"router_weights": weights} if weights is not None else {}), + **lists, + **({"team_id": team_id} if team_id else {}), + **({"listing_team_id": listing_team_id} if listing_team_id is not None else {}), + **caller_fields(request), + } + + +# --- views -------------------------------------------------------------------------------------- + + +class RustSettings(RouterSettings): + """``settings()``: the resolved settings as a frozen snapshot.""" + + def guardrail(self, name: str) -> "GuardrailTypedDict": + router: Final = self._router + if not isinstance(router, _SettingsSource): + return super().guardrail(name) + return router.available_guardrail(name) # pyright: ignore[reportReturnType] # the configured guardrail entry + + def as_dict(self) -> Mapping[str, object]: + router: Final = self._router + if not isinstance(router, _SettingsSource): + return super().as_dict() + return router.settings_dict() + + +class _SettingsSource: + def settings_dict(self) -> Mapping[str, object]: + raise NotImplementedError + + def available_guardrail(self, name: str) -> object: + raise NotImplementedError + + +class RustHealth(RouterHealth): + """``health()``: cooldowns, fail counts and usage, read through the host's routing cache and + written through the Rust core.""" + + __slots__ = ("_core",) + + def __init__(self, core: "RustCore") -> None: # pyright: ignore[reportMissingSuperCall] # reads the Rust core, not a Router + self._core = core + + def __getattribute__(self, name: str) -> object: + if name in _HEALTH_SERVED or name.startswith("__") or name == "_core": + found: Final[object] = object.__getattribute__(self, name) # pyright: ignore[reportAny] # the view's own member + return found + raise RustCoreUnsupported(f"health().{name} is not served by the Rust router core", name=name, obj=self) + + @property + def store(self) -> "DualCache": + return self._core._cache # pyright: ignore[reportPrivateUsage] # the health view reads its own core + + @property + def cooldowns(self) -> "CooldownCache": + return self._core._cooldowns # pyright: ignore[reportPrivateUsage] # see above + + def failures_this_minute(self, deployment_id: str) -> int: + count: Final[object] = self.store.get_cache(f"{deployment_id}:fails", local_only=True) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return int(count) if isinstance(count, int | float) else 0 + + def record_failure(self, deployment_id: str, exception: Exception, exception_status: str | int) -> bool: + self._core._sync_allowed_fails() # pyright: ignore[reportPrivateUsage] # see above + failure: Final = { + **attempt_failure(exception), + "status": failure_status(exception_status), + "caller_scoped": self._core.per_user_deployment(deployment_id), + } + handle: Final = self._core._handle # pyright: ignore[reportPrivateUsage] # see above + call: Final = handle.report_failure(deployment_id, failure, count_only=exception_status == "") + done: Final = self._core._drive_sync_steps(call) # pyright: ignore[reportPrivateUsage] # see above + return self._core.cooled_down(done, deployment_id, exception_status) + + def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + return self._core._hosted._core_team_ptu_ceiling(team_id, model_group) # pyright: ignore[reportPrivateUsage] # the host registry's ported ceiling + + def split_priority(self, model: str, kwargs: Mapping[str, object]) -> tuple[int | None, Mapping[str, object]]: + """``_core_split_request_priority``, as the endpoints resolve a request's priority.""" + return self._core._endpoints._split_priority(model, kwargs) # pyright: ignore[reportPrivateUsage] # see store + + def priority_header_fields( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: + """``priority_header_update``, as the endpoints read a priority header.""" + return self._core._endpoints.priority_header_update(model=model, kwargs=kwargs, header=header) # pyright: ignore[reportPrivateUsage] # see store + + @property + def health_checks(self) -> "DeploymentHealthCache": + return self._core._health_checks # pyright: ignore[reportPrivateUsage] # see store + + @property + def latency(self) -> "UntypedDict": # the live latency map, as the Python core keeps it + return self._core._hosted._core_deployment_latency_map # pyright: ignore[reportPrivateUsage] # see team_ptu_ceiling + + @staticmethod + def is_blocked(deployment: "Deployment") -> bool: + return deployment_is_blocked(deployment) + + @staticmethod + def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + return deployments_all_blocked(deployments) + + def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + return blocked_model_names(self._core._hosted._core_get_model_list() or []) # pyright: ignore[reportPrivateUsage, reportArgumentType] # see team_ptu_ceiling + + async def fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + core: Final = self._core + return await fully_unhealthy_model_names( + core._hosted._core_get_model_list() or [], # pyright: ignore[reportPrivateUsage, reportArgumentType] # see team_ptu_ceiling + core._health_checks, # pyright: ignore[reportPrivateUsage] # see store + core._resolved.allowed_fails_policy, # pyright: ignore[reportPrivateUsage] # see store + ) + + async def usage( # pyright: ignore[reportIncompatibleMethodOverride] # the same two kinds + self, model_group: str, *, kind: Literal["current", "remaining"] = "current" + ) -> tuple[int | None, int | None] | Mapping[str, int]: + """The group's usage this minute, read from the routing cache over the host registry's + live model list, as ``get_model_group_usage`` / ``get_remaining_model_group_usage`` read it.""" + hosted: Final = self._core._hosted # pyright: ignore[reportPrivateUsage] # see above + if kind == "current": + return await hosted._core_get_model_group_usage(model_group) # pyright: ignore[reportPrivateUsage] # the ported usage read + if kind == "remaining": + return await hosted._core_get_remaining_model_group_usage(model_group) # pyright: ignore[reportPrivateUsage] # see above + raise ValueError(f"unknown usage kind {kind!r}") + + +# Deprecated attribute writes (``router. = ...`` reaches the core as ``_core_``) of +# registry state this core keeps on its host registry, and of routing state it rebinds. +_HOSTED_STATE: Final = frozenset( + { + "_core_model_names", + "_core_deployment_names", + "_core_team_public_model_names", + "_core_model_name_to_deployment_indices", + "_core_team_model_to_deployment_indices", + "_core_cached_deployment_model_info", + "_core_pattern_router", + "_core_team_pattern_routers", + "_core_routing_groups", + "_core_auto_routers", + "_core_complexity_routers", + "_core_quality_routers", + "_core_adaptive_routers", + "_core_routing_plugins", + "_core_deployment_latency_map", + } +) +_REBOUND_STATE: Final = MappingProxyType( + { + "_core_cache": "_cache", + "_core_cooldown_cache": "_cooldowns", + "_core_health_state_cache": "_health_checks", + "_core_slack_alerting_logger": "_slack_alerting_logger", + } +) + + +_HEALTH_SERVED: Final = frozenset( + { + "store", + "cooldowns", + "failures_this_minute", + "record_failure", + "usage", + "team_ptu_ceiling", + "split_priority", + "priority_header_fields", + "health_checks", + "latency", + "is_blocked", + "all_blocked", + "fully_blocked_model_names", + "fully_unhealthy_model_names", + } +) + + +# --- the core ----------------------------------------------------------------------------------- + + +_NO_INPUT: Final = object() +_EndpointImpl: TypeAlias = Callable[..., Awaitable[object]] +# A strategy select step's strategy, selector, keyed name and candidate deployments. +_StrategyInputs: TypeAlias = tuple[str, object, str, list[dict[str, object]]] + + +@final +class _PickCounts(Protocol): + """The router cache as ``track_deployment_pick`` uses it: each deployment id holds its pick count.""" + + def get_cache( + self, key: object, parent_otel_span: "Span | None" = None, local_only: bool = False + ) -> int | None: ... + + def set_cache(self, key: object, value: int, local_only: bool = False, *, ttl: int = ...) -> None: ... + + +class _EndpointPort: + """``RustCore`` as its endpoints see it (``EndpointCore``): each member forwards to the core's + own state or call drivers, read live, since the core replaces its handle and settings. The + port is the core's own adapter, so it is the one place its private members are read from + outside the class; every ``reportPrivateUsage`` ignore below is for that reason.""" + + __slots__ = ("_core",) + + def __init__(self, core: "RustCore") -> None: + self._core: Final = core + + @property + def router(self) -> "RustCore": + return self._core + + @property + def host(self) -> PythonHost: + return self._core._host # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def handle(self) -> "RouterCoreHandle": + return self._core._handle # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def resolved(self) -> "Resolved": + return self._core._resolved # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def shaping(self) -> Shaping: + return self._core._shaping # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def routing_cache(self) -> "DualCache": + return self._core._cache # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def selectors(self) -> StrategySelectors: + return self._core._selectors # pyright: ignore[reportPrivateUsage] # see the class docstring + + @property + def endpoints(self) -> RustEndpoints: + return self._core._endpoints # pyright: ignore[reportPrivateUsage] # see the class docstring + + async def drive( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: Callable[[Mapping[str, object], str], Awaitable[object]] | None = None, + prepared_already: bool = False, + ) -> object: + return await self._core._drive( # pyright: ignore[reportPrivateUsage] # see the class docstring + plan, request, overrides, passthrough=passthrough, prepared_already=prepared_already + ) + + def call_sync( + self, plan: Plan, request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: + return self._core._call_sync(plan, request, overrides) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def routing_callbacks(self) -> Iterable[object]: + return self._core._checks.routing_callbacks() # pyright: ignore[reportPrivateUsage] # see the class docstring + + def resolve_alias(self, model: str) -> str | None: + return self._core._hosted._core_get_model_from_alias(model) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def group_deployments(self, model_name: str) -> Sequence[Mapping[str, object]] | None: + return self._core._hosted._core_get_model_list(model_name=model_name) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def routed_name(self, model: object) -> str: + return self._core.routed_name(model) + + def service_logged(self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object]) -> None: + self._core.service_logged(call_type, start_time, end_time, request) + + async def set_response_headers( + self, response: object, model_group: str | None, request_kwargs: Mapping[str, object] + ) -> object: + return await self._core.set_response_headers(response, model_group, request_kwargs) + + def configured_deployment(self, model_id: str) -> "Deployment | None": + return self._core._hosted._core_get_deployment(model_id) # pyright: ignore[reportPrivateUsage] # see the class docstring + + def registry(self) -> RouterRegistry: + return self._core.registry() + + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + return await self._core.select( + model, request_kwargs, kind=kind, specific_deployment=specific_deployment, messages=messages + ) + + async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._core.acompletion(*args, **kwargs) + + +class RustCore( + _SettingsSource, + RustLegacyMembers, + RustRegistryMembers, + RustRetries, + RustFallbacks, + RustHealthCallbacks, + RustOptionalChecks, +): + """A ``RouterCore`` served by the Rust router core over a ``PythonHost``. It holds no Python core.""" + + def __init__(self, config: RouterConfig, host: PythonHost, resolved: Resolved) -> None: + from litellm._service_logger import ServiceLogging + from litellm.router_utils.cooldown_cache import CooldownCache + + self._config: Final = config + self._host: Final = host + self.service_logger_obj: ServiceLogging = ServiceLogging() + apply_construction_effects(config, host) + self._resolved: Resolved = resolved + # `cooldown_time` as the Python core stores it (`cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS`, + # then each value assigned or updated, as given): what `settings()` reads back. + self._cooldown_given: object = config.get("cooldown_time") or DEFAULT_COOLDOWN_TIME_SECONDS + # Settings the Python core keeps as plain attributes, and settings deleted with `del`. + self._plain: Final = initial_plain(config) + self._deleted: frozenset[str] = frozenset() + self._request_timeout: Final = request_timeout(config) + legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes + self._legacy: Final = legacy + self._health_view: RustHealth | None = None + self._cache: DualCache = host.routing_cache(config) + self._cooldowns: CooldownCache = CooldownCache( + cache=self._cache, + default_cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the constructor's value, as the Python core passes it + ) + self._health_checks: DeploymentHealthCache = resolved.health_routing.health_checks(self._cache) + self._version: int = 1 + self._settings: RustSettings | None = None + self._tag_filtering_models: tuple[int, bool] | None = None + # The registry indexes written through the deprecated attributes (`_core_`), and the + # document router-core last received for them. + self._index_written: frozenset[str] = frozenset() + self._index_sent: Mapping[str, object] | None = None + self._selectors: Final = StrategySelectors(host, self._cache) + self._selectors.init_default(resolved.routing_strategy, resolved.routing_strategy_args) + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + self._port: Final = _EndpointPort(self) + self._endpoints: Final = RustEndpoints(self._port, config) + # The resolver every picked deployment with `kubernetes_pod_discovery` goes through. + self._pod_discovery: KubernetesPodDiscovery = KubernetesPodDiscovery() + general: Final = config.get("router_general_settings", _DEFAULT_GENERAL_SETTINGS) + self._general_settings: object = general or RouterGeneralSettings() + self._registry_view: RouterRegistry | None = None + self._hosted: Final = HostRegistry( + host, + model_group_alias=dict(resolved.model_group_alias), # pyright: ignore[reportArgumentType] # the configured alias map + ignore_invalid_deployments=bool(config.get("ignore_invalid_deployments")), + on_change=self._sync_registry, + ) + self._hosted._core_cache = self._cache # pyright: ignore[reportPrivateUsage] # usage reads come from the cache the Rust core writes + ctx: Final = RoutingContext(self) # pyright: ignore[reportArgumentType] # the strategies read registry(), settings(), health() and select() + strategies: Final = PreRoutingStrategies( + self._hosted, host, ctx, config.get("auto_router_capability_limit"), self._cache + ) + self._hosted.strategies = strategies + self._prerouter: Final = PreRouter(strategies, self._hosted, host, ctx, self._cache) + # `register_plugin`: a custom strategy that replaces every pick (the host answers the + # Rust core's pre_route step with its deployment). + self._custom_strategy: CustomRoutingStrategyBase | None = None + self._custom_discovery: KubernetesPodDiscovery | None = None + # router-core builds the deployments, asking the host registry about each one: the handle + # starts empty, and is rebuilt over the built list for the settings the list's names feed. + self._handle_default_ids: tuple[str, ...] = () # the provider default ids the handle was built with + self._allowed_fails_default_sent: bool = False # what the handle's dynamic document last said + self._walked_fallbacks: Mapping[str, object] = {} # the fallback lists that document held + self._handle: RouterCoreHandle = self._new_handle([]) + self._hosted.build_runner = self._run_build + self._hosted.missing_attribute = self._no_router_attribute + self._hosted.changes_runner = self._apply_registry_changes + self._hosted.handle_lookup = self._live_handle + self._hosted.build(_as_list(config.get("model_list")) if config.get("model_list") is not None else None) + self._hosted.set_routing_groups(resolved.routing_groups) + track_live_registry(self._hosted) + self._hosted.routable_lookup = self._routable + self._hosted.max_parallel_requests_lookup = self._max_parallel_requests_client + self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry + self._rebuild() + self._join_callback_holders() + self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments) + self._hosted.checks = self._checks + strategies.ensure_affinity = self._checks.ensure_deployment_affinity_callback + self._checks.build(config.get("optional_pre_call_checks"), config.get("model_list")) + self._slack_alerting_logger: SlackAlerting | None = initialize_alerting(host, config.get("alerting_config")) # pyright: ignore[reportArgumentType] # the argument as passed + if self._slack_alerting_logger is not None: + self._checks.slack_alerting_loggers = (self._slack_alerting_logger,) + if resolved.enable_mid_stream_fallback_continuation: + register_continuation_prefill_check(host) + if resolved.routing_strategy == "lar1": + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy(ctx, resolved.routing_strategy_args) # pyright: ignore[reportArgumentType] # the core's RoutingContext + + @property + def _registry_state(self) -> HostRegistry: + return self._hosted + + @property + def _python_host(self) -> PythonHost: + return self._host + + @property + def _written_retry_policy(self) -> object: + return self._plain["retry_policy"] + + @classmethod + def build(cls, config: RouterConfig, host: PythonHost) -> "RustCore": + from litellm.rust_bridge._native import RustBridgeDeclined + + extra: Final = unsupported_arguments(config) + if extra: + raise RustBridgeDeclined(f"router arguments not served by the Rust core: {sorted(extra)}") + RustEndpoints.prepare_host(config, host) + return cls(config, host, Resolved.of(config, host.dynamic())) + + @property + def enable_mid_stream_fallback_continuation(self) -> bool: + return self._resolved.enable_mid_stream_fallback_continuation + + def _set_continuation(self, enabled: bool) -> None: + """``_set_mid_stream_fallback_continuation``: an enabled flag registers the process-wide prefill check.""" + self._resolved = dataclasses.replace(self._resolved, enable_mid_stream_fallback_continuation=enabled) + self._settings = None + if enabled: + register_continuation_prefill_check(self._host) + + def __getattr__(self, name: str) -> object: + if name in self.__dict__.get("_deleted", ()): + raise self._no_attribute(name) + if name in SELECTOR_ATTRIBUTES.values(): + return self._selectors.attribute(name) + if name == "_core_async_get_available_deployment" and self.__dict__.get("_custom_strategy") is not None: + return self._custom_async_pick() + # Only reached for members this class does not define: the endpoints served through the + # host, else refused. Never forwarded anywhere. + endpoints: Final[object] = self.__dict__.get("_endpoints") + endpoint: Final = endpoints.member(name) if isinstance(endpoints, RustEndpoints) else None + if endpoint is not None: + return endpoint + attribute: Final = self._settings_attribute(name) if "_resolved" in self.__dict__ else MISSING + if attribute is not MISSING: + return attribute + raise RustCoreUnsupported(f"'{name}' is not served by the Rust router core", name=name, obj=self) + + @property + def _unset_attributes(self) -> frozenset[str]: + """The instance attributes ``del`` removed, which ``_settings_attribute`` no longer serves.""" + unset: Final[object] = self.__dict__.get("_unset_attributes") + return unset if isinstance(unset, frozenset) else frozenset() # pyright: ignore[reportUnknownVariableType] # only names are stored + + def _settings_attribute(self, name: str) -> object: + """A router attribute that reads a setting, the constructor's arguments or a constant.""" + if name in self._unset_attributes: + return MISSING + r: Final = self._resolved + resolved: Final[Mapping[str, object]] = { + "num_retries": r.read("num_retries"), + "max_fallbacks": r.read("max_fallbacks"), + "timeout": r.read("timeout"), + "retry_after": r.read("retry_after"), + "explicit_allowed_fails": r.given["allowed_fails"] + if r.explicit_allowed_fails is not None and "allowed_fails" in r.given + else r.explicit_allowed_fails, + "routing_strategy_args": r.read("routing_strategy_args"), + "request_timeout": self._request_timeout, + "default_priority": self._endpoints._default_priority, # pyright: ignore[reportPrivateUsage] # parsed by the endpoint plans + } + if name in resolved: + return resolved[name] + if name in self._plain: + return self._plain[name] + if name in self._legacy: + return self._legacy[name] + if name == "healthy_deployments": + return self._hosted._core_model_list # pyright: ignore[reportPrivateUsage] # the live list, as the core keeps it + if name == "get_model_group_io_token_usage": + return self._hosted.get_model_group_io_token_usage + if name == "scheduler": + return self._endpoints._get_scheduler() # pyright: ignore[reportPrivateUsage] # the endpoint plans' scheduler + if name == "chat": + from litellm.main import Chat + + self._legacy["chat"] = Chat(params=default_litellm_params(self._config), router_obj=self) # pyright: ignore[reportArgumentType] # Chat calls the router's endpoints, which this core serves + return self._legacy["chat"] + return argument_attribute(self._config, name) + + @property + def _shaping(self) -> Shaping: + return self._endpoints.shaping(self._resolved) + + # -- building -------------------------------------------------------------------------------- + + def _new_handle(self, model_list: Sequence[object]) -> "RouterCoreHandle": + from litellm.rust_bridge._native import RouterCoreHandle + + names: Final = [ + str(name) + for entry in model_list + if isinstance(name := (as_str_mapping(entry) or {}).get("model_name"), str) + ] + default_ids: Final = self._provider_default_ids() + self._handle_default_ids = default_ids + static: Final = { + **self._resolved.static_document(model_list), + **{name: value for name, value in failover_document(self._plain).items() if value is not None}, + **({"provider_default_deployment_ids": list(default_ids)} if default_ids else {}), + } + return RouterCoreHandle(static, self._dynamic_document(names)) + + def _dynamic_document(self, model_names: Sequence[str]) -> Mapping[str, object]: + self._allowed_fails_default_sent = self._resolved.allowed_fails_is_default() + # Settings are static: an in-place edit of a live list reaches neither the walk nor its notes. + resolved: Final = self._resolved + self._walked_fallbacks = { + "fallbacks": detached(resolved.fallbacks), + "context_window_fallbacks": detached(resolved.context_window_fallbacks), + "content_policy_fallbacks": detached(resolved.content_policy_fallbacks), + } + return resolved.dynamic_document(self._version, model_names) + + def _provider_default_ids(self) -> tuple[str, ...]: + """``provider_default_deployment_ids`` as the cooldown check reads it (``deployment in + settings().provider_default_deployment_ids``): the ids it holds now. router-core holds them + with its static settings.""" + handle: Final = self._live_handle() + return () if handle is None else tuple(handle.provider_default_ids()) + + def _pick_discovery(self, *, sync: bool) -> KubernetesPodDiscovery | None: + """The pod discovery a pick resolves through: the one bound when a custom strategy was + registered (``resolve_pods_after_bound``), else the router's own (``resolve_pods_after``); + none for an async pick assigned through the deprecated attribute, which Python calls bare.""" + if not sync and callable(self.__dict__.get("_core_async_get_available_deployment")): + return None + if self._custom_strategy is not None and self._custom_discovery is not None: + return self._custom_discovery + return self._pod_discovery + + # -- the plugin context (``RoutingContext``) ----------------------------------------------- + + def _core_arm_routing_read_prefetch(self, model: str, request_kwargs: Mapping[str, object] | None = None) -> None: + """``RoutingContext.arm_read_prefetch``: declare the cooldown reads (and, for + usage-based-routing-v2, the usage reads) the pick for ``model`` makes on the request's + Redis batch, so admission's flush carries them. A miss costs nothing.""" + from litellm.router_utils.routing_read_batch import RoutingPrefetch + + try: + usage_selector: Final = self._prefetch_usage_selector(model, request_kwargs) + deployments: Final = self._hosted._core_get_model_list(model_name=model) # pyright: ignore[reportPrivateUsage] # see __init__ + if deployments: + RoutingPrefetch.arm(RoutingContext(self), usage_selector, deployments) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the prefetch reads registry() and health(); its deployments list is untyped + except Exception as e: # noqa: BLE001 # a prefetch is an optimisation, never a reason to fail the request + from litellm._logging import verbose_router_logger + + verbose_router_logger.debug( + "routing read prefetch not armed for %s: %s", + str(model).replace("\r", "").replace("\n", ""), + str(e).replace("\r", "").replace("\n", ""), + ) + + def _prefetch_usage_selector(self, model: str, request_kwargs: Mapping[str, object] | None) -> object: + """``_get_routing_context``'s selector when it is a usage-based-routing-v2 one: the request's + strategy override, else the routing group ``model`` resolves into, else the router's own.""" + from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 + + strategy, selector = self._prefetch_strategy_and_selector(model, request_kwargs or {}) + if strategy == "usage-based-routing-v2" and isinstance(selector, LowestTPMLoggingHandler_v2): + return selector + return None + + def _prefetch_strategy_and_selector(self, model: str, request: Mapping[str, object]) -> tuple[str | None, object]: + raw: Final = request.get("routing_strategy") + override: Final = normalize_strategy(raw) if isinstance(raw, str | RoutingStrategy) else None + if override is not None and override in {"simple-shuffle", *SELECTOR_ATTRIBUTES}: + return override, self._selectors.selector_for(override, "override", None, request) + groups: Final = self._hosted._core_routing_groups # pyright: ignore[reportPrivateUsage] # see __init__ + resolved: Final = self._hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # see __init__ + group_name: Final = ( + resolved + if self._hosted.get_routing_group(resolved) is not None + else next( + ( + group.group_name + for group in groups.values() + if group.routing_strategy != "priority" and resolved in group.models + ), + None, + ) + ) + if group_name is None: + return normalize_strategy(self._resolved.routing_strategy), self._selectors.default + if groups[group_name].routing_strategy == "priority": + return None, None + group_strategy: Final = normalize_strategy(groups[group_name].routing_strategy) + return group_strategy, self._selectors.selector_for(group_strategy or "", "group", group_name, None) + + async def _core_async_pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the request kwargs the hook stamps + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + ) -> "PreRoutingHookResponse | None": + """``RoutingContext.pre_routing_hook``: the pre-routing strategies' answer for ``model`` + (the tier group and its params), with the routing plugins run first.""" + r: Final = self._resolved + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return await self._prerouter._hook( # pyright: ignore[reportPrivateUsage] # the core's own pre-router + model, + request_kwargs, + messages=messages, + input=input, + fallback_depth=0, + enable_tag_filtering=r.enable_tag_filtering, + match_any=r.tag_filtering_match_any, + public_router=public if public is not None else self, + run_plugins=partial(self._run_plugins, request_kwargs, messages) + if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list + else None, + ) + + def _core_log_retry( + self, + kwargs: dict[str, object], # mutable-ok: the request kwargs the record joins + e: Exception, + ) -> dict[str, object]: # mutable-ok: the same kwargs, as Python's returns them + """``RoutingContext.log_retry``.""" + return log_retry_into(kwargs, e, self._deployment_params_base_model) + + def _deployment_params_base_model(self, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` a deployment declares under ``litellm_params``.""" + deployment: Final = self._hosted._core_get_deployment(model_id=deployment_id) # pyright: ignore[reportPrivateUsage] # see __init__ + if deployment is None: + return None + base_model: Final = (deployment.litellm_params.model_extra or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + def _sync_routing_state(self) -> None: + """Before a pick: sync the routing state that callers can change in place, which the Python + core reads live on every pick. This exists only for the deprecated writable attributes and + goes away with them. + + Only these cases send anything; otherwise both checks return without touching router-core:: + + router.model_names = {...} # assigning syncs at once (__setattr__) + router.model_names.add("new-group") # an in-place change is picked up here, on the next pick + router.model_name_to_deployment_indices["g"].append(3) # likewise + + router.routing_plugins.append(plugin) # first plugin: rebuilds the handle with the plugin stage + router.routing_plugins.clear() # list emptied: rebuilds without it + + Until one of the index attributes has been assigned, ``_sync_index`` sends nothing. After + that it sends the index document only when it differs from the last one it sent. + """ + self._sync_plugin_stage() + self._sync_index() + self._sync_allowed_fails() + + def _sync_allowed_fails(self) -> None: + """``_is_allowed_fails_set_on_router`` compares with the live ``litellm.allowed_fails`` on + every failure, so a write to the global can change the rule; ``_settings`` stays as copied.""" + if self._resolved.allowed_fails_is_default() != self._allowed_fails_default_sent: + names: Final = [str(d.get("model_name")) for d in self._handle.deployments()] + self._handle.apply({"settings": self._dynamic_document(names)}) + + def _sync_index(self) -> None: + """Hand router-core the registry indexes written through the deprecated attributes + (``model_names``, ``model_name_to_deployment_indices``, ``team_model_to_deployment_indices``) + as the host registry holds them now (its own deployment changes update the written objects, + as the Python core's do), whenever they differ from what it holds.""" + if not self._index_written: + return + document: Final = index_document(self._hosted, self._index_written) + if document != self._index_sent: + _ = self._handle.apply({"index": document}) + self._index_sent = document + + def _sync_plugin_stage(self) -> None: + """Give router-core the routing-plugin stage when the live plugin list (``routing_plugins``, + which callers may assign or append to) gains its first plugin, and drop it when the list + empties: the Python core checks the list on every pick.""" + live: Final = _as_list(self._hosted._core_routing_plugins) # pyright: ignore[reportPrivateUsage] # the live plugin list + if bool(live) == bool(self._resolved.routing_plugins): + return + self._resolved = dataclasses.replace(self._resolved, routing_plugins=tuple(live)) # pyright: ignore[reportArgumentType] # the plugins as listed + self._rebuild() + + def _rebuild(self) -> None: + """A new Rust core for changed static settings, over the same deployments and the + providers the old one's builds resolved for them.""" + old: Final = self._handle + model_list: Final = list(self._hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # see __init__ + self._handle = self._new_handle(model_list) + self._handle.adopt_providers(old) + old.retire() + # The live list and indexes read the new handle's (the same entries, kept as the same objects). + self._hosted.refresh() + # The new handle builds its indexes from the deployments: written ones are handed again. + self._index_sent = None + + def _run_build( + self, + entries: Sequence[object], + path: str, + resolve_secrets: bool, + install: BuildInstall, + answer: Callable[[Mapping[str, object]], object], + ) -> None: + """Drive one router-core deployment build (``RouterCoreHandle.build``), answering each of + its steps with ``answer``; router-core installs each deployment as it accepts it, and the + host registry reads the list and indexes back before answering the next step.""" + call: Final = self._handle.build(_plain(list(entries)), path, resolve_secrets, install) # pyright: ignore[reportArgumentType] # plain JSON entries + outcome: object = None # rebind-ok: each step's answer + while True: + step = call.next(outcome) + self._hosted.refresh() + if step["kind"] == "done": + break + if step["kind"] == "failed": + raise ValueError(str(step.get("error"))) + outcome = _plain(answer(step)) + + def _live_handle(self) -> "RouterCoreHandle | None": + """The handle the live model_list reads, once the constructor made one.""" + from litellm.rust_bridge._native import RouterCoreHandle + + handle: Final[object] = self.__dict__.get("_handle") + return handle if isinstance(handle, RouterCoreHandle) else None + + def _apply_registry_changes(self, changes: Mapping[str, object]) -> None: + """Apply a host registry write router-core's build did not make (a removal is a delete, a + swapped list a replace) as a changeset.""" + _ = self._handle.apply(_plain(dict(changes))) # pyright: ignore[reportArgumentType] # a plain JSON changeset + + def _sync_registry(self) -> None: + """After a host registry write: router-core already holds it (the build installed what it + added, a removal came as a changeset), so only the settings it feeds move.""" + from litellm.rust_bridge._native import RouterCoreHandle + + self._tag_filtering_models = None + handle: Final[object] = self.__dict__.get("_handle") + # A wildcard deployment added an id the cooldown check skips. + if isinstance(handle, RouterCoreHandle) and self._provider_default_ids() != self._handle_default_ids: + self._rebuild() + + # -- entry points ---------------------------------------------------------------------------- + + def registry(self) -> RouterRegistry: + """The registry view, built once (``registry()`` returns the same object each time).""" + if self._registry_view is None: + self._registry_view = hosted_registry_view(self._hosted) + return self._registry_view + + def settings(self) -> RustSettings: + """A cached snapshot, replaced when a changeset changes a setting.""" + if self._settings is None: + self._settings = self._settings_snapshot() + return self._settings + + def _settings_snapshot(self) -> RustSettings: + r: Final = self._resolved + snapshot: Final = RustSettings( + routing_strategy=r.routing_strategy, + fallbacks=r.read("fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + context_window_fallbacks=r.read("context_window_fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + content_policy_fallbacks=r.read("content_policy_fallbacks"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + default_deployment=r.default_deployment, + default_litellm_params=dict(r.default_litellm_params) + if "default_litellm_params" in r.assigned + else default_litellm_params(self._config), + default_max_parallel_requests=self._endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given + cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + allowed_fails=r.read("allowed_fails"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + allowed_fails_policy=r.allowed_fails_policy, # pyright: ignore[reportArgumentType] # the policy as passed + disable_cooldowns=r.read("disable_cooldowns"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + enable_pre_call_checks=r.enable_pre_call_checks, + enable_health_check_routing=r.health_routing.enable_health_check_routing, + enable_tag_filtering=r.read("enable_tag_filtering"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + tag_filtering_match_any=r.read("tag_filtering_match_any"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + tag_routing_prefix=r.read("tag_routing_prefix"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + fallback_access_check=r.fallback_access_check, # pyright: ignore[reportArgumentType] # the check as passed + fallback_budget_check=r.fallback_budget_check, # pyright: ignore[reportArgumentType] # the check as passed + cache_responses=self._endpoints.cache_responses, + search_tools=self._endpoints.search_tools, # pyright: ignore[reportArgumentType] # the configured entries, as passed + guardrail_list=self._endpoints.guardrail_list, # pyright: ignore[reportArgumentType] # same + provider_default_deployment_ids=self._hosted._core_provider_default_deployment_ids, # pyright: ignore[reportPrivateUsage] # the host registry's live list + health_check_ignore_transient_errors=r.health_routing.health_check_ignore_transient_errors, + background_health_check_model_groups=r.health_routing.background_health_check_model_groups, + router_general_settings=self._general_settings, # pyright: ignore[reportArgumentType] # the setting as given + provider_budget_config=self._checks.provider_budget_config, # pyright: ignore[reportArgumentType] # the budget limiter's live config + _router=self, # pyright: ignore[reportArgumentType] # the snapshot reads as_dict from this core + kubernetes_pod_discovery=self._pod_discovery, + ) + if not self._deleted: + return snapshot + # A deleted setting reads as unset. + return dataclasses.replace( + snapshot, + **dict.fromkeys(self._deleted.intersection(SETTING_FIELDS), None), # pyright: ignore[reportArgumentType] # settings fields only: an alias map or retry count is deletable too + _unset=snapshot._unset | self._deleted, # pyright: ignore[reportPrivateUsage] # the snapshot's own unset set + ) + + def available_guardrail(self, name: str) -> object: + return self._endpoints.available_guardrail(name) + + def settings_dict(self) -> Mapping[str, object]: + r: Final = self._resolved + latency: Final = self._selectors.latency_args() if r.routing_strategy == "latency-based-routing" else None + # `get_settings` reports the latency selector's full routing args. + args: Final = ( + latency if latency is not None else r.given.get("routing_strategy_args", dict(r.routing_strategy_args)) + ) + values: Final = { + "routing_strategy_args": args, + "routing_strategy": r.routing_strategy, + "allowed_fails": r.read("allowed_fails"), + "cooldown_time": self._cooldown_given, + "num_retries": r.read("num_retries"), + "timeout": r.read("timeout"), + "retry_after": r.read("retry_after"), + "fallbacks": r.read("fallbacks"), + "context_window_fallbacks": r.read("context_window_fallbacks"), + "model_group_alias": dict(r.model_group_alias), + "enable_mid_stream_fallback_continuation": r.enable_mid_stream_fallback_continuation, + "enable_tag_filtering": r.read("enable_tag_filtering"), + "tag_routing_prefix": r.read("tag_routing_prefix"), + "routing_groups": group_settings(r.routing_groups), + } + return settings_view(values, self._plain, self._deleted) + + def health(self) -> RustHealth: + """The health view, built once.""" + if self._health_view is None: + self._health_view = RustHealth(self) + return self._health_view + + def observe(self, logging_obj: "LiteLLMLogging | None" = None, *, request_scoped: bool = False) -> RoutingObservers: + """The routing observers: the Slack alerting logger ``alerting_config`` built, if any. + ``request_scoped=True`` takes the router's callbacks off the host lists; passing a request's + ``logging_obj`` binds its selectors and policies to it. This core registers no outcome + callbacks, so it binds none either: router-core reads what they would do for the request + (``OptionalChecks.outcome_callbacks``).""" + if request_scoped: + self._serve_through_request_logging() + if logging_obj is not None: + self._checks.bind_to_request_logging(logging_obj) + return RoutingObservers(slack_alerting_logger=self._slack_alerting_logger) + + def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: + """``_core_set_custom_routing_strategy``: ``plugin`` picks every deployment from now on, until + the routing strategy changes.""" + self._custom_strategy = plugin + # `resolve_pods_after_bound`: its picks resolve pods through the discovery bound now. + self._custom_discovery = self._pod_discovery + # Python assigns the strategy's picks over its own: a caller of the pick reaches the + # strategy with the arguments it passed and no others. + self.__dict__["get_available_deployment"] = self._custom_sync_pick(plugin) + # It replaces an async pick assigned through the deprecated attribute, as Python's does. + if "_core_async_get_available_deployment" in self.__dict__: + del self.__dict__["_core_async_get_available_deployment"] + + def _custom_sync_pick(self, plugin: "CustomRoutingStrategyBase") -> object: + discovery: Final = self._custom_discovery + pick: Final = plugin.get_available_deployment + return pick if discovery is None else resolve_pods_after_bound(pick, discovery) + + def _custom_async_pick(self) -> object: + """``_core_set_custom_routing_strategy``'s ``_core_async_get_available_deployment``.""" + custom: Final = self._custom_strategy + discovery: Final = self._custom_discovery + pick: Final = custom.async_get_available_deployment # pyright: ignore[reportOptionalMemberAccess] # __getattr__ checked it + return pick if discovery is None else async_resolve_pods_after_bound(pick, discovery) + + def set_custom_routing_strategy(self, CustomRoutingStrategy: "CustomRoutingStrategyBase") -> None: + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the deprecated alias notes its use like Python's + ) + + _note("set_custom_routing_strategy", "register_plugin(strategy)") + self.register_plugin(CustomRoutingStrategy) + + def retire(self) -> None: + """``_core_discard``: only the router's global registrations go; it keeps serving calls.""" + untrack_live_registry(self._hosted) + self._retire_callbacks() + if self._hosted.strategies is not None: + self._hosted.strategies.retire() + + def apply(self, changes: ChangeSet) -> ApplyResult: + """``apply_changes``: assigned settings, then ``update_settings``, upserts, deletes and the + Redis cache, in that order. An unknown name refuses the changeset; a setting + ``update_settings`` raises for ends it there, with the settings before it applied.""" + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + if changes.refresh_model_info: + raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()") + unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES}) + if unknown: + raise ValueError(f"not assignable router settings: {unknown}") + unserved: Final = sorted(set(changes.assign) - _STATIC_SETTINGS - _DYNAMIC_SETTINGS - _HOST_SETTINGS) + if unserved: + raise RustCoreUnsupported(f"the Rust router core does not serve the setting(s) {', '.join(unserved)} yet") + update: Final = read_settings_update(changes.settings, self._resolved.routing_strategy) + # `apply_changes` assigns before it runs `update_settings`: a budget config assigned beside + # `optional_pre_call_checks` builds the budget limiter with the new config. + self._assign_host_settings(changes.assign) + if update.optional_pre_call_checks is not MISSING: + self._checks.set(update.optional_pre_call_checks) + if update.continuation is not MISSING: + self._set_continuation(continuation_flag(update.continuation)) + self._apply_settings(changes.assign, update) + if update.failure is not None: + raise update.failure + # Upserts first, then deletes (`apply_changes`), through the host registry, which hands + # each change to the Rust core. + upserted: Final = [self._hosted._core_upsert_deployment(deployment=d) for d in changes.upsert] # pyright: ignore[reportPrivateUsage] # see __init__ + deleted: Final = [self._hosted._core_delete_deployment(id=i) for i in changes.delete] # pyright: ignore[reportPrivateUsage] # see __init__ + if changes.redis_cache is not None: + # `_core_update_redis_cache`: the routing state's cache gains the Redis cache. + self._cache.attach_redis_cache(changes.redis_cache) + self._checks.prompt_caching_pin_cache.attach_redis_cache(changes.redis_cache) + return ApplyResult( + version=self._hosted._registry_version, # pyright: ignore[reportPrivateUsage] # see __init__ + upserted=upserted, + deleted=deleted, # pyright: ignore[reportArgumentType] # the removed model_list entries, as the core returns them + ) + + async def aapply(self, changes: ChangeSet) -> ApplyResult: + """``apply``, then the model info refresh, which needs I/O.""" + result: Final = self.apply(dataclasses.replace(changes, refresh_model_info=False, refresh_client=None)) + if changes.refresh_model_info: + await self._hosted.arefresh_model_info(changes.refresh_client) + return result + + def _apply_settings(self, assign: Mapping[str, object], update: SettingsUpdate) -> None: + values: Final = { + **without(assign, *_HOST_SETTINGS), + **update.values, + } + given: Final = dict(update.values) # the settings passed through `update_settings` + groups_input: Final[object] = ( + update.routing_groups if update.routing_groups is not MISSING else self._resolved.routing_groups_input + ) + # What `_core_update_settings` runs after its loop, which a setting that raised never reaches. + settled: Final = update.failure is None + groups_failure: Exception | None = None # rebind-ok: set when the groups passed do not parse + if settled and (update.routing_groups is not MISSING or self._changes_strategy(update.values)): + # `update_settings` re-inits the routing groups from the groups passed, else (on a + # strategy change) from the groups last passed, whatever a raw write assigned since. + try: + values["routing_groups"] = parse_groups( + groups_input, + frozenset(str(d.get("model_name")) for d in self._handle.deployments()), + frozenset(self._resolved.model_group_alias), + ) + except Exception as raised: # noqa: BLE001 # raised below: `_init_routing_groups` runs after the loop + groups_failure = raised # rebind-ok: None until the groups fail + else: + given["routing_groups"] = groups_input + self._store_plain(update.plain) + if self._deleted & set(values): + self._deleted = self._deleted - set(values) + self._settings = None + if values: + previous: Final = self._resolved + resolved: Final = dataclasses.replace( + previous.changed(values, frozenset(given)), + routing_groups_input=groups_input if "routing_groups" in values else previous.routing_groups_input, + ) + static_changed: Final = bool(_STATIC_SETTINGS & set(values)) + self._resolved = resolved + if set(values) & set(SETTING_FIELDS): + self._settings = None # the snapshot holds settings fields, not the plain attributes + if "model_group_alias" in values: + self._hosted._core_model_group_alias = dict(resolved.model_group_alias) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # see __init__ + self._apply_strategy(previous, resolved, given, settled=settled) + if "routing_groups" in values: + self._settings = None + self._hosted.set_routing_groups(resolved.routing_groups) + if "cooldown_time" in values: + self._cooldown_given = values["cooldown_time"] + if values["cooldown_time"] is None: + # `add_deployment_to_cooldown(cooldown_time=None)` cools down for the cooldown + # cache's default, which keeps the constructor's value. + self._resolved = dataclasses.replace( + resolved, cooldown_time=float(self._cooldowns.default_cooldown_time) + ) + if static_changed: + self._rebuild() + else: + names: Final = [str(d.get("model_name")) for d in self._handle.deployments()] + self._handle.apply({"settings": self._dynamic_document(names)}) + if groups_failure is not None: + raise groups_failure + + def _store_plain(self, plain: Mapping[str, object]) -> None: + """Settings the Python core keeps as plain attributes, read from the next call on.""" + flags_before: Final = failover_document(self._plain) + self._plain.update(plain) + if failover_document(self._plain) != flags_before: + self._rebuild() + policies: Final = {name: plain[name] for name in _RETRY_POLICY_SETTINGS if name in plain} + if policies: + self._set_retry_policies(policies) + + def _changes_strategy(self, updated: Mapping[str, object]) -> bool: + """``update_settings`` names a ``routing_strategy`` other than the router's.""" + if "routing_strategy" not in updated: + return False + return _strategy_name(updated["routing_strategy"]) != self._resolved.routing_strategy + + def _assign_host_settings(self, assign: Mapping[str, object]) -> None: + """The assigned settings only the host reads, each stored as given (detached, as + ``apply_changes`` stores it) where its readers find it from the next call on.""" + given: Final = {name: detached(value) for name, value in assign.items() if name in _HOST_SETTINGS} + if not given: + return + for name, value in given.items(): + if name in ("enable_health_check_routing", "background_health_check_model_groups"): + self._resolved = dataclasses.replace( + self._resolved, + health_routing=dataclasses.replace(self._resolved.health_routing, **{name: value}), # pyright: ignore[reportArgumentType] # stored as given, as the Python core stores it + ) + elif name == "provider_default_deployment_ids": + # The list wildcard deployments added later are appended to, as on the Python core. + self._hosted._core_provider_default_deployment_ids = value # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name == "cache_responses": + self._endpoints.set_cache_responses(bool(value)) + elif name == "provider_budget_config": + # Read when the budget limiter is built; a limiter built already keeps its config. + self._checks.provider_budget_config = value + elif name == "health_check_ignore_transient_errors": + self._resolved = dataclasses.replace( + self._resolved, + health_routing=dataclasses.replace( + self._resolved.health_routing, + health_check_ignore_transient_errors=value, # pyright: ignore[reportArgumentType] # stored as given, as the Python core stores it + ), + ) + elif name == "router_general_settings": + self._general_settings = value + elif name == "kubernetes_pod_discovery": + self._pod_discovery = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + else: + self._endpoints.assign_setting(name, value) + self._deleted = self._deleted - set(given) + self._settings = None + if {"enable_health_check_routing", "background_health_check_model_groups"} & set(given) or ( + self._provider_default_ids() != self._handle_default_ids + ): + self._rebuild() + + def _set_retry_policies(self, policies: Mapping[str, object]) -> None: + """The ``retry_policy`` and ``model_group_retry_policy`` the retry loop reads from then on, + from ``update_settings`` or an attribute write: router-core holds them with its static + settings.""" + previous: Final = self._resolved + resolved: Final = dataclasses.replace( + previous, + retry_policy=readable_retry_policy(policies.get("retry_policy", previous.retry_policy)), + model_group_retry_policy=policies.get("model_group_retry_policy", previous.model_group_retry_policy), + ) + if resolved != previous: + self._resolved = resolved + self._rebuild() + + def _apply_strategy( + self, previous: Resolved, resolved: Resolved, values: Mapping[str, object], *, settled: bool + ) -> None: + """``_core_update_settings``: a changed strategy re-inits the selectors with the args passed + alongside it (or none); changed args alone rebuild the default selector. What runs after + the loop needs ``settled``: no setting raised, and the groups are in ``values``.""" + if "routing_strategy" in values and resolved.routing_strategy != previous.routing_strategy: + if resolved.routing_strategy == "lar1": + # `_switch_routing_strategy`: LAR-1 is a custom strategy, registered in place of + # `routing_strategy_init`, so the selectors the router held stay as they were. + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy( + RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + as_str_mapping(values.get("routing_strategy_args")), + ) + else: + # `routing_strategy_init` drops a custom strategy (`_reset_custom_routing_strategy`), + # and an async pick assigned through the deprecated attribute. + self._custom_strategy = None + self.__dict__.pop("get_available_deployment", None) + if "_core_async_get_available_deployment" in self.__dict__: + del self.__dict__["_core_async_get_available_deployment"] + self._selectors.init_default( + resolved.routing_strategy, as_str_mapping(values.get("routing_strategy_args")) or {} + ) + if "routing_groups" in values: + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + elif "routing_groups" in values: + self._selectors.init_groups(_group_selectors(resolved.routing_groups)) + elif ( + settled + and "routing_strategy_args" in values + and resolved.routing_strategy_args != previous.routing_strategy_args + ): + if resolved.routing_strategy == "lar1": + from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy + + apply_lar1_routing_strategy(RoutingContext(self), resolved.routing_strategy_args) # pyright: ignore[reportArgumentType] # the core's RoutingContext + return + self._selectors.rebuild_default(resolved.routing_strategy_args) + + def upsert_deployment(self, deployment: Deployment) -> Deployment | None: + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("upsert_deployment", "apply(ChangeSet(upsert=[deployment]))") + return self.apply(ChangeSet(upsert=[deployment])).upserted[0] + + def delete_deployment(self, id: str) -> Deployment | None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see above + + _note("delete_deployment", "apply(ChangeSet(delete=[id]))") + return self.apply(ChangeSet(delete=[id])).deleted[0] + + def update_settings(self, **kwargs: object) -> None: # kwargs-ok: the deprecated signature, unchanged + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("update_settings", "apply(ChangeSet(settings={...}))") + _ = self.apply(ChangeSet(settings=kwargs)) + + def update_redis_cache(self, cache: "RedisCache") -> None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))") + _ = self.apply(ChangeSet(redis_cache=cache)) + + def get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("get_model_list", "registry().deployments(model_name, team_id=...)") + return self._hosted._core_get_model_list(model_name=model_name, team_id=team_id) # pyright: ignore[reportPrivateUsage, reportReturnType] # the host registry's listing, as the deprecated member returns it + + @staticmethod + def get_valid_args() -> list[str]: # mutable-ok: the old return type, a fresh list + """The arguments ``Router.__init__`` accepts.""" + return list(ROUTER_INIT_ARGS) + + async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None: + from litellm.router_deprecated import _note # pyright: ignore[reportPrivateUsage] # see upsert_deployment + + _note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))") + _ = await self.aapply(ChangeSet(refresh_model_info=True, refresh_client=client)) + + def flush_cache(self) -> None: + """Drop the host's response cache and the routing state.""" + self._host.set_global_cache(None) + self._cache.flush_cache() + self._checks.prompt_caching_pin_cache.in_memory_cache.flush_cache() + + def reset(self) -> None: + """Empty the host's success and failure callback lists (every entry, not only this core's), + drop the retry policy and flush the caches.""" + for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"): + self._host.set_callback_list(name, []) + self._plain["retry_policy"] = None + self._set_retry_policies({"retry_policy": None}) + self.flush_cache() + + def __delattr__(self, name: str) -> None: + # `del router.` unsets it: the settings view reports it unset from then on. + from litellm.router_api import SETTING_FIELDS + + # The deprecated attribute deletes the core's own name (``_core_``). + setting: Final = name.removeprefix("_core_") + if setting not in SETTING_FIELDS and setting not in _DELETABLE_ATTRIBUTES: + if name in self.__dict__ or name in self._unset_attributes: + object.__delattr__(self, name) + return + if self._settings_attribute(name) is MISSING: + object.__delattr__(self, name) + return + # An attribute the Python core keeps on the instance (`alerting_config`, an argument + # as passed): reading it raises AttributeError from now on. + self.__dict__["_unset_attributes"] = self._unset_attributes | {name} + return + if setting in self._deleted: + raise self._no_attribute(setting) + self._deleted = self._deleted | {setting} + self._settings = None + if setting == "model_group_alias": + del self._hosted._core_model_group_alias # pyright: ignore[reportPrivateUsage] # every read of it raises from now on, as on the Python core + elif setting == "scheduler": + self._endpoints.delete_scheduler() + + def _no_attribute(self, name: str) -> AttributeError: + """What reading an attribute deleted from the core raises (the facade renames it).""" + return AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'", name=name, obj=self) + + def _no_router_attribute(self, name: str) -> AttributeError: + """``_no_attribute`` for a read through ``registry()``, which the deprecated attribute + renames when it is the router's own.""" + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'", + name=name, + obj=self if public is None else public, + ) + + def _refuse_deleted_reads(self, request: Mapping[str, object]) -> None: + """What a call reads of the router before any attempt, once deleted: routing reads + ``_core_model_group_alias``, and ``async_function_with_retries`` reads ``self.num_retries`` + when the request names none. Every hop raises it, so no fallback serves the call.""" + if "model_group_alias" in self._deleted: + raise self._no_attribute("_core_model_group_alias") + if "num_retries" in self._deleted and request.get("num_retries") is None: + raise self._no_attribute("num_retries") + + def __setattr__(self, name: str, value: object) -> None: + # A setting write is a changeset (`router.num_retries = 2` is `apply(ChangeSet(assign=...))`), + # a `model_list` write swaps the live list, and any other attribute is kept as written. + from litellm.router_api import ASSIGNABLE_ALIASES, PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS + + if name in PLAIN_SETTING_ATTRIBUTES or name in SETTING_FIELDS or name in ASSIGNABLE_ALIASES: + _ = self.apply(ChangeSet(assign={name: value})) + elif name in PLAIN_DEFAULTS or name == "max_retries": + # Plain attributes on the Python core, outside the setting lists: stored as given. + self._store_plain({name: value}) + elif name == "scheduler": + self._endpoints.set_scheduler(value) + self._deleted = self._deleted - {name} + elif name in ("model_list", "_core_model_list"): + # `router.model_list = [...]` reaches the core as `_core_model_list` through the shim. + self._assign_model_list(_as_list(value)) + elif name in _HOSTED_STATE: + # A deprecated attribute write of the registry state: it lands where this core keeps it. + setattr(self._hosted, name, value) + if name in INDEX_ATTRIBUTES: + self._index_written = self._index_written | {name} + self._sync_index() + elif name == "_core_routing_groups": + # The Python core routes on the assigned groups from the next call on, while its + # member map and group selectors stay as they were; router-core holds the groups and + # the member map with its static settings. + assigned: Final = as_str_mapping(value) or {} + previous: Final = self._resolved + self._resolved = dataclasses.replace( + previous, + routing_groups=tuple(g for g in assigned.values() if isinstance(g, RoutingGroup)), + routing_group_members=previous.routing_group_members + if previous.routing_group_members is not None + else member_map(previous.routing_groups), + ) + self._settings = None + self._rebuild() + elif name == "_core_routing_plugins": + self._sync_plugin_stage() + elif name == "_core_prompt_caching_pin_cache": + # Read when a prompt caching check is built, by `update_redis_cache` and by `flush_cache`. + self._checks.prompt_caching_pin_cache = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name in _REBOUND_STATE: + # A deprecated attribute write of the routing state (``router.cache = ...``): this core + # reads the new object from now on, as the Python core reads its rebound attribute. + object.__setattr__(self, _REBOUND_STATE[name], value) + if name == "_core_cache": + self._hosted._core_cache = value # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # usage reads come from the routing cache + elif name == "_core_slack_alerting_logger": + from litellm.integrations.custom_logger import CustomLogger + + # `_slack_alerting_loggers` reads the attribute each time: the new logger is the one bound. + self._checks.slack_alerting_loggers = (value,) if isinstance(value, CustomLogger) else () + else: + # Any other attribute is kept as written; `_core_async_get_available_deployment` (the + # deprecated async pick) is then asked by every async pick (see `_pre_route`). + object.__setattr__(self, name, value) + + async def select( # pyright: ignore[reportIncompatibleMethodOverride] # the RouterCore selection + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + input: object = None, + health_check_probe: bool = False, + **_: object, # kwargs-ok: the RouterCore select options this core ignores + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + """Pick a deployment for ``model`` without calling it (``pick``), list the deployments a + pick would choose among (``healthy``), or pick one for a pass-through route + (``pass_through``, which picks with the sync strategies, as Python does).""" + if kind not in ("pick", "healthy", "pass_through"): + raise ValueError(f"unknown select kind {kind!r}") + self._refuse_deleted_reads({"num_retries": 0}) # a pick reads no retry budget + request: Final = request_kwargs if request_kwargs is not None else {} + selection: Final = _Selection( + self, + model, + request, + kind, + specific_deployment, + messages, + input, + probe=kind == "healthy" and health_check_probe, + ) + # A pick that makes no call holds no least-busy reservation (`select` on the Python core). + with picks_reserve(False): + if kind == "pass_through": + return selection.run_sync() + return await selection.run() + + async def async_get_available_deployment_for_pass_through( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the legacy signature + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + ) -> Mapping[str, object]: + """The full healthy chain, then only ``use_in_pass_through`` deployments, picked with the + async strategies.""" + selection: Final = _Selection( + self, model, request_kwargs, "async_pass_through", specific_deployment, messages, input + ) + picked: Final = await selection.run() + return picked if isinstance(picked, Mapping) else {} + + def get_available_deployment( + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the legacy signature + ) -> Mapping[str, object]: + """The sync pick: no pre-routing, the sync strategies.""" + selection: Final = _Selection( + self, + model, + request_kwargs if request_kwargs is not None else {}, + "sync_pick", + specific_deployment, + messages, + input, + ) + picked: Final = selection.run_sync() + return picked if isinstance(picked, Mapping) else {} + + def get_available_deployment_for_pass_through( + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool | None = False, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the legacy signature + ) -> Mapping[str, object]: + """Deprecated: ``select(model, kind="pass_through")``.""" + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("get_available_deployment_for_pass_through", 'select(model, kind="pass_through")') + selection: Final = _Selection( + self, + model, + request_kwargs if request_kwargs is not None else {}, + "pass_through", + specific_deployment, + messages, + input, + ) + picked: Final = selection.run_sync() + return picked if isinstance(picked, Mapping) else {} + + # -- endpoints ------------------------------------------------------------------------------- + + async def call(self, op: str, request: Mapping[str, object]) -> object: + """Run one endpoint with the request's keyword payload, as ``getattr(core, op)(**request)``.""" + member: Final[object] = getattr(self, op) # pyright: ignore[reportAny] # litellm's untyped helpers and request values + if not callable(member): + raise TypeError(f"{op!r} is not an endpoint") + result: Final[object] = member(**request) + return await result if inspect.isawaitable(result) else result + + async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.acompletion(request_from_call("acompletion", args, kwargs)) + + async def atext_completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.atext_completion(request_from_call("atext_completion", args, kwargs)) + + async def aembedding(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("aembedding"), request_from_call("aembedding", args, kwargs)) + + async def aresponses(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.generic(plan_for("aresponses"), request_from_call("aresponses", args, kwargs)) + + async def aanthropic_messages(self, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.generic(plan_for("aanthropic_messages"), kwargs) + + async def anthropic_messages(self, **kwargs: object) -> object: # kwargs-ok: endpoint payload + """The proxy's ``/v1/messages`` route name for ``aanthropic_messages``; async, as in PythonCore.""" + return await self._endpoints.generic(plan_for("anthropic_messages"), kwargs) + + async def aimage_generation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("aimage_generation"), request_from_call("aimage_generation", args, kwargs)) + + async def atranscription(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._drive(plan_for("atranscription"), request_from_call("atranscription", args, kwargs)) + + async def aspeech(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + request: Final = dict(request_from_call("aspeech", args, kwargs)) + request.setdefault("voice", None) + return await self._drive(plan_for("aspeech"), request) + + async def arerank(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + # `arerank` sets kwargs["input"] = input: the Python builtin, which reaches the deployment call. + request: Final = {**request_from_call("arerank", args, kwargs), "input": input} + return await self._drive(plan_for("arerank"), request) + + async def amoderation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return await self._endpoints.amoderation(request_from_call("amoderation", args, kwargs)) + + def completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._call_sync(plan_for("completion"), request_from_call("completion", args, kwargs)) + + def text_completion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._endpoints.text_completion(request_from_call("text_completion", args, kwargs)) + + def embedding( + self, + model: str, + input: str | list[object], # mutable-ok: the request field, as the Python core takes it + is_async: bool | None = False, + **kwargs: object, # kwargs-ok: endpoint payload + ) -> object: + """``embedding``'s own signature; as there, ``is_async`` is named and goes no further.""" + return self._call_sync(plan_for("embedding"), {**kwargs, "model": model, "input": input}) + + def image_generation(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload + return self._call_sync(plan_for("image_generation"), request_from_call("image_generation", args, kwargs)) + + # -- driving a call -------------------------------------------------------------------------- + + def _start( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + prepared_already: bool = False, + ) -> "tuple[RouterCall[_CallSteps], Mapping[str, object]]": + self._refuse_deleted_reads({**request, **(overrides or {})}) + self._sync_routing_state() + model: Final = request.get("model") + if model is None and "model" in request: + raise self._no_model_error(plan, request) + if "model" not in request: + raise TypeError(f"{plan.endpoint}() needs a model name") + name: Final = model if isinstance(model, str) else self.routed_name(model) + field: Final = plan.prepare_field or "metadata" + if request.get(field) is not None and not isinstance(request.get(field), Mapping): + raise TypeError(f"{field} must be a mapping") + prepared: Final = dict(request) if prepared_already else self._endpoints.prepare(plan, request, model) # pyright: ignore[reportArgumentType] # the name as given, which the metadata keeps + return self._handle.start( + plan.endpoint, self._routing_inputs(plan, request, prepared, name, overrides) + ), prepared + + def service_logged(self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object]) -> None: + """The ``ServiceTypes.ROUTER`` success event of a timed step, as a task of the running loop.""" + import asyncio + + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.types.services import ServiceTypes + + _ = asyncio.create_task( # fire and forget, as the Python core schedules it + self.service_logger_obj.async_service_success_hook( # pyright: ignore[reportUnknownMemberType] # its unused `event_metadata` is an untyped dict + service=ServiceTypes.ROUTER, + duration=end_time - start_time, + call_type=call_type, + start_time=start_time, + end_time=end_time, + parent_otel_span=get_parent_otel_span_from_kwargs(request), + ) + ) + + def track_deployment_pick(self, deployment_id: object, request: Mapping[str, object]) -> None: + """``_track_deployment_metrics`` for a pick: ``_update_usage``'s count of the deployment's + picks in the router cache, on this worker only. A default deployment served under a + placeholder id counts under the id Python's copy of it holds.""" + from litellm._logging import verbose_router_logger + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + parent_otel_span: Final = get_parent_otel_span_from_kwargs(request) + try: + model_id: Final = ( + self._default_deployment_id() + if deployment_id in (DEFAULT_DEPLOYMENT_FAILING_ID, DEFAULT_DEPLOYMENT_COPIED_ID) + else deployment_id + ) + if model_id is None: + return + counts: Final = self._pick_counts() + count: Final = counts.get_cache(key=model_id, parent_otel_span=parent_otel_span, local_only=True) + if count is None: + counts.set_cache(key=model_id, value=1, local_only=True, ttl=60) # only store for 60s + else: + counts.set_cache(key=model_id, value=count + 1, local_only=True) + except Exception as e: # noqa: BLE001 # as `_track_deployment_metrics` catches it + verbose_router_logger.error("Error in _track_deployment_metrics: %s", e) + + def _pick_counts(self) -> _PickCounts: + return self._cache + + def _default_deployment_id(self) -> object: + return self._resolved.default_deployment.get("model_info", {}).get("id", None) # pyright: ignore[reportOptionalMemberAccess, reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # read as Python reads it + + def routed_name(self, model: object, *, read_as_member: bool = False) -> str: + """The name a ``model`` that is no string routes under, for the native calls, which take strings + alone. Python's first reads raise for most values: the alias lookup hashes it (a sync pick reads + ``model in _core_model_names`` first, ``read_as_member``), then ``_core_get_model_list`` reaches + ``declared_provider``, which searches it. A value both accept (falsy, a tuple) is a name no + deployment serves, and ``str(model)`` is how every error names it.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if read_as_member: + _ = model in frozenset[object]() + else: + _ = self._hosted._core_get_model_from_alias(model) # pyright: ignore[reportPrivateUsage, reportArgumentType] # raising is the point + _ = inferred_provider(model) # pyright: ignore[reportArgumentType] # see above + return str(model) + + def _no_model_error(self, plan: Plan, request: Mapping[str, object]) -> BaseException: + """``model=None``: the pick fails in ``_common_checks_available_deployment``, naming the + first default fallback when there is one, and with no group the fallback wrapper raises + that error as it is (``async_function_with_fallbacks_common_utils``).""" + import litellm.exceptions as litellm_exceptions + + named: Final = next( + ( + targets[0] + for entry in _as_list(self._resolved.fallbacks) + if (targets := _as_list((as_str_mapping(entry) or {}).get("*"))) + ), + None, + ) + error: Final = litellm_exceptions.BadRequestError( + message=f"You passed in model={named}. {RouterErrors.no_healthy_deployments.value}", + model=named, # pyright: ignore[reportArgumentType] # None, as the Python router passes it + llm_provider="", + ) + if not plan.sync: + _Driver(self, plan, request)._log_pick_failure(error) # pyright: ignore[reportPrivateUsage] # the async pick's own logging + return error + + def _routing_inputs( + self, + plan: Plan, + request: Mapping[str, object], + prepared: Mapping[str, object], + model: str, + overrides: Mapping[str, object] | None, + ) -> Mapping[str, object]: + """The document router-core starts a call from: what it routes on, then what this call changes.""" + fields: dict[str, object] = {"model": model} # mutable-ok: built up below, later keys win # rebind-ok: same + fields.update(self._routing_fields(prepared, plan.pick_input(prepared)[0])) + fields.update(pinned_video_deployment(plan, request)) + fields.update(unrouted_fields(plan)) + fields.update(failover_fields(plan)) + fields.update(scoped_fallback_fields(plan, request)) + fields.update(self._selection_fields(plan, prepared)) + fields.update(self._outcome_fields(plan, request)) + # The sync path never runs pre-routing; a custom strategy picks on both. + if not plan.sync or self._custom_strategy is not None: + fields["pre_routes"] = True + if (mock := mock_failure(request)) is not None: + fields["mock_failure"] = mock + fields.update(self._retry_policy_raises(request)) + if overrides is not None and "num_retries" in overrides: + fields.pop("uncounted_retries", None) + fields.update(overrides or {}) + return _document(fields) + + def _outcome_fields(self, plan: Plan | None, request: Mapping[str, object]) -> Mapping[str, object]: + """Which outcome callbacks run for the request: an endpoint that makes one attempt skips + ``make_call``, and a recovered stream is async.""" + sync: Final = plan is not None and plan.sync + make_call: Final = plan is None or not plan.single_attempt + callbacks: Final = self.outcome_callbacks(request, sync=sync, make_call=make_call) + return {"outcome_callbacks": callbacks} if callbacks else {} + + def _selection_fields(self, plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """What an endpoint's pick reads beyond the routing fields: the narrowing keys the caller + passed, the endpoint ``endpoint_selection`` names, and whether it asks for web search, + which only an async pick filters by.""" + endpoint: Final = selection_endpoint(self._port, plan) + web_search: Final = not plan.sync and self._picks_async() and is_web_search_request(request) + # The generic handler `_route_capability_pin` names: a call only some providers serve + # drops the others, and a Decisions call carrying media skips the deployments refusing it. + handler: Final = plan.host_op.rsplit(".", 1)[-1] + decisions_media: Final = handler in DECISIONS_MEDIA_CALL_TYPES and bool( + request.get("audio") or request.get("videos") + ) + return { + **_request_narrowing(request), + **({"endpoint": endpoint} if endpoint is not None else {}), + **({"web_search": True} if web_search else {}), + **({"route_handler": handler} if serves_route_capability(handler) else {}), + **({"decisions_media": True} if decisions_media else {}), + } + + def _picks_async(self) -> bool: + return picks_async(normalize_strategy(self._resolved.routing_strategy)) + + def _process_renamed(self, model: str, group: str) -> str: + """``model`` as the error names it (``_common_checks_available_deployment``): the group a name + resolved to by name, alias or routing group, renamed through the process ``model_alias_map``.""" + resolution: Final = self._handle.resolve(model) + if resolution is None or resolution["kind"] not in ("Group", "Alias", "RoutingGroup"): + return model + renames: Final = self._host.dynamic().model_alias_map + return renames[group] if group in renames else group # noqa: SIM401 # `renames` is any process mapping, read as Python reads it + + def _max_parallel_requests_client(self, deployment: Mapping[str, object], kwargs: Mapping[str, object]) -> object: + """``_get_client(client_type="max_parallel_requests")``: the deployment's slot limit.""" + return max_parallel_requests_limit(self._port, deployment, kwargs) + + def _routable( + self, model: str, request: Mapping[str, object], call_type: str | None = None + ) -> Sequence[Mapping[str, object]]: + """``_core_get_routable_deployments``: ``call_type`` names the endpoint it asks about.""" + endpoint: Final = endpoint_for_call(call_type) + return self._handle.routable( + model, + _document( + { + **_start_fields(request), + **({"endpoint": endpoint} if endpoint is not None else {}), + "wildcard_request": wildcard_request(model), + } + ), + ) + + def _routing_fields(self, request: Mapping[str, object], messages: object = None) -> Mapping[str, object]: + """The request fields the Rust core routes on, and whether its tag stage may apply: the + router's ``enable_tag_filtering``, the request's own, or a deployment's + ``model_info.enable_tag_filtering`` (``get_deployments_for_tag``'s effective setting).""" + filter_tags: Final = ( + self._resolved.enable_tag_filtering + or request.get("enable_tag_filtering") is True + or self._model_tag_filtering() + ) + fields: dict[str, object] = dict(_start_fields(request)) # mutable-ok: built up below # rebind-ok: same + fields.update(self._group_policy_fields(request)) + if filter_tags: + fields["filter_tags"] = True + if request_carries_image(messages, request): + fields["has_image"] = True + return fields + + def _group_policies(self, request: Mapping[str, object]) -> tuple[object, GroupPolicies]: + """The ``model_group_retry_policy`` the retry loop reads for this request (the request's own, + even ``None``, replaces the router's: ``kwargs.pop(..., self.model_group_retry_policy)``), + resolved for every group name the call may look up.""" + policies: Final = request.get("model_group_retry_policy", self._resolved.model_group_retry_policy) + names: Final = self._policy_lookup_names(request) + resolved: Final = group_policies(policies, names) + if reads_as_policy(self._written_retry_policy): + return policies, resolved + # A group no entry names looks up the router's `retry_policy`, and that lookup raises. + unnamed: Final = names - frozenset(resolved.document) + return policies, dataclasses.replace(resolved, raising=resolved.raising | unnamed) + + def _policy_lookup_names(self, request: Mapping[str, object]) -> frozenset[str]: + """The model names a retry policy lookup may run on (each hop looks up the name it calls): + the requested one and every name in the fallback lists the walk may take.""" + resolved: Final = self._resolved + lists: Final = ( + resolved.fallbacks, + resolved.context_window_fallbacks, + resolved.content_policy_fallbacks, + *(request.get(key) for key in _REQUEST_FALLBACK_KEYS), + ) + return frozenset((*_strings(request.get("model")), *_strings(lists))) + + def _retry_policy_raises(self, request: Mapping[str, object]) -> Mapping[str, object]: + """The groups whose ``model_group_retry_policy`` lookup raises on a hop's first failure, for + router-core to end that hop with the error the lookup raises (see ``_Driver.take_events``).""" + raising: Final = raising_groups(request, self._group_policies(request)[1]) + return {"retry_policy_raises": list(raising)} if raising else {} + + def _group_policy_fields(self, request: Mapping[str, object]) -> Mapping[str, object]: + """The request's ``model_group_retry_policy`` for router-core; a router policy that is not a + mapping of policies rides on every request, resolved for it.""" + policies, resolved = self._group_policies(request) + if "model_group_retry_policy" not in request and is_policy_per_group(policies): + return {} + return {"model_group_retry_policy": _plain(resolved.document)} + + def _model_tag_filtering(self) -> bool: + """Whether any deployment sets ``model_info.enable_tag_filtering``, cached per registry version.""" + cached: Final = self._tag_filtering_models + if cached is not None and cached[0] == self._version: + return cached[1] + found: Final = any( + (as_str_mapping((as_str_mapping(record) or {}).get("model_info")) or {}).get("enable_tag_filtering") + is not None + for record in self._handle.deployments() + ) + self._tag_filtering_models = (self._version, found) + return found + + async def _drive( + self, + plan: Plan, + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + prepared_already: bool = False, + ) -> object: + routed: Final = ( + overrides if passthrough is None else {**(overrides or {}), "passthrough_on_no_deployment": True} + ) + # `owning_picks`: least-busy frees every pick this request made but never started. + with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + call, prepared = self._start(plan, request, routed, prepared_already=prepared_already) + arm_compaction_state(self._port, plan, prepared) # pyright: ignore[reportArgumentType] # the call's own dict + with picks_reserve(plan.endpoint not in _UNRESERVED_OPS): + driver, step = await self._run(plan, call, prepared, fresh=True, passthrough=passthrough) + if step["kind"] == "failed": + raise driver.failed(step) + return await driver.finish_async(step) + + async def _run( + self, + plan: Plan, + call: "RouterCall[_CallSteps]", + prepared: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + fresh: bool = False, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + ) -> "tuple[_Driver, Mapping[str, object]]": + """Drive ``call`` to its ``done`` or ``failed`` step, answering every step on the host. + ``cut_deployment`` is the attempt whose stream failed, for a recovery call, and + ``stream_failure`` the level it served with the failure a fallback walk re-enters from. + ``fresh`` is a call from the top, through the retry and fallback wrappers, and + ``passthrough`` serves an attempt whose pick failed.""" + driver: Final = _Driver( + self, plan, prepared, cut_deployment, stream_failure, fresh=fresh, passthrough=passthrough + ) + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + driver.take_events(step.get("events")) + # `run_async_fallback` stamps a served response before it logs the hop's success: `finish` + # runs what a served call still has pending. + if step["kind"] != "done": + await await_all(driver.outcomes.take_pending()) + match step["kind"]: + case "attempt": + outcome = await driver.attempt(step) + case "sleep": + await self._host.sleep(_sleep_seconds(step)) + outcome = None + case "allow_fallback": + outcome = await driver.allow_fallback(str(step["target"])) + case "refuse_fallbacks": + outcome = driver.refuse_fallbacks(step) + case "cache": + outcome = await self._answer_cache(step) + case "select": + outcome = await driver.select(step) + case "pre_route": + outcome = await driver.pre_route(step) + case "done" | "failed": + return driver, step + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + def _call_sync( + self, plan: Plan, request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: + # The sync routed calls pick with `picks_reserve(True)` inside `owning_picks`. + with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + call, prepared = self._start(plan, request, overrides) + with picks_reserve(plan.endpoint not in _UNRESERVED_OPS): + driver, step = self._run_sync(plan, call, prepared, fresh=True) + if step["kind"] == "failed": + raise driver.failed(step) + return driver.finish(step) + + def _run_sync( + self, + plan: Plan, + call: "RouterCall[_CallSteps]", + prepared: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + fresh: bool = False, + ) -> "tuple[_Driver, Mapping[str, object]]": + driver: Final = _Driver(self, plan, prepared, cut_deployment, stream_failure, sync=True, fresh=fresh) + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + driver.take_events(step.get("events")) + if step["kind"] != "done": # see _run + driver.log_outcomes_sync() + match step["kind"]: + case "attempt": + outcome = driver.attempt_sync(step) + case "sleep": + self._host.sleep_sync(_sleep_seconds(step)) + outcome = None + case "allow_fallback": + from litellm.litellm_core_utils import asyncify + + outcome = asyncify.run_async_function(driver.allow_fallback, str(step["target"])) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + case "refuse_fallbacks": + outcome = driver.refuse_fallbacks(step) + case "cache": + outcome = self._answer_cache_sync(step) + case "select": + outcome = driver.select_sync(step) + case "pre_route": + outcome = driver.pre_route_sync(step) + case "done" | "failed": + return driver, step + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + # -- stream recovery (the host's stream wrappers, router_rust.invoke.streaming) --------------------- + + def _recovery_call( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> "RouterCall[_CallSteps]": + start: Final = self._handle.retry_stream if recovery == "retry" else self._handle.fallback_stream + plan: Final = generic_plan(op) + routing: Final = _document( + { + "model": _routed_model(request), + **self._routing_fields(request), + **(self._selection_fields(plan, request) if plan is not None else {}), + **self._retry_policy_raises(request), + **continuation_routing(self, request), + **self._outcome_fields(plan, request), + } + ) + return start(op, routing, _document(origin), attempt_failure(failure)) + + def stream_policy( + self, request: Mapping[str, object], origin: Mapping[str, object], trigger: BaseException | None + ) -> Mapping[str, object]: + routing: Final = _document( + { + "model": _routed_model(request), + **_start_fields(request), + **self._group_policy_fields(request), + **self._retry_policy_raises(request), + } + ) + kind: Final = None if trigger is None else str(attempt_failure(trigger)["kind"]) + return self._handle.stream_policy(routing, _document(origin), kind) + + async def recover_stream( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> object: + """The fallback walk for a stream that failed before content: the fallback's response with the + router's headers, or the error the walk ended on.""" + call: Final = self._recovery_call(recovery, op, request, origin, failure) + driver, step = await self._run( + plan_for(op), call, request, origin.get("deployment_id"), (_origin_level(origin), failure) + ) + if step["kind"] == "failed": + raise driver.failed(step) + return await driver.finish_async(step) + + def recover_stream_sync( + self, + recovery: str, + op: str, + request: Mapping[str, object], + origin: Mapping[str, object], + failure: BaseException, + ) -> object: + call: Final = self._recovery_call(recovery, op, request, origin, failure) + driver, step = self._run_sync( + plan_for(op), call, request, origin.get("deployment_id"), (_origin_level(origin), failure) + ) + if step["kind"] == "failed": + raise driver.failed(step) + return driver.finish(step) + + async def retry_stream( + self, op: str, request: Mapping[str, object], origin: Mapping[str, object], failure: BaseException + ) -> "tuple[object, int, int] | BaseException": + """A same-group retry of a stream that failed before content: the retried response (without the + router's headers) with the retry count and budget, or the newest failure (``failure`` itself + when no retry ran).""" + call: Final = self._recovery_call("retry", op, request, origin, failure) + driver, step = await self._run(plan_for(op), call, request, origin.get("deployment_id")) + await await_all(driver.outcomes.take_pending()) + if step["kind"] == "failed": + return driver.last_error if driver.last_error is not None else failure + attempted: Final = step.get("attempted_retries") + budget: Final = step.get("max_retries") + return ( + driver.response, + attempted if isinstance(attempted, int) else 0, + budget if isinstance(budget, int) else 0, + ) + + def params_sent_to_deployment(self, request: Mapping[str, object], deployment_id: object) -> Mapping[str, object]: + """The request as a deployment received it: its params, the router defaults, its tool choice, then + the request (``_params_sent_to_deployment``).""" + record: Final = self._handle.deployment(deployment_id) if isinstance(deployment_id, str) else None + params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {} + defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} + tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} + return MappingProxyType({**params, **defaults, **tool_choice, **request}) + + def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]: + """Drive a call whose only steps are cache ops (``report_failure``) to its end.""" + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + match step["kind"]: + case "cache": + outcome = self._answer_cache_sync(step) + case "done": + return step + case "failed": + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core asked for {other!r} while recording an outcome") + + async def _drive_cache_steps( + self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]" + ) -> Mapping[str, object]: + """``_drive_sync_steps`` answered through the async cache (``report_success``).""" + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = call.next(outcome) + match step["kind"]: + case "cache": + outcome = await self._answer_cache(step) + case "done": + return step + case "failed": + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core asked for {other!r} while recording an outcome") + + # -- pre-route steps ------------------------------------------------------------------------- + + async def _pre_route( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object = _NO_INPUT, + input: object = _NO_INPUT, + specific_deployment: object = _NO_INPUT, + *, + pass_through: bool = False, + ) -> object: + """The host's pre-routing of one pick (router_rust.steps.prerouting): the model group to resolve + instead of the step's ``model``, a custom strategy's deployment, or ``None``. ``pass_through`` + is ``_async_get_available_deployment_for_pass_through``, which runs the hook whatever the + router's strategy and which no replaced pick answers.""" + assigned: Final[object] = None if pass_through else self.__dict__.get("_core_async_get_available_deployment") + if callable(assigned): + # The async pick assigned through the deprecated attribute answers, as Python's + # replaced `_core_async_get_available_deployment` does. + answer: Final[object] = assigned( + model=str(step.get("model")), + request_kwargs=request, + messages=request.get("messages") if messages is _NO_INPUT else messages, + input=request.get("input") if input is _NO_INPUT else input, + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, + ) + chosen: Final[object] = await answer if inspect.isawaitable(answer) else answer + return _custom_pick(chosen) + custom: Final = None if pass_through else self._custom_strategy + if custom is not None: + picked: Final[object] = await custom.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick + model=str(step.get("model")), + messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=request.get("input") if input is _NO_INPUT else input, # pyright: ignore[reportArgumentType] # the request's input, as given + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, # pyright: ignore[reportArgumentType] # same + request_kwargs=request, # pyright: ignore[reportArgumentType] # the request kwargs, as given + ) + return _custom_pick(picked) + if not isinstance(request, dict): + return None + depth: Final = step.get("fallback_depth") + r: Final = self._resolved + owner: Final[object] = self.__dict__.get("_core_owner") + public: Final[object] = owner() if callable(owner) else None + return await self._prerouter.pre_route( + str(step.get("model")), + request, # pyright: ignore[reportUnknownArgumentType] # the request kwargs the hook stamps + messages=request.get("messages") if messages is _NO_INPUT else messages, + input=request.get("input") if input is _NO_INPUT else input, + fallback_depth=depth if isinstance(depth, int) else 0, + router_strategy=normalize_strategy(r.routing_strategy), + pass_through=pass_through, + enable_tag_filtering=r.enable_tag_filtering, + match_any=r.tag_filtering_match_any, + public_router=public if public is not None else self, + run_plugins=partial( + self._run_plugins, request, request.get("messages") if messages is _NO_INPUT else messages + ) # pyright: ignore[reportUnknownArgumentType] # the request kwargs the plugins stamp + if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list + else None, + ) + + async def _run_plugins( + self, + request: dict[str, object], # mutable-ok: the plugins stamp their outcome on the caller's request + messages: object, + model: str, + ) -> None: + """``_run_routing_plugins`` over the pre-routing hook's session-resolved ``model``.""" + await run_plugins( + tuple(self._hosted._core_routing_plugins), # pyright: ignore[reportPrivateUsage] # the live plugin list + self._hosted, # pyright: ignore[reportArgumentType] # the host registry serves the listing + self._resolved.model_group_alias, + model, + request, + messages, + ) + + def _custom_pick_sync( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object = _NO_INPUT, + input: object = _NO_INPUT, + specific_deployment: object = _NO_INPUT, + ) -> object: + """A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing.""" + custom: Final = self._custom_strategy + if custom is None: + return None + picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick + model=str(step.get("model")), + messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route + input=request.get("input") if input is _NO_INPUT else input, # pyright: ignore[reportArgumentType] # same + specific_deployment=request.get("specific_deployment") + if specific_deployment is _NO_INPUT + else specific_deployment, # pyright: ignore[reportArgumentType] # same + request_kwargs=request, # pyright: ignore[reportArgumentType] # same + ) + return _custom_pick(picked) + + # -- select steps ---------------------------------------------------------------------------- + + def _strategy_inputs(self, step: Mapping[str, object], request: Mapping[str, object]) -> "_StrategyInputs | None": + """The strategy, its selector, the name it keys state by and the candidate deployments of a + ``strategy`` select step; ``None`` for a stage the host passes through.""" + if step.get("stage") != "strategy": + return None + strategy: Final = str(step.get("strategy")) + group: Final = step.get("group") + selector: Final = self._selectors.selector_for( + strategy, str(step.get("scope")), str(group) if group is not None else None, request + ) + candidates: Final = [ + dict(as_str_mapping(self._handle.deployment(str(id))) or {}) for id in _as_list(step.get("candidates")) + ] + return strategy, selector, str(step.get("model")), candidates + + def _records_of(self, ids: object) -> list[dict[str, object]]: # mutable-ok: the deployment dicts filters take + return [dict(as_str_mapping(self._handle.deployment(str(id))) or {}) for id in _as_list(ids)] + + async def _answer_tags(self, step: Mapping[str, object], request: Mapping[str, object]) -> object: + """``get_deployments_for_tag`` over the step's candidates (bridged: not ported to Rust yet).""" + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + + kept: Final = await get_deployments_for_tag( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the filter reads registry() and settings() + model=str(step.get("model")), + healthy_deployments=self._records_of(step.get("candidates")), # pyright: ignore[reportArgumentType] # deployment dicts + request_kwargs=dict(request), # pyright: ignore[reportArgumentType] # the request kwargs + metadata_variable_name=get_metadata_variable_name_from_kwargs(dict(request)), + request_deployments=self._records_of(step.get("pool")), # pyright: ignore[reportArgumentType] # deployment dicts + ) + return [found for record in kept if (found := picked_id(record)) is not None] + + def _pre_call_facts( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object, input: object + ) -> PreCallFacts: + hosted: Final = self._hosted + + def record(id: str) -> Mapping[str, object]: + return hosted._core_get_model_info(id) or as_str_mapping(self._handle.deployment(id)) or {} # pyright: ignore[reportPrivateUsage] # see __init__ + + return PreCallFacts(self._host, hosted, step, record, request, messages, input) # pyright: ignore[reportArgumentType] # the host registry serves the reads + + def _text_only(self, step: Mapping[str, object]) -> list[str]: # mutable-ok: the step's answer + """The candidates that declare no vision support (bridged: needs the cost map).""" + group: Final = str(step.get("model_group")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if declares_no_vision(as_str_mapping(self._handle.deployment(str(id))) or {}, group) + ] + + def _route_incapable(self, step: Mapping[str, object]) -> list[str]: # mutable-ok: the step's answer + """The candidates whose provider cannot serve the call the step's handler names.""" + group: Final = str(step.get("model_group")) + handler: Final = str(step.get("handler")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if cannot_serve_route(as_str_mapping(self._handle.deployment(str(id))) or {}, group, handler) + ] + + def _media_refusing( + self, step: Mapping[str, object], request: Mapping[str, object] + ) -> list[str]: # mutable-ok: the step's answer + """The candidates that refuse the audio or videos a Decisions call carries.""" + group: Final = str(step.get("model_group")) + defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} + sent: Final = {**defaults, **request} + audio: Final = bool(request.get("audio")) + videos: Final = bool(request.get("videos")) + return [ + str(id) + for id in _as_list(step.get("candidates")) + if refuses_decisions_media( + as_str_mapping(self._handle.deployment(str(id))) or {}, group, sent, audio=audio, videos=videos + ) + ] + + async def _answer_select( + self, + step: Mapping[str, object], + request: Mapping[str, object], + messages: object, + input: object, + *, + pass_through: bool = False, + ) -> object: + if step.get("stage") == "pre_call": + return await self._pre_call_facts(step, request, messages, input).answer_async() + if step.get("stage") == "routing_plugins": + # The plugins already ran in this pick's pre_route step (see `_pre_route`), on the + # session-resolved name and before a strategy was selected, as Python's hook runs them. + return candidate_models(request) + if step.get("stage") == "tags": + return await self._answer_tags(step, request) + if step.get("stage") == "text_only": + return self._text_only(step) + if step.get("stage") == "route_incapable": + return self._route_incapable(step) + if step.get("stage") == "media_refusing": + return self._media_refusing(step, request) + if step.get("stage") == "provider_request": + return _provider_request(step) + inputs: Final = self._strategy_inputs(step, request) + if inputs is None: + return step["candidates"] + strategy, selector, model, candidates = inputs + pinned: Final = bool(request.get("_encrypted_content_affinity_pinned")) and len(candidates) == 1 + if selector is None and not pinned: + return [] + # The strategy's service event: `_async_get_available_deployment` times it for the strategies + # with an async pick, `_async_get_available_deployment_for_pass_through` for any, on another clock. + timed: Final = not pinned and strategy != "simple-shuffle" and (pass_through or self._picks_async()) + clock: Final = perf_counter if pass_through else unix_time + start_time: Final = clock() + # `_async_get_available_deployment`: a pin by encrypted content takes no strategy's pick. + picked: Final = ( + candidates[0] + if pinned + else await pick_async( + strategy, + selector, # pyright: ignore[reportArgumentType] # a selector from selector_for + model, + candidates, + messages, + input, + dict(request), + # `_async_get_available_deployment` reserves a request's pick unless told not to; + # the pass-through pick only when told to. + reserve=pick_reserves(default=not pass_through) and bool(request), + ) + ) + if picked is not None and selector is not None and self._selectors.is_override(strategy, selector): + await selector.async_pre_call_check(dict(as_str_mapping(picked) or {}), None) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # every selector is a CustomLogger + if picked is not None and timed: + self.service_logged(".async_get_available_deployments", start_time, clock(), request) + found: Final = picked_id(picked) + return [found] if found is not None else [] + + def _answer_select_sync( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object, input: object + ) -> object: + if step.get("stage") == "pre_call": + return self._pre_call_facts(step, request, messages, input).answer() + if step.get("stage") == "routing_plugins": + return candidate_models(request) + if step.get("stage") == "callback_filters": + # Only the async candidate lists run the callback filters. + return step["candidates"] + if step.get("stage") == "text_only": + return self._text_only(step) + if step.get("stage") == "route_incapable": + return self._route_incapable(step) + if step.get("stage") == "media_refusing": + return self._media_refusing(step, request) + if step.get("stage") == "provider_request": + return _provider_request(step) + inputs: Final = self._strategy_inputs(step, request) + if inputs is None: + return step["candidates"] + strategy, selector, model, candidates = inputs + if selector is None: + return [] + picked: Final = pick_sync( + strategy, + selector, # pyright: ignore[reportArgumentType] # a selector from selector_for + model, + candidates, + messages, + input, + dict(request), + # The sync picks reserve only when a routed call site says so (`picks_reserve(True)`). + reserve=pick_reserves(default=False) and bool(request), + ) + if picked is not None and self._selectors.is_override(strategy, selector): + selector.pre_call_check(dict(as_str_mapping(picked) or {})) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # see above + found: Final = picked_id(picked) + return [found] if found is not None else [] + + # -- cache steps ----------------------------------------------------------------------------- + + def _store_for(self, key: str) -> "DualCache": + """Cooldowns live in the cooldown store; every other routing key in the routing cache.""" + return self._cooldowns.cooldown_store if key.endswith(":cooldown") else self._cache + + async def _batch_read(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list + """One batch read, as the Python core makes it: the keys memory cannot serve, at most once per + read interval. Cooldowns are ``async_get_cooldown_deployments``' read, through the request's + armed prefetch when there is one.""" + if not keys or not keys[0].endswith(":cooldown"): + read: Final[object] = await self._cache.async_batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above + store: Final = self._cooldowns.cooldown_store + with service_target(ROUTER_COOLDOWNS_TARGET): + prefetched: Final = await RoutingReadBatch._read_prefetched(((store, keys),)) # pyright: ignore[reportPrivateUsage] # the batch's own prefetch read + results: Final[object] = ( + prefetched[0] if prefetched is not None else await store.async_batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType] # see above + ) + return self._corrected_cooldowns(keys, results) + + def _batch_read_sync(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list + """``_batch_read`` for a sync pick: ``get_active_cooldowns``' read.""" + if not keys or not keys[0].endswith(":cooldown"): + read: Final[object] = self._cache.batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above + with service_target(ROUTER_COOLDOWNS_TARGET): + results: Final = as_object_list(self._cooldowns.cooldown_store.batch_get_cache(keys=keys)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # see above + return self._corrected_cooldowns(keys, results) + + def _corrected_cooldowns(self, keys: Sequence[str], results: object) -> Sequence[object]: + """The cooldowns a batch read found, once ``active_cooldowns_from_results`` corrected the in-memory + TTL of each it promoted from Redis and dropped each that expired.""" + if not isinstance(results, list): + return [None] * len(keys) + ids: Final = [key.removeprefix("deployment:").removesuffix(":cooldown") for key in keys] + _ = self._cooldowns.active_cooldowns_from_results(ids, results) # pyright: ignore[reportUnknownArgumentType] # see above + return results # pyright: ignore[reportUnknownVariableType] # see above + + async def _increment_usage(self, keys: Sequence[object], amounts: Sequence[object], ttl: float | None) -> None: + """``_increment_deployment_usage``'s write: one pipeline, then this worker's memory holds what + Redis answered, so its usage reads see what the other workers counted.""" + from litellm.types.caching import RedisPipelineIncrementOperation + + operations: Final = [ + RedisPipelineIncrementOperation( + key=str(key), increment_value=_amount(amount), ttl=None if ttl is None else int(ttl) + ) + for key, amount in zip(keys, amounts) + ] + values: Final = await self._cache.async_increment_cache_pipeline(increment_list=operations) + if values is None or self._cache.redis_cache is None: + return + for operation, value in zip(operations, values): + await self._cache.async_set_cache(operation["key"], int(value), local_only=True, ttl=operation["ttl"]) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + + async def _answer_cache(self, step: Mapping[str, object]) -> object: + with service_target(_cache_target(str(step.get("key")))): + return await self._answer_cache_in_target(step) + + async def _answer_cache_in_target(self, step: Mapping[str, object]) -> object: + key: Final = str(step.get("key")) + ttl: Final = _ttl(step) + local: Final = step.get("local") is True + match step.get("op"): + case "get": + return _plain(await self._store_for(key).async_get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped + case "batch_get": + return _plain(await self._batch_read([str(k) for k in _as_list(step.get("keys"))])) + case "set": + store: Final = self._store_for(key) + if ttl is None: + await store.async_set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + else: + await store.async_set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + return None + case "increment": + by: Final = _amount(step.get("value")) + counter: Final = self._store_for(key) + total: Final = ( + await counter.async_increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType] # see above + if ttl is None + else await counter.async_increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + ) + return by if total is None else total + case "increment_usage": + await self._increment_usage(_as_list(step.get("keys")), _as_list(step.get("value")), ttl) + return None + case "delete": + await self._store_for(key).async_delete_cache(key) + return None + case other: + raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + + def run_routing_cache_sync(self, read_or_write: "Callable[[], Coroutine[object, object, object]]") -> object: + """A routing-cache coroutine on a sync call, whose `make_call` runs on the one event loop + `run_async_function` opened for the whole call. Without Redis the coroutine never suspends, so + it runs here: a loop of its own would hand the GIL to the logging thread, which then reads the + response before the router's headers are on it.""" + if self._cache.redis_cache is not None: + from litellm.litellm_core_utils import asyncify + + return asyncify.run_async_function(read_or_write) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the untyped sync bridge + running: Final = read_or_write() + try: + while True: + _ = running.send(None) + except StopIteration as finished: + return finished.value # pyright: ignore[reportAny] # the coroutine's result + + def _answer_cache_sync(self, step: Mapping[str, object]) -> object: + with service_target(_cache_target(str(step.get("key")))): + return self._answer_cache_sync_in_target(step) + + def _answer_cache_sync_in_target(self, step: Mapping[str, object]) -> object: + key: Final = str(step.get("key")) + ttl: Final = _ttl(step) + local: Final = step.get("local") is True + match step.get("op"): + case "get": + return _plain(self._store_for(key).get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped + case "batch_get": + return _plain(self._batch_read_sync([str(k) for k in _as_list(step.get("keys"))])) + case "set": + store: Final = self._store_for(key) + if ttl is None: + store.set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + else: + store.set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + return None + case "increment": + by: Final = _amount(step.get("value")) + counter: Final = self._store_for(key) + # DualCache.increment_cache is annotated int but adds floats as given. + return ( + counter.increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above + if ttl is None + else counter.increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above + ) + case "increment_usage": + keys: Final = _as_list(step.get("keys")) + amounts: Final = _as_list(step.get("value")) + self.run_routing_cache_sync(lambda: self._increment_usage(keys, amounts, ttl)) + return None + case "delete": + self._store_for(key).delete_cache(key) # pyright: ignore[reportUnknownMemberType] # DualCache's delete is untyped + return None + case other: + raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + + # -- errors the core raises when no attempt failed ------------------------------------------ + + def _failure_error( # noqa: C901 # one branch per failure Python tells apart + self, + step: Mapping[str, object], + model: str, + last: BaseException | None, + *, + pass_through: str = "", + request: Mapping[str, object] | None = None, + ) -> BaseException: + """The error a call or pick that failed before any attempt raises; ``pass_through`` names + the selection kind (``pass_through``/``async_pass_through``) whose errors it words.""" + import litellm.exceptions as litellm_exceptions + + kind: Final = step.get("error_kind") + aliases: Final = self._resolved.model_group_alias + refused: Final = selection_refusal(step, model, request or {}) + if refused is not None: + return refused + if kind == "paused": + # The message names the deployment's litellm model, not the id that was asked for; the + # default deployment's copy is named by the requested name. + record: Final = as_str_mapping(self._handle.deployment(model)) + litellm_model: Final = ( + str((as_str_mapping(record.get("litellm_params")) or {}).get("model")) if record is not None else model + ) + return litellm_exceptions.ServiceUnavailableError( + message=f"Model '{litellm_model}' is currently paused and cannot accept requests.", + model=litellm_model, + llm_provider="", + ) + if kind == "only_markers": + named_model: Final = str(step.get("error_model") or model) + return litellm_exceptions.BadRequestError( + message=f"You passed in model={named_model}. {RouterErrors.only_strategy_marker_deployments.value}", + model=named_model, + llm_provider="", + ) + if kind == "endpoint_not_supported": + return litellm_exceptions.EndpointNotSupportedByModelError( + message=str(step.get("error_detail")), + model=str(step.get("error_model") or model), + llm_provider="", + ) + if kind == "ptu_reserved": + reserved: Final = str(step.get("error_model") or model) + return litellm_exceptions.BadRequestError( + message=f"Deployment {reserved} is reserved for the teams holding a PTU share of it", + model=reserved, + llm_provider="", + ) + if kind == "not_pass_through": + # The sync pick names what `_common_checks_available_deployment` resolved the id to. + named_record: Final = ( + as_str_mapping(self._handle.deployment(model)) if pass_through == "pass_through" else None + ) + error_model: Final = ( + str((as_str_mapping(named_record.get("litellm_params")) or {}).get("model")) + if named_record is not None + else model + ) + return litellm_exceptions.BadRequestError( + message=f"Deployment {model} does not support pass-through endpoint (use_in_pass_through=False)", + model=error_model, + llm_provider="", + ) + if kind == "no_pass_through_deployments": + # The sync pick names what `_common_checks_available_deployment` resolved the name to. + named_group: Final = str(step.get("error_model") or model) + listed_model: Final = ( + named_group if pass_through == "async_pass_through" else self._process_renamed(model, named_group) + ) + message: Final = ( + f"Model {listed_model} has no deployments configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true to the deployment configuration" + if pass_through == "async_pass_through" + else f"Model {listed_model} has no deployment configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true in the deployment configuration" + ) + return litellm_exceptions.BadRequestError(message=message, model=listed_model, llm_provider="") + if kind == "context_window_exceeded": + return litellm_exceptions.ContextWindowExceededError( + message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context " + f"window large enough for this call.\n{step.get('error_detail')}", + model=str(step.get("error_model") or model), + llm_provider="", + ) + if kind == "unsupported_params": + params: Final = sorted(str(param) for param in _as_list(step.get("error_params"))) + unsupported_model: Final = str(step.get("error_model") or model) + return litellm_exceptions.UnsupportedParamsError( + message=( + f"no deployment of model={unsupported_model} supports {params}. " + "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, " + f"or send allowed_openai_params={params} to pass them through" + ), + model=unsupported_model, + ) + if kind == "routing_plugins_left_nothing": + left: Final = [str(name) for name in _as_list(step.get("error_params"))] + return ValueError(f"No deployments left after routing-plugin filtering. candidate_models={left}") + if kind == "routing_plugins_need_async": + return ValueError(SYNC_PATH_MESSAGE) + looked_up: Final = _lookup_error(kind, str(step.get("error_model") or model), self._resolved.default_deployment) + if looked_up is not None: + return looked_up + if kind == "unknown_model": + # router-core names what `_common_checks_available_deployment` looked for last. + aliased: Final = _alias_target(aliases.get(model)) or model if model in aliases else model + named: Final = str(step.get("error_model") or aliased) + return litellm_exceptions.BadRequestError( + message=f"You passed in model={named}. {RouterErrors.no_healthy_deployments.value}", + model=named, + llm_provider="", + ) + if kind == "no_healthy_deployments": + group: Final = self.registry().resolve(model) + group_ids: Final = self.registry().names(str(group), kind="ids") + # A listed pass-through pick names only the pass-through deployments. + ids: Final = ( + [id for id in group_ids if _uses_pass_through(self._handle.deployment(id))] + if pass_through == "pass_through" + else list(group_ids) + ) + active: Final = self._cooldowns.get_active_cooldowns(list(ids), None) # pyright: ignore[reportArgumentType] # a list of ids + # `async_get_cooldown_deployments_with_debug_info`: every deployment cooling, in any group. + listed: Final = self._cooldowns.get_active_cooldowns(list(self.registry().names(kind="ids")), None) # pyright: ignore[reportArgumentType] # a list of ids + cooling: Final = [id for id, _ in listed] + min_cooldown: Final = min( + (_seconds(entry.get("cooldown_time"), self._cooldowns.default_cooldown_time) for _, entry in active), + default=_seconds(None, self._cooldowns.default_cooldown_time), + ) + return RouterRateLimitError( + model=self._process_renamed(model, str(group)), + cooldown_time=min_cooldown, + enable_pre_call_checks=self._resolved.enable_pre_call_checks, + cooldown_list=cooling, + model_ids=list(ids), # pyright: ignore[reportArgumentType] # a list of ids + cooldown_deployments=get_cooldown_deployment_info( + RoutingContext(self), # pyright: ignore[reportArgumentType] # reads registry() + ids, + cooling, + ), + ) + if last is not None: + return last + return RuntimeError(f"router-core ({kind}): {step.get('error')}") + + +def wildcard_request(model: str) -> str: + """``f"{provider}/{model}"``, the spelling ``PatternMatchRouter.get_pattern`` tries second.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return f"{inferred_provider(model)}/{model}" + + +def _provider_request(step: Mapping[str, object]) -> Mapping[str, object]: + return {"models": [wildcard_request(str(step.get("model")))]} + + +def _lookup_error(kind: object, model: str, default_deployment: object) -> BaseException | None: + """The plain Python error a pick's lookup raised for ``model``, or None for any other failure.""" + if kind == "default_deployment_raises": + return default_deployment_copy_error(default_deployment, model) + if kind == "index_out_of_range": + # `self._core_model_list[index]` for a written index position past the list. + return IndexError("list index out of range") + if kind == "missing_routing_group": + # `_get_routing_context` reads the stale member map's group from the replaced groups. + return KeyError(model) + return None + + +def _origin_level(origin: Mapping[str, object]) -> int: + level: Final = origin.get("level") + return level if isinstance(level, int) else 0 + + +def _sleep_seconds(step: Mapping[str, object]) -> float: + """A retry's sleep: ``_time_to_sleep_before_retry`` returns a plain ``0`` when another + deployment is healthy, else a backoff in float seconds.""" + seconds: Final = float(str(step["seconds"])) + return 0 if seconds == 0 else seconds + + +def _uses_pass_through(record: object) -> bool: + params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {} + return bool(params.get("use_in_pass_through")) + + +def _seconds(value: object, default: float) -> float | int: + """A cooldown time as stored: an int stays an int, so error texts read ``60`` like Python's.""" + if isinstance(value, int | float) and not isinstance(value, bool): + return value + return int(default) if float(default).is_integer() else default + + +def _ttl(step: Mapping[str, object]) -> float | None: + ttl: Final = step.get("ttl") + return ttl if isinstance(ttl, int | float) else None + + +def _amount(value: object) -> float | int: + return value if isinstance(value, int | float) and not isinstance(value, bool) else 1 + + +# The plans whose kwargs carry `original_generic_function`, which the fallback cooldown reads. +_GENERIC_API_KINDS: Final = frozenset(("generic", "search")) + + +class _NotSynchronous(TypeError): + """A sync attempt's host endpoint returned an awaitable: a host bug, never an attempt failure.""" + + +_MEMBER_READ_PICKS: Final = frozenset({"healthy", "pass_through", "sync_pick"}) + + +class _Selection: + """One ``select``: the Rust selection call, its select and cache steps answered on the host.""" + + def __init__( + self, + core: RustCore, + model: object, + request: dict[str, object], # mutable-ok: the caller's request, whose narrowing keys are popped + kind: str, + specific_deployment: bool | None, + messages: object, + input: object, + *, + probe: bool = False, + ) -> None: + self._core: Final = core + # `_async_get_available_deployment` reads the alias map first, every other pick its names. + self._model: Final = ( + model if isinstance(model, str) else core.routed_name(model, read_as_member=kind in _MEMBER_READ_PICKS) + ) + self._request: Final = request + self._kind: Final = kind + self._messages: Final = messages + self._input: Final = input + fields: Final = { + **core._routing_fields(request, messages), # pyright: ignore[reportPrivateUsage] # the selection serves its own core + **_narrowing_fields(request, kind), + } # pyright: ignore[reportPrivateUsage] # the selection serves its own core + # The pick's argument alone: a `specific_deployment` key in its request is no argument, and + # `_common_checks_available_deployment` reads nothing but True. + fields.pop("specific_deployment", None) + if specific_deployment is True: + fields["specific_deployment"] = True + # `filter_web_search_deployments` is in the async chain, which a strategy without an async + # selector never reaches on a pick. + if (kind in ("healthy", "async_pass_through") or (kind == "pick" and core._picks_async())) and ( # pyright: ignore[reportPrivateUsage] # the selection serves its own core + is_web_search_request(request) + ): + fields["web_search"] = True + # `_text_completion` wraps its sync pick in `endpoint_selection`. + endpoint: Final = selected_endpoint() + if endpoint is not None: + fields["endpoint"] = endpoint + if probe: + fields["health_check_probe"] = True + if kind in ("pick", "async_pass_through") or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core + fields["pre_routes"] = True + self._specific_deployment: Final = specific_deployment + # The error a callback filter raised, which a failed selection raises. + self._filter_error: BaseException | None = None + if kind == "pass_through": + fields["pass_through"] = "listed" + elif kind == "async_pass_through": + fields["pass_through"] = "healthy" + core._sync_routing_state() # pyright: ignore[reportPrivateUsage] # see above + handle: Final = core._handle # pyright: ignore[reportPrivateUsage] # see above + self._call: Final = (handle.healthy if kind == "healthy" else handle.select)(self._model, _document(fields)) + + def _done(self, step: Mapping[str, object]) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + if self._kind == "healthy": + if step.get("single"): + # A deployment named by its id is listed alone, as a dict. + (record,) = _as_list(step.get("deployments")) + return for_requested_model(typed_deployment(as_str_mapping(record) or {}), self._model) + # The deployments as served: a priority group's members carry their priority as order. + return [ + for_requested_model(typed_deployment(as_str_mapping(record) or {}), self._model) + for record in _as_list(step.get("deployments")) + ] + if step.get("deployment_id") in (DEFAULT_DEPLOYMENT_FAILING_ID, DEFAULT_DEPLOYMENT_COPIED_ID): + # A default deployment every call to fails, or whose id is not a string, is picked as Python copies it. + picked: Final = default_deployment_copy(self._core._resolved.default_deployment, self._model) # pyright: ignore[reportPrivateUsage] # the selection serves its own core + return picked # pyright: ignore[reportReturnType] # the assigned value's copy, as the Python core returns it + return for_requested_model(typed_deployment(as_str_mapping(step.get("deployment")) or {}), self._model) + + def _failed(self, step: Mapping[str, object]) -> BaseException: + core: Final = self._core + error: Final = core._failure_error( # pyright: ignore[reportPrivateUsage] # see above + step, self._model, self._filter_error, pass_through=self._kind, request=self._request + ) + if self._kind not in ("pass_through", "sync_pick"): + _log_refused_pick(self._request, error) + return error + + async def run(self) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + core: Final = self._core + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = self._call.next(outcome) + match step["kind"]: + case "cache": + outcome = await core._answer_cache(step) # pyright: ignore[reportPrivateUsage] # see above + case "select" if step.get("stage") == "callback_filters": + outcome, self._filter_error = await core._answer_callback_filters( # pyright: ignore[reportPrivateUsage] # see above + step, self._request, self._messages + ) + case "select": + outcome = await core._answer_select( # pyright: ignore[reportPrivateUsage] # see above + step, + self._request, + self._messages, + self._input, + pass_through=self._kind == "async_pass_through", + ) + case "pre_route": + outcome = await core._pre_route( # pyright: ignore[reportPrivateUsage] # see above + step, + self._request, + self._messages, + self._input, + self._specific_deployment, + pass_through=self._kind == "async_pass_through", + ) + case "done": + done = self._done(step) + if self._kind != "pick" or not isinstance(done, Mapping): + return done + # `_core_async_get_available_deployment` resolves its pick to a pod. + return await self._resolved_pod(core, done) + case "failed": + raise self._failed(step) + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + async def _resolved_pod(self, core: "RustCore", done: Mapping[str, object]) -> Mapping[str, object]: + discovery: Final = core._pick_discovery(sync=False) # pyright: ignore[reportPrivateUsage] # the selection serves its own core + if discovery is None: + return done + resolved: Final[object] = await discovery.async_resolve_deployment(done, self._request) + return resolved # pyright: ignore[reportReturnType] # the deployment, resolved + + def _resolved_pod_sync(self, core: "RustCore", done: Mapping[str, object]) -> Mapping[str, object]: + discovery: Final = core._pick_discovery(sync=True) # pyright: ignore[reportPrivateUsage] # see _resolved_pod + return done if discovery is None else discovery.resolve_deployment(done, self._request) + + def run_sync(self) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + core: Final = self._core + outcome: object = None # rebind-ok: the answer to the last step + while True: + step = self._call.next(outcome) + match step["kind"]: + case "cache": + outcome = core._answer_cache_sync(step) # pyright: ignore[reportPrivateUsage] # see above + case "select": + outcome = core._answer_select_sync(step, self._request, self._messages, self._input) # pyright: ignore[reportPrivateUsage] # see above + case "pre_route": + outcome = core._custom_pick_sync( # pyright: ignore[reportPrivateUsage] # see above + step, self._request, self._messages, self._input, self._specific_deployment + ) + case "done": + done = self._done(step) + if self._kind != "sync_pick" or not isinstance(done, Mapping): + return done + # `get_available_deployment` resolves its pick to a pod. + return self._resolved_pod_sync(core, done) + case "failed": + raise self._failed(step) + case other: # pyright: ignore[reportUnnecessaryComparison] # a kind the bridge returns but the stub does not list + raise RuntimeError(f"router-core returned an unknown step {other!r}") + + +def _log_refused_pick(request: Mapping[str, object], error: BaseException) -> None: + """A refused pick is logged to the request's logging object (`_async_get_available_deployment`).""" + dispatch: Final[object] = getattr(request.get("litellm_logging_obj"), "dispatch_failure_handlers", None) + if not callable(dispatch): + return + import asyncio + import traceback + + trace: Final = "Traceback (most recent call last):\n" + "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + pending: Final[object] = dispatch(exception=error, traceback_exception=trace, prefer_async_handlers=True) + if inspect.isawaitable(pending): + _ = asyncio.ensure_future(pending) + + +def _log_refused_image_pick(request: Mapping[str, object], error: BaseException) -> None: + """``_aimage_generation``'s refused pick, for a request with no logging object of its own.""" + from litellm._uuid import uuid + from litellm.router_rust.invoke.endpoints import ROUTING_KEYS + from litellm.utils import ( + Rules, + _restore_correlation_context_if_supported, # pyright: ignore[reportPrivateUsage] # same task-local cleanup as the Python router + function_setup, + ) + + kwargs: Final = without(request, *(ROUTING_KEYS - {"model"})) + kwargs.setdefault("litellm_call_id", str(uuid.uuid4())) + logging_obj, _ = function_setup("aimage_generation", Rules(), datetime.now(), **kwargs) # noqa: DTZ005 # SDK logging uses naive local timestamps # pyright: ignore[reportArgumentType] # opaque SDK options, as the Python router passes them + try: + _log_refused_pick({"litellm_logging_obj": logging_obj}, error) + finally: + _restore_correlation_context_if_supported(logging_obj) + + +def _custom_pick(picked: object) -> Mapping[str, object]: + """A custom strategy's answer as the deployment document the Rust core serves.""" + document: Final = as_str_mapping(picked) + if document is None: + raise TypeError(f"the custom routing strategy returned {type(picked).__name__}, not a deployment") + return _document(document) + + +def _with_target_order(kwargs: Mapping[str, object], order: object) -> Mapping[str, object]: + """The attempt's kwargs with the order level its hop targets, as `run_async_fallback` sets + `_target_order` before `_acreate_file` picks every healthy deployment of that level.""" + if not isinstance(order, int) or isinstance(order, bool): + return kwargs + return {**kwargs, "_target_order": order} + + +def _narrowing_fields(request: dict[str, object], kind: str) -> Mapping[str, object]: # mutable-ok: popped + """``_target_order``, ``_excluded_deployment_ids`` and ``_retry_skipped_deployment_ids``, popped + from the request as the healthy chain pops them (a listed pass-through pick only reads the order).""" + fields: Final = _request_narrowing(request) + if kind == "pass_through": + return {key: value for key, value in fields.items() if key == "target_order"} + for key in ("_target_order", "_excluded_deployment_ids", "_retry_skipped_deployment_ids"): + request.pop(key, None) + return fields + + +def _request_narrowing(request: Mapping[str, object]) -> Mapping[str, object]: + """The narrowing keys a request carries, as the fields router-core reads them.""" + from litellm.litellm_core_utils.routing_common_utils import ( + as_excluded_deployment_ids, + as_retry_skipped_deployment_ids, + ) + + target_order: Final = request.get("_target_order") + excluded: Final = as_excluded_deployment_ids(request.get("_excluded_deployment_ids")) + skipped: Final = as_retry_skipped_deployment_ids(request.get("_retry_skipped_deployment_ids")) + return { + **( + {"target_order": target_order} + if isinstance(target_order, int) and not isinstance(target_order, bool) + else {} + ), + **({"excluded_ids": list(excluded)} if excluded else {}), + **({"retry_skipped_ids": list(skipped)} if skipped else {}), + } + + +_NO_ROUTING_PRE_CALL_CHECKS: Final = frozenset({"search", "rerank", "moderation"}) +# Attempts whose deployment is no pick (`_core_async_get_available_deployment` / +# `get_available_deployment`), so Kubernetes pod discovery never resolves it. +_UNPICKED_KINDS: Final = frozenset({"search", "create_file"}) + + +class _Driver: + """Answers the attempt and fallback steps of one call, and shapes its result. + + Each attempt is one host call against the deployment Rust picked. The driver keeps the error + each fallback level ended on: a level that fails raises its own error, whatever its fallbacks + raised, as the Python router does, unless an order chain handed it another + (``order_chain_failed``). + """ + + def __init__( + self, + core: RustCore, + plan: Plan, + request: Mapping[str, object], + cut_deployment: object = None, + stream_failure: tuple[int, BaseException] | None = None, + *, + sync: bool = False, + fresh: bool = False, + passthrough: "Callable[[Mapping[str, object], str], Awaitable[object]] | None" = None, + ) -> None: + self._core: Final = core + self._sync: Final = sync + self._passthrough: Final = passthrough + self._plan: Final = plan + self._op: Final = plan.endpoint + self._request: Final = request + self._cut_deployment: Final = cut_deployment if isinstance(cut_deployment, str) else None + self._last_response: object = None + self._last_error: BaseException | None = None + # The level the call raises from: a fallback walk re-entered from a failed stream starts at + # the level that stream served, on the failure ``async_function_with_fallbacks_common_utils`` + # was handed, which is what it raises when no fallback answers. + self._base_level: Final = stream_failure[0] if stream_failure is not None else 0 + self._level_errors: Final[dict[int, BaseException]] = dict( # mutable-ok: filled as the call runs + (stream_failure,) if stream_failure else () + ) + # The retry counts an exhausted retry loop stamps on the pick error its level then raises. + self._pending_stamps: Final[dict[int, tuple[object, object]]] = {} # mutable-ok: filled as the call runs + # Each level's current hop's first failure, once it retries: the error its retry loop raises + # last is raised while that one is handled, so it chains to it. + self._hop_first: Final[dict[int, BaseException]] = {} # mutable-ok: filled as the call runs + # Each level's latest pick error: the one object it logged, recorded and raises. + self._level_picks: Final[dict[int, BaseException]] = {} # mutable-ok: filled as the call runs + self._served: tuple[str | None, Mapping[str, object]] = (None, MappingProxyType({})) + # The latest attempt's deployment and call, which a blocked response is raised from. + self._latest_call: tuple[Mapping[str, object], AttemptCall] | None = None + # A sync attempt's stream is reading its first chunk, so a failure now is the stream's to log. + self._reading_first_chunk: bool = False + self._attempts: int = 0 + # When the latest attempt started: Python reads it off the logging object, when one is set up. + self._attempt_started: datetime | None = None + # A single-attempt endpoint runs neither of the wrappers that stamp the walk on the request. + self._walk: Final = WalkMetadata( + request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt + ) + self.outcomes: Final = FallbackOutcomes(request) + # The fallback level the walk is at, from its fallback events. + self._level: int = self._base_level + # The primary level reached the end of `async_function_with_fallbacks_common_utils`, the + # only place the outcome message is appended: not its early raises, nor its order path. + self._explains_failure: bool = False + + def _blocked(self, response: object, endpoint_name: str) -> str | None: + """How the provider blocked a served response, the first check of + ``_should_raise_content_policy_error`` (a chat response with no choices, or whose first + choice was content-filtered) and of ``_should_raise_anthropic_refusal_error`` (a + /v1/messages safeguard refusal carrying ``stop_details``).""" + from litellm.llms.anthropic.messages.native.utils import get_safeguard_refusal_stop_details + from litellm.types.utils import ModelResponse + + if endpoint_name == "anthropic_messages" and get_safeguard_refusal_stop_details(response) is not None: + return "refusal" + if self._op not in ("acompletion", "completion") or not isinstance(response, ModelResponse): + return None + return ( + "content_filter" if not response.choices or response.choices[0].finish_reason == "content_filter" else None + ) + + def _blocked_error(self, event: Mapping[str, object]) -> BaseException: + """The error the router raises for a blocked response a fallback would serve.""" + import litellm.exceptions as litellm_exceptions + from litellm.llms.anthropic.messages.native.utils import ( + get_safeguard_refusal_stop_details, + safeguard_refusal_error, + ) + + model: Final = str(event.get("model")) + stop_details: Final = get_safeguard_refusal_stop_details(self._last_response) + if event.get("blocked") == "refusal" and stop_details is not None: + return safeguard_refusal_error(model=model, stop_details=stop_details) + return litellm_exceptions.ContentPolicyViolationError( + message="Response output was blocked.", model=model, llm_provider="" + ) + + def _stamp_blocked(self, error: BaseException) -> None: + """The attempt helper's ``except`` runs for the error it raises over a blocked response: the + response then counts as a failure, not a success.""" + if self._latest_call is None: + return + deployment, call = self._latest_call + endpoints: Final = self._core._endpoints # pyright: ignore[reportPrivateUsage] # the driver serves its own core + stamp_error(self._plan, error, deployment, call.served) + endpoints.counters.retract_success(self._plan, call) + endpoints.attempt_failed(self._plan, error, deployment, call, call.group) + + def _params_base_model(self, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" + record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + def _logged_error(self, event: Mapping[str, object]) -> BaseException | None: + """The error a ``retry_logged`` event records.""" + return self._cause_error(event) + + def _policy_raised(self, event: Mapping[str, object]) -> None: + """A hop's first failure looked its group's retry policy up and the lookup raised: the hop + ends with that error, raised from the failure the event names.""" + model: Final = str(event.get("model")) + original: Final = self._cause_error(event) or RuntimeError(f"router-core: the hop serving {model} failed") + policies: Final = self._request.get( + "model_group_retry_policy", + self._core._resolved.model_group_retry_policy, # pyright: ignore[reportPrivateUsage] # the driver serves its own core + ) + raised: Final = policy_lookup_error(original, model, self._core._written_retry_policy, policies) # pyright: ignore[reportPrivateUsage] # see above + level: Final = event.get("level") + self._last_error = raised + self._level_errors[level if isinstance(level, int) else 0] = raised + + def _stamped_budget(self, event: Mapping[str, object]) -> Mapping[str, object]: + """A ``retries_opened`` event as ``_metadata["max_retries"]`` reads it: a request + ``num_retries`` that is no count is stamped as given.""" + given: Final = self._request.get("num_retries") + return {**event, "max_retries": given} if "uncounted_retries" in request_retry_fields(given) else event + + def _count_raised(self, event: Mapping[str, object]) -> None: + """A hop's first failure reached the retry loop's read of a request ``num_retries`` that + raises: the hop ends with that error, raised from the failure the event names.""" + model: Final = str(event.get("model")) + original: Final = self._cause_error(event) or RuntimeError(f"router-core: the hop serving {model} failed") + raised: Final = retry_count_error(original, self._request.get("num_retries")) + level: Final = event.get("level") + self._last_error = raised + self._level_errors[level if isinstance(level, int) else 0] = raised + + def _cause_error(self, event: Mapping[str, object]) -> BaseException | None: + """The error the ``cause`` of a ``retry_logged`` or ``retry_policy_raised`` event names.""" + match event.get("cause"): + case "level": + level: Final = event.get("level") + return self._level_errors.get(level if isinstance(level, int) else 0, self._last_error) + case "pick": + at_level: Final = event.get("level") + held: Final = self._level_picks.get(at_level if isinstance(at_level, int) else 0) + return held if held is not None else self._picked_error(event) + case _: + return self._last_error + + def _picked_error(self, event: Mapping[str, object]) -> BaseException: + """The error a pick an event names raised, as a failed step with that error raises it.""" + model: Final = str(event.get("failed_model") or event.get("model")) + return self._core._failure_error(event, model, None, request=self._request) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + + def _pick_raised(self, error: Exception, *, logged: bool) -> Mapping[str, object]: + """A pick raised on the host (a strategy, a tag filter, a pre-routing hook): the try fails with + that error, which the Rust walk retries and falls back from like any attempt's. ``logged`` + when the async pick's ``except`` reports it, as ``_async_get_available_deployment`` does.""" + if isinstance(error, RustCoreUnsupported): + raise error + if logged: + self._log_pick_failure(error) + else: + mark_deployment_selection_failure(error) + self._chain_to_hop_first(error, self._level) + self._last_error = error + self._level_errors[self._level] = error + return {**attempt_failure(error), "raised": True} + + def _log_pick_failure(self, error: BaseException) -> None: + """The ``except`` of ``_async_get_available_deployment``, which only the async selection runs: + the router's strategy may have no async selector, and a custom strategy replaces the method. + ``_aimage_generation`` logs a refused pick itself when the request set up no logging object.""" + core: Final = self._core + assigned: Final = "_core_async_get_available_deployment" in core.__dict__ + custom: Final = assigned or core._custom_strategy is not None # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if not custom and core._picks_async(): # pyright: ignore[reportPrivateUsage] # see above + mark_deployment_selection_failure(error) + _log_refused_pick(self._request, error) + if self._plan.kind == "image" and self._request.get("litellm_logging_obj") is None: + _log_refused_image_pick(self._request, error) + + def _pick_failed(self, event: Mapping[str, object]) -> None: + """A pick failed in the Rust core: its error is raised once, however many events name it.""" + error: Final = self._picked_error(event) + level: Final = event.get("level") + self._level_picks[level if isinstance(level, int) else 0] = error + if not self._sync: + self._log_pick_failure(error) + + def _unroutable_error(self, model: object) -> BaseException: + """What a hop to a dict target's non-string ``model`` raises: no group answers ``None``, and + any other value fails the level's fallback lookup, which hashes it and then searches it.""" + if model is None: + return self._core._failure_error({"error_kind": "unknown_model"}, "None", None) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + try: + _ = model in {} or "/" in model # pyright: ignore[reportOperatorIssue] # raising is the point + except TypeError as error: + return error + return TypeError(f"{model!r} is not a model group") + + def _level_failed(self, event: Mapping[str, object]) -> None: + """A level raises the error a pick raised (``level_failed``).""" + level: Final = event.get("level") + at_level: Final = level if isinstance(level, int) else 0 + picked: Final = ( + self._picked_error(event) + if event.get("handling") is True + else self._level_picks.pop(at_level, None) or self._picked_error(event) + ) + if event.get("handling") is True: + # Raised by the level's fallback handling, while the level's error was handled. + picked.__context__ = self._level_errors.get(at_level, self._last_error) + self._level_errors[at_level] = picked + if at_level == self._base_level: + self._explains_failure = False + return + stamp: Final = self._pending_stamps.pop(at_level, None) + if stamp is not None and type(picked) in self._core._host.dynamic().exception_types: # pyright: ignore[reportPrivateUsage] # the driver serves its own core + # The exhausted retry loop raised this pick's error: stamped as `retries_exhausted` stamps. + picked.max_retries, picked.num_retries = stamp # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes + self._chain_to_hop_first(picked, at_level) + self._level_errors[at_level] = picked + + def _carry_superseded_usage(self) -> None: + """``carry_superseded_attempt_usage``, before each same-group retry and each fallback hop.""" + from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage + + carry_superseded_attempt_usage(self._request) + + def _order_chain_failed(self, event: Mapping[str, object]) -> None: + """The order path of ``async_function_with_fallbacks_common_utils`` raised: the chain's last + hop's error, which the hop's level ended on, unless the level keeps its own.""" + level: Final = event.get("level") + at: Final = level if isinstance(level, int) else 0 + last_hop: Final = self._level_errors.get(at + 1) + if event.get("last_hop") is True and last_hop is not None: + self._level_errors[at] = last_hop + if at == self._base_level: + self._explains_failure = False + + def _logged_off_the_loop(self) -> bool: + """Whether litellm logged the attempt's failure on a thread with no running loop. An async + call's mocked failure raises in ``completion``, which ``acompletion`` runs in an executor + thread; a sync stream's first chunk logs in a ``threading.Thread``.""" + from litellm.utils import should_run_mock_completion + + if self._sync: + return self._reading_first_chunk + kwargs: Final[Mapping[str, object]] = self._latest_call[1].kwargs if self._latest_call is not None else {} + return self._plan.kind in ("chat", "text") and should_run_mock_completion( + kwargs.get("mock_response"), kwargs.get("mock_tool_calls"), kwargs.get("mock_timeout") + ) + + def _cooldown_set(self, event: Mapping[str, object]) -> None: + """An attempt's failure cooled its deployment down: ``set_cooldown_deployments`` as the failure + callback runs it. Outside a running loop it raises, which ``CustomLogger.log_event`` swallows; + a sync walk with a private loop has one (``cooldown_event_on_own_loop``).""" + deployment_id: Final = event.get("deployment_id") + seconds: Final = event.get("cooldown_time") + if self._logged_off_the_loop() or not isinstance(deployment_id, str): + return + fire: Final = self._core.cooldown_event_on_own_loop if self._plan.on_its_own_loop else self._core.cooldown_event + with contextlib.suppress(RuntimeError): + fire( + deployment_id, + getattr(self._last_error, "status_code", ""), + self._python_cooldown_time(seconds) if isinstance(seconds, float) else None, + ) + + def _python_cooldown_time(self, seconds: float) -> float: + """``_time_to_cooldown`` of ``deployment_callback_on_failure`` for the attempt that failed.""" + deployment: Final[Mapping[str, object]] = self._latest_call[0] if self._latest_call is not None else {} + configured: Final = first_cooldown( + as_str_mapping(deployment.get("model_info")), + self._request, + as_str_mapping(deployment.get("litellm_params")), + ) + return self._core.python_cooldown(seconds, configured, self._last_error) + + def _selected(self, event: Mapping[str, object]) -> None: + """``_acompletion``'s service event for the pick it awaited, and its ``_track_deployment_metrics``.""" + seconds: Final = event.get("pick_seconds") + if self._plan.kind == "chat" and not self._sync and isinstance(seconds, float): + end_time: Final = unix_time() + self._core.service_logged("async_get_available_deployment", end_time - seconds, end_time, self._request) + self._core.track_deployment_pick(event.get("deployment_id"), self._request) + + def _fallbacks_failed(self, event: Mapping[str, object]) -> None: + """The ``except`` around the regular fallbacks of ``async_function_with_fallbacks_common_utils``: + it logs what the ``try`` raised, the last hop's error or the level's own. A level with no + target raises its own error into it, except where no fallback list was set to search.""" + import litellm.exceptions as litellm_exceptions + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + level: Final = event.get("level") + at: Final = level if isinstance(level, int) else 0 + own: Final = self._level_errors.get(at, self._last_error) + request: Final = self._request + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if event.get("chain") is not True: + policy_list: Final = request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) + if not isinstance(own, litellm_exceptions.ContentPolicyViolationError) or policy_list is None: + if request.get("fallbacks", walked["fallbacks"]) is None: + return + self._explain_no_fallback_group(own, at) + raised: Final = self._level_errors.get(at + 1, own) if event.get("last_hop") is True else own + self.outcomes.failed_fallbacks( + RoutingContext(self._core), # pyright: ignore[reportArgumentType] # the read uses registry() and health() + truncate_fallback_error_detail(redact_string(str(raised))), + get_parent_otel_span_from_kwargs(dict(request)), + ) + + def _explain_no_fallback_group(self, primary: BaseException | None, at: int) -> None: + """The primary level names the missing fallback group on its error before raising it into + the ``except``, so the log reads it and no outcome message follows.""" + if ( + at == self._base_level == 0 + and self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # the driver serves its own core + and primary is not None + and hasattr(primary, "message") + ): + primary.message += self._explanation(primary, str(self._request.get("model"))) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above # rebind-ok: the message is what this explains + self._explains_failure = False + + def take_events(self, events: object) -> None: # noqa: C901 # one branch per router-core event + for raw_event in _as_list(events): + event = as_str_mapping(raw_event) + if event is None: + continue + if event.get("kind") == "retry": + self._carry_superseded_usage() + continue + if event.get("kind") == "cooldown_set": + self._cooldown_set(event) + continue + if event.get("kind") == "selected": + self._selected(event) + continue + if event.get("kind") == "retries_opened": + self._walk.retries_opened(self._stamped_budget(event)) + continue + if event.get("kind") == "retry_budget": + self._walk.retry_budget(event) + continue + if event.get("kind") == "pick_failed": + self._pick_failed(event) + continue + if event.get("kind") == "order_chain_failed": + self._order_chain_failed(event) + continue + if event.get("kind") == "mock_failed": + resolved = self._core._resolved # pyright: ignore[reportPrivateUsage] # the driver serves its own core + mocked = mock_error( + str(event.get("mock")), + str(event.get("model")), + event.get("num_retries"), + self._request, + { + "fallbacks": resolved.fallbacks, + "context_window_fallbacks": resolved.context_window_fallbacks, + "content_policy_fallbacks": resolved.content_policy_fallbacks, + }, + ) + level = event.get("level") + self._last_error = mocked + self._level_errors[level if isinstance(level, int) else 0] = mocked + continue + if event.get("kind") == "retry_policy_raised": + self._policy_raised(event) + continue + if event.get("kind") == "retry_count_raised": + self._count_raised(event) + continue + if event.get("kind") == "level_failed": + self._level_failed(event) + continue + if event.get("kind") == "fallbacks_failed": + self._fallbacks_failed(event) + continue + if event.get("kind") == "fallback": + hop_to = event.get("level") + self._hop_first.pop(hop_to if isinstance(hop_to, int) else 0, None) + self._walk.hop(event) + self._carry_superseded_usage() + hop_level = event.get("level") + self._level = hop_level if isinstance(hop_level, int) else self._level + if "unroutable" in event: + self._level_errors[self._level] = self._unroutable_error(event["unroutable"]) + continue + if event.get("kind") == "blocked": + level = event.get("level") + blocked = self._blocked_error(event) + self._stamp_blocked(blocked) + self._last_error = blocked + self._level_errors[level if isinstance(level, int) else 0] = blocked + continue + if event.get("kind") == "falling_back": + level = event.get("level") + at = level if isinstance(level, int) else 0 + self._explains_failure = self._explains_failure or at == self._base_level + self._hint_level_failure(self._level_errors.get(at, self._last_error), str(event.get("model"))) + continue + if event.get("kind") == "fallback_done": + level = event.get("level") + at = level if isinstance(level, int) else 0 + self._level = at + self.outcomes.done( + event, + self._level_errors.get(at, self._last_error), + self._level_errors.get(at + 1), + self._walk.hop_kwargs(event), + ) + if event.get("succeeded") is not True: + self._core.trigger_fallback_cooldown( + self._request, + self._level_errors.get(at + 1), + self._plan.kind in _GENERIC_API_KINDS, + own_loop=self._plan.on_its_own_loop, + ) + continue + if event.get("kind") == "retry_logged": + logged = self._logged_error(event) + self._walk.log_retry(event, logged) + logged_level = event.get("level") + if event.get("attempted_retries") == 0 and logged is not None: + self._hop_first[logged_level if isinstance(logged_level, int) else 0] = logged + continue + if event.get("kind") != "retries_exhausted": + continue + if event.get("picked") is True: + # The level raises the last retry's pick error, which its `level_failed` names. + exhausted_level = event.get("level") + self._pending_stamps[exhausted_level if isinstance(exhausted_level, int) else 0] = ( + event.get("max"), + event.get("attempted"), + ) + continue + error: BaseException | None = self._last_error + # `async_function_with_retries` stamps the error its exhausted retry loop raises. + if error is not None and type(error) in self._core._host.dynamic().exception_types: # pyright: ignore[reportPrivateUsage] # the driver serves its own core + error.max_retries = event.get("max") # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes + error.num_retries = event.get("attempted") # pyright: ignore[reportAttributeAccessIssue] # same + + def _call(self, step: Mapping[str, object]) -> "tuple[Mapping[str, object], AttemptCall]": + name: Final = step.get("model") + requested: Final = str(name) if name is not None else str(self._request.get("model")) + deployment: Final = ( + # A default deployment whose id is not a string is called as Python copies it, id and all. + as_str_mapping(default_deployment_copy(self._core._resolved.default_deployment, requested)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if step.get("deployment_id") == DEFAULT_DEPLOYMENT_COPIED_ID + else for_requested_model(typed_deployment(as_str_mapping(step.get("deployment")) or {}), requested) + ) + self._attempts += 1 + self._attempt_started = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline's api_call_start_time + level: Final = step.get("fallback_level") + # The request params the walk's dict targets merged in (`kwargs.update(target)`). + params: Final = as_str_mapping(step.get("params")) + call: Final = attempt_call( + self._core._port, # pyright: ignore[reportPrivateUsage] # the driver serves its own core + self._plan, + self._request if params is None else {**self._request, **params}, + deployment, + group=requested, + requested=requested, + fallback_level=level if isinstance(level, int) else 0, + ) + self._strip_undecryptable_reasoning(step, deployment, call.kwargs) + shaped: Final = shape_attempt_kwargs(call.kwargs, deployment) + # The walk's metadata for the attempt's level (retries, fallback hops) and its hop kwargs. + call.kwargs = self._walk.stamp(step, shaped) + field: Final = self._plan.metadata_field + if isinstance(call.served, dict) and field in call.kwargs: + call.served[field] = call.kwargs[field] + self._served = (requested, call.served) + self._latest_call = (deployment, call) + if call.refused is None: + self._core._endpoints.counters.started(self._plan, call) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return deployment, call + + def _strip_undecryptable_reasoning( + self, step: Mapping[str, object], deployment: Mapping[str, object], kwargs: Mapping[str, object] + ) -> None: + """``_update_kwargs_with_deployment`` on a fallback hop (``_is_fallback_hop``): the encrypted + reasoning the target cannot decrypt is dropped; reasoning with no origin marker came from + the deployment that just failed.""" + depth: Final = step.get("fallback_depth") + if not isinstance(depth, int) or depth <= 0: + return + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + RoutingContext(self._core), # pyright: ignore[reportArgumentType] # the check reads registry() + kwargs.get("input"), + kwargs.get("messages"), + (deployment,), + unmarked_origin=self._walk.deployment_that_just_failed(step) or self._cut_deployment, + ) + + @property + def response(self) -> object: + return self._last_response + + @property + def last_error(self) -> BaseException | None: + return self._last_error + + def _stream_attempt( + self, step: Mapping[str, object], kwargs: Mapping[str, object], slot: "contextlib.ExitStack | None" = None + ) -> StreamAttempt: + def number(key: str) -> int: + value: Final = step.get(key) + return value if isinstance(value, int) else 0 + + order: Final = step.get("target_order") + deployment_id: Final = step.get("deployment_id") + return StreamAttempt( + core=self._core, + op=self._op, + request=self._request, + kwargs=kwargs, + group=str(step.get("model") or step.get("model_group")), + order=order if isinstance(order, int) else None, + depth=number("fallback_depth"), + level=number("fallback_level"), + deployment_id=deployment_id if isinstance(deployment_id, str) else None, + attempted_retries=number("retry"), + slot=slot, + ) + + def _failed( + self, + step: Mapping[str, object], + deployment: Mapping[str, object], + call: "AttemptCall | None", + error: BaseException, + ) -> Mapping[str, object]: + # What litellm's failure logging read, before the attempt helper's `except` wrote on it. + detail: Final = str(error) + served: Final = call.served if call is not None else {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + stamp_error(self._plan, error, deployment, served) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + self._core._endpoints.attempt_failed(self._plan, error, deployment, call, str(step.get("model"))) # pyright: ignore[reportPrivateUsage] # see above + self._last_error = error + level: Final = step.get("fallback_level") + retry: Final = step.get("retry") + if isinstance(retry, int) and retry > 0: + self._chain_to_hop_first(error, level if isinstance(level, int) else 0) + self._level_errors[level if isinstance(level, int) else 0] = error + logged: Final = logged_call( + call.host_op if call is not None else self._plan.host_op, + served or self._request, # pyright: ignore[reportUnknownArgumentType] # see above + self._attempt_started, + ) + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + return { + **attempt_failure(error), + "detail": detail, + **self._core.failure_exits(error, logged, model_info.get("id")), + # `async_deployment_callback_on_failure` does not run for a sync call, and skips a batch retrieve. + "uncounted_request": self._plan.sync or is_batch_retrieve_call_type(logged.get("call_type")), + } + + def _chain_to_hop_first(self, error: BaseException, level: int) -> None: + """A retry's failure is raised while the hop's first failure is handled (the retry loop + runs in its ``except`` block), so it chains to it.""" + first: Final = self._hop_first.get(level) + if first is not None and first is not error: + error.__context__ = first # rebind-ok: chained as the retry loop's raise chains it + + def _succeeded( + self, call: "AttemptCall", response: object, host_op: str, stream: "StreamAttempt | None" = None + ) -> Mapping[str, object]: + self._last_response = response + tokens: Final = total_tokens(response) + # A wrapped stream reports its success or failure when it ends; the attempt counts the + # request and the tokens read so far, which the end's report leaves out. + # A realtime attempt counts its session's success itself (`attempt`), when it ended cleanly. + deferred: Final = (stream is not None and stream.reports_outcome) or reports_own_success(call) + if stream is not None and deferred: + stream.counted_tokens = tokens # rebind-ok: what the end's report leaves out + return { + "ok": True, + "total_tokens": tokens, + "blocked": self._blocked(response, host_op), + "deferred": deferred, + "tracks_usage": self._tracks_usage(call, response, deferred), + } + + def _tracks_usage(self, call: "AttemptCall", response: object, deferred: bool) -> bool: + """Whether the attempt counts against its deployment's usage (``_deployment_tracks_usage``). A + ``None`` response counts nothing (``increment_deployment_usage_for_response``), and an attempt + that reports no success then is a realtime session, which logs none to count it either.""" + if deferred and response is None: + return False + model: Final = call.kwargs.get("model") + return self._core.deployment_tracks_usage( + (as_str_mapping(call.kwargs.get("model_info")) or {}).get("id"), + model if isinstance(model, str) else call.group, + ) + + async def _refused( + self, + step: Mapping[str, object], + deployment: Mapping[str, object], + call: "AttemptCall", + error: BaseException, + ) -> Mapping[str, object]: + """A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``): + a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure + is recorded by nothing else.""" + import litellm.exceptions as litellm_exceptions + + core: Final = self._core + if isinstance(error, litellm_exceptions.RateLimitError): + deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id") + if isinstance(deployment_id, str): + failure: Final = {**attempt_failure(error), "caller_scoped": core.per_user_deployment(deployment_id)} + report: Final = core._handle.report_failure(deployment_id, failure, uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + done: Final = await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above + _ = core.cooled_down(done, deployment_id, error.status_code) + return {**self._failed(step, deployment, call, error), "recorded": True} + + def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]: + """The request kwargs the pick sees: the request with its level's retry count.""" + count: Final = self._walk.request_retry_count(step) + if count is None: + return self._request + field: Final = self._plan.metadata_field + metadata: Final = as_str_mapping(self._request.get(field)) or {} + return {**self._request, field: {**metadata, "request_retry_count": count}} + + def _with_deployment(self, step: Mapping[str, object], resolved: object) -> Mapping[str, object]: + return step if resolved is step.get("deployment") else {**step, "deployment": resolved} + + async def _resolve_pods(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``async_resolve_pods_after``: a picked deployment with Kubernetes pod discovery on is + sent to one of its service's pods.""" + if self._plan.kind in _UNPICKED_KINDS or step.get("deployment") is None: + return step + discovery: Final = self._core._pick_discovery(sync=False) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if discovery is None: + return step + resolved: Final[object] = await discovery.async_resolve_deployment( + step.get("deployment"), self._pick_kwargs(step) + ) + return self._with_deployment(step, resolved) + + def _resolve_pods_sync(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``resolve_pods_after``, on the sync pick.""" + if self._plan.kind in _UNPICKED_KINDS or step.get("deployment") is None: + return step + discovery: Final = self._core._pick_discovery(sync=True) # pyright: ignore[reportPrivateUsage] # see _resolve_pods + if discovery is None: + return step + return self._with_deployment( + step, discovery.resolve_deployment(step.get("deployment"), self._pick_kwargs(step)) + ) + + def _default_copy(self, step: Mapping[str, object]) -> Mapping[str, object]: + """The default deployment as Python copied it for the call that fails on it: the error is + stamped from this copy, never from the id router-core serves it under.""" + copied: Final = default_deployment_copy(self._core._resolved.default_deployment, str(step.get("model"))) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return as_str_mapping(copied) or {} + + def _default_call_error(self, step: Mapping[str, object]) -> BaseException | None: + """The error a call to a default deployment every call fails to raises (its first + reads of the copy, before any endpoint runs): nothing records it against a deployment.""" + if step.get("deployment_id") != DEFAULT_DEPLOYMENT_FAILING_ID: + return None + return default_deployment_call_error(self._core._resolved.default_deployment) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + + async def _passthrough_attempt(self, step: Mapping[str, object]) -> Mapping[str, object]: + """``_ageneric_api_call_with_fallbacks_helper`` with ``passthrough_on_no_deployment``: the pick + failed, so the handler is called under the attempt's name. Nothing was picked, so nothing is + stamped on a failure or recorded against a deployment.""" + passthrough: Final = self._passthrough + if passthrough is None: + raise RuntimeError("router-core asked for an unrouted attempt the host did not offer") + name: Final = str(step.get("model")) + params: Final = as_str_mapping(step.get("params")) + self._attempts += 1 + self._attempt_started = datetime.now() # noqa: DTZ005 # see _call + try: + answered: Final = await passthrough(self._request if params is None else {**self._request, **params}, name) + except Exception as error: # noqa: BLE001 # handed to the Rust call loop like any attempt failure + return {**self._failed(step, {}, None, error), "recorded": True} + self._last_response = answered + self._served = (name, self._request) + return {"ok": True, "total_tokens": total_tokens(answered)} + + async def attempt(self, step: Mapping[str, object]) -> Mapping[str, object]: + if self._passthrough is not None and not step.get("deployment_id"): + return await self._passthrough_attempt(step) + step = await self._resolve_pods(step) # rebind-ok: the picked deployment, resolved to a pod + deployment: Mapping[str, object] = as_str_mapping(step.get("deployment")) or {} # rebind-ok: the picked one + call: AttemptCall | None = None # rebind-ok: built inside the try + # Whether a failure raised now runs the failure callbacks: none raised before the litellm + # call or after it answered does, nor any but the first on a shared logging object. + logs_failure: bool = False # rebind-ok: set around the litellm call + reading_stream: StreamAttempt | None = None # rebind-ok: set while the first chunk is read + refused_call: Final = self._default_call_error(step) + if refused_call is not None: + return {**self._failed(step, self._default_copy(step), None, refused_call), "recorded": True} + try: + deployment, call = self._call(step) # rebind-ok: rebound as the Python router rebinds it + if call.refused is not None: + await log_refused_call(call, call.refused) + raise call.refused + host: Final = self._core._host # pyright: ignore[reportPrivateUsage] # the driver serves its own core + try: + compacted: Final = await compacted_attempt(self._core._port, self._plan, deployment, call) # pyright: ignore[reportPrivateUsage] # see above + except Exception as refused: # noqa: BLE001 # raised before the provider call, so no cooldown reads it + return {**self._failed(step, deployment, call, refused), "recorded": True} + # The deployment's max_parallel_requests slot; an open chat stream takes it along. + port = self._core._port # pyright: ignore[reportPrivateUsage] # the driver serves its own core # rebind-ok: the same port on every attempt + with enter_deployment_slot(port, self._plan, deployment, call.kwargs) as slot: + # A search tool is no deployment: nothing to check; Python's `_arerank` and + # `amoderation` run no routing pre-call checks either. + if self._plan.kind not in _NO_ROUTING_PRE_CALL_CHECKS: + try: + await routing_pre_call_checks( + self._core._checks.routing_callbacks(), # pyright: ignore[reportPrivateUsage] # see above + as_str_mapping(step.get("deployment")) or {}, + ) + except Exception as refused: # noqa: BLE001 # handed to the Rust call loop as the attempt's failure + # Only `_acompletion` hands the check its logging object. + if self._plan.kind == "chat": + _log_refused_pick(self._request, refused) + return await self._refused(step, deployment, call, refused) + first_to_log: Final = not failure_already_logged(call.kwargs, "async_failure") + logs_failure = first_to_log # rebind-ok: whether a failure here is this attempt's to log + pending: Final[object] = ( + self._core._endpoints.create_file_everywhere( # pyright: ignore[reportPrivateUsage] # see above + _with_target_order(call.kwargs, step.get("target_order")), + call.model_name, + self._request.get("specific_deployment"), + ) + if self._plan.kind == "create_file" + else host.invoke(call.host_op)(**compacted) + ) + # The generic helper awaits whatever the handler returns, as the Python router does. + awaited: Final = self._plan.kind == "generic" or inspect.isawaitable(pending) + answered: Final[object] = await pending if awaited else pending # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType] # see above + logs_failure = False # rebind-ok: see above + self._core._endpoints.check_response(self._plan, answered, call) # pyright: ignore[reportPrivateUsage, reportUnknownArgumentType] # see above + stream: Final = self._stream_attempt(step, call.kwargs, slot) + logs_failure = first_to_log # rebind-ok: the stream's own failure logging, for its first chunk + reading_stream = stream # rebind-ok: None until the attempt opens a stream + response: Final = await open_attempt_stream(stream, answered) # pyright: ignore[reportUnknownArgumentType] # see above + logs_failure = False # rebind-ok: see above + # `self.success_calls[model_name] += 1`, inside the try: a counter replaced with a + # plain dict fails the attempt. + self._core._endpoints.counters.succeeded(self._plan, call) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # every attempt failure is classified and handed to the Rust call loop + failed: Final = self._failed(step, deployment, call, error) + if reading_stream is None or not logs_failure: + return {**failed, "recorded": not logs_failure} + # `_dispatch_stream_failure`: the stream logs its failure in a task of its own. + reading_stream.report_first_chunk_failure(error) + return {**failed, "recorded": True} + served: Final = self._succeeded(call, response, call.host_op, stream) + # The session re-pin the Python core's success callback runs; a wrapped stream runs it when it ends. + if not served.get("deferred") and self._core.outcome_callbacks(call.kwargs, sync=False).get("success", True): + await self._core.repin_after_success(stream.logged(), response) + session_deployment: Final = clean_realtime_session_deployment(call) + if session_deployment is not None: + await self._core.count_realtime_session_success(session_deployment) + return served + + def attempt_sync(self, step: Mapping[str, object]) -> Mapping[str, object]: + step = self._resolve_pods_sync(step) # rebind-ok: see attempt + deployment: Mapping[str, object] = as_str_mapping(step.get("deployment")) or {} # rebind-ok: see attempt + call: AttemptCall | None = None # rebind-ok: see attempt + logs_failure: bool = False # rebind-ok: see attempt + self._reading_first_chunk = False + refused_call: Final = self._default_call_error(step) + if refused_call is not None: + return {**self._failed(step, self._default_copy(step), None, refused_call), "recorded": True} + try: + deployment, call = self._call(step) # rebind-ok: rebound as the Python router rebinds it + if call.refused is not None: + log_refused_call_sync(call, call.refused) + raise call.refused + try: + self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above + except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure + # The sync path records no outcome for a refused pre-call check: no cooldown. + return {**self._failed(step, deployment, call, refused), "recorded": True} + first_to_log: Final = not failure_already_logged(call.kwargs, "sync_failure") + logs_failure = first_to_log # rebind-ok: see attempt + answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint + logs_failure = False # rebind-ok: see attempt + if inspect.isawaitable(answered): + raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous") + self._core._endpoints.check_response(self._plan, answered, call) # pyright: ignore[reportPrivateUsage] # see above + stream: Final = self._stream_attempt(step, call.kwargs) + logs_failure = first_to_log # rebind-ok: see attempt + self._reading_first_chunk = True + response: Final = open_attempt_stream_sync(stream, answered) + self._reading_first_chunk = False + logs_failure = False # rebind-ok: see attempt + self._core._endpoints.counters.succeeded(self._plan, call) # pyright: ignore[reportPrivateUsage] # see attempt + except _NotSynchronous: + raise + except Exception as error: # noqa: BLE001 # see attempt + return {**self._failed(step, deployment, call, error), "recorded": not logs_failure} + return self._succeeded(call, response, "", stream) + + async def select(self, step: Mapping[str, object]) -> object: + request: Final = self._request + messages, input = self._plan.pick_input(request) + if step.get("stage") == "callback_filters": + answer, error = await self._core._answer_callback_filters(step, request, messages) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if isinstance(error, Exception): + # The try failed with the filter's error: the level raises it if nothing else serves. + _ = self._pick_raised(error, logged=True) + return answer + try: + return await self._core._answer_select(step, request, messages, input) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + except Exception as error: # noqa: BLE001 # handed to the Rust call loop as the try's failure + return self._pick_raised(error, logged=True) + + def select_sync(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return self._core._answer_select_sync(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=False) + + async def pre_route(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return await self._core._pre_route(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=True) + + def pre_route_sync(self, step: Mapping[str, object]) -> object: + request: Final = self._request + try: + return self._core._custom_pick_sync(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + except Exception as error: # noqa: BLE001 # see select + return self._pick_raised(error, logged=False) + + async def allow_fallback(self, target: str) -> object: + """`_is_fallback_target_authorized`, then `_is_fallback_target_within_budget`. A check that raises + ends the chain outside any hop's ``try``, as a chain's own failure: the level raises what it + would have (``async_function_with_fallbacks_common_utils`` catches it), and its note or an + order path reads the error as the next level's.""" + from litellm.router_utils.fallback_event_handlers import get_pre_routing_selection + from litellm.types.router import FallbackSkipReason + + ctx: Final = RoutingContext(self._core) # pyright: ignore[reportArgumentType] # the checks read registry() and settings() + request: Final = dict(self._request) + try: + if target != get_pre_routing_selection(request) and not await ctx.fallback_access_allowed( + model=target, request_kwargs=request + ): + self._walk.targets().record_skip(target, FallbackSkipReason.NOT_AUTHORIZED) + return False + if not await ctx.fallback_budget_allowed(model=target, request_kwargs=request): + self._walk.targets().record_skip(target, FallbackSkipReason.OVER_BUDGET) + return False + except Exception as error: # noqa: BLE001 # handed to the Rust call loop, which ends the chain + self._level_errors[self._level + 1] = error + return {**attempt_failure(error), "raised": True} + return True + + def refuse_fallbacks(self, step: Mapping[str, object]) -> bool: + """``run_async_fallback``'s check of a chain's dict targets: a refused chain ends the walk + with the check's error, which the first hop's level would have raised.""" + refused: Final = rejected_fallback_target(_as_list(step.get("targets"))) + if refused is None: + return False + level: Final = step.get("level") + self._level_errors[(level if isinstance(level, int) else 0) + 1] = refused + return True + + def _head_response(self) -> object | None: + """``set_response_headers`` up to its usage read: the deployment id, the model group the + response was served as and the routers' decisions.""" + response: Final = prepare_response_for_header_attachment(self._last_response) + if response is None: + return None + name, kwargs = self._served + additional: Final = ensure_response_additional_headers(response) + apply_response_model_id(response, find_deployment_metadata(kwargs)) + additional["x-litellm-model-group"] = name + apply_quality_router_decision_headers(additional, kwargs) + additional.update(complexity_router_decision_headers(kwargs)) + return response + + def _count_response(self, response: object, step: Mapping[str, object]) -> object: + """What the wrappers above `make_call` stamp: ``async_function_with_retries``'s retry count, + then the fallback count of ``async_function_with_fallbacks`` and ``run_async_fallback``.""" + retries: Final = step.get("attempted_retries") + if isinstance(retries, int): + max_retries: Final = step.get("max_retries") + response = add_retry_headers_to_response( # rebind-ok: gains the counts + response=response, + attempted_retries=retries, + max_retries=max_retries if isinstance(max_retries, int) and retries > 0 else None, + ) + fallbacks: Final = step.get("attempted_fallbacks") + attempted: Final = fallbacks if isinstance(fallbacks, int) else 0 + return add_fallback_headers_to_response( + response=self.outcomes.add_headers(response, attempted), attempted_fallbacks=attempted + ) + + def log_outcomes_sync(self) -> None: + if pending := self.outcomes.take_pending(): + from litellm.litellm_core_utils import asyncify + + asyncify.run_async_function(await_all, pending) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + + def finish(self, step: Mapping[str, object]) -> object: + """The response with the router's headers, in the order ``make_call`` and the wrappers above it + stamp them: a usage read that raises leaves the counts off.""" + if not self._plan.sets_headers: + self.log_outcomes_sync() + return self._last_response + response: Final = self._head_response() + if response is None: + self.log_outcomes_sync() + return self._last_response + name: Final = self._served[0] + if name is not None: + core: Final = self._core + remaining: Final = as_str_mapping( + core.run_routing_cache_sync(lambda: core.health().usage(name, kind="remaining")) + ) + if remaining is not None: + apply_remaining_usage_headers(ensure_response_additional_headers(response), dict(remaining)) + counted: Final = self._count_response(response, step) + self.log_outcomes_sync() + return counted + + async def finish_async(self, step: Mapping[str, object]) -> object: + """``finish``, with the usage read on the call's own loop.""" + if not self._plan.sets_headers: + await await_all(self.outcomes.take_pending()) + return self._last_response + response: Final = self._head_response() + if response is None: + await await_all(self.outcomes.take_pending()) + return self._last_response + name: Final = self._served[0] + if name is not None: + remaining: Final = await self._core.health().usage(name, kind="remaining") + if isinstance(remaining, Mapping): + apply_remaining_usage_headers(ensure_response_additional_headers(response), dict(remaining)) + counted: Final = self._count_response(response, step) + await await_all(self.outcomes.take_pending()) + return counted + + def failed(self, step: Mapping[str, object]) -> BaseException: + model: Final = str(self._request.get("model")) + primary: Final = self._level_errors.get(self._base_level) + if primary is None: + return self._core._failure_error(step, model, self._last_error, request=self._request) # pyright: ignore[reportPrivateUsage] # see above + if ( + self._base_level == 0 + and self._explains_failure + and self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # see above + and hasattr(primary, "message") + ): + primary.message += self._explanation(primary, model) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + return primary + + def _hint_level_failure(self, error: BaseException | None, model: str) -> None: + """A level that reaches its fallback handling on a context-window or content-policy error + with no list of its own appends the hint to set one, before any hop runs, as every level of + the Python walk does: a stream's failure event, still queued then, reads the hint.""" + from litellm.router_utils.fallback_event_handlers import check_non_standard_fallback_format + + if ( + error is None + or not hasattr(error, "message") + or not self._core._host.dynamic().expose_router_debug_in_errors # pyright: ignore[reportPrivateUsage] # the driver serves its own core + ): + return + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # see above + fallbacks: Final = self._request.get("fallbacks", walked["fallbacks"]) + if check_non_standard_fallback_format(fallbacks=_as_list(fallbacks)): + return + hint: Final = self._kind_hint(error, model, fallbacks) + if hint: + error.message += hint # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above # rebind-ok: the router appends to the error it raises + + def _kind_hint(self, error: BaseException, model: str, fallbacks: object) -> str: + """The hint a context-window or content-policy error gets when its own list is unset.""" + import litellm.exceptions as litellm_exceptions + + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + if isinstance(error, litellm_exceptions.ContextWindowExceededError): + if self._request.get("context_window_fallbacks", walked["context_window_fallbacks"]) is not None: + return "" + return ( + f"\nmodel={model}. context_window_fallbacks=None. fallbacks={fallbacks}.\n\n" + "Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" + ) + if isinstance(error, litellm_exceptions.ContentPolicyViolationError): + if self._request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) is not None: + return "" + return "\n" + format_content_policy_fallback_hint(model, None, fallbacks) # pyright: ignore[reportArgumentType] # the list as configured + return "" + + def _explanation(self, primary: BaseException, model: str) -> str: + """What ``async_function_with_fallbacks_common_utils`` appends to the error it re-raises + under ``expose_router_debug_in_errors``, from the resolved settings and what the call did.""" + import litellm.exceptions as litellm_exceptions + from litellm.router_utils.fallback_event_handlers import ( + check_non_standard_fallback_format, + fallback_lookup_groups, + get_fallback_model_group_for_lookup_groups, + ) + + walked: Final = self._core._walked_fallbacks # pyright: ignore[reportPrivateUsage] # the driver serves its own core + request: Final = self._request + # The request's own lists replace the router's (`kwargs.get("fallbacks", router)`). + fallbacks: Final = request.get("fallbacks", walked["fallbacks"]) + window_list: Final = request.get("context_window_fallbacks", walked["context_window_fallbacks"]) + policy_list: Final = request.get("content_policy_fallbacks", walked["content_policy_fallbacks"]) + lookup: Final = fallback_lookup_groups(request, model) + + def group_of(source: object) -> tuple[Sequence[object] | None, int | None]: + return get_fallback_model_group_for_lookup_groups(_as_list(source), lookup) # pyright: ignore[reportArgumentType] # the fallbacks config shape + + def outcome(group: Sequence[object] | None) -> str: + return format_fallback_outcome_message( + model, group, self._fallback_detail(primary), skipped=self._walk.targets().skipped + ) + + # `context_window_fallback_model_group`: only a chain the group matched takes precedence. + window_group: Final = ( + group_of(window_list)[0] + if isinstance(primary, litellm_exceptions.ContextWindowExceededError) and window_list is not None + else None + ) + if window_group is None and check_non_standard_fallback_format(fallbacks=_as_list(fallbacks)): + return outcome(_as_list(fallbacks)) + if window_group is not None: + return outcome(window_group) + # A content-policy list decides on its own: one with no match raises into the `except`. + if isinstance(primary, litellm_exceptions.ContentPolicyViolationError) and policy_list is not None: + return outcome(group_of(policy_list)[0]) + if fallbacks is None or not lookup: + return outcome(None) + found, generic = group_of(fallbacks) + group: Final[Sequence[object] | None] = ( + found if found is not None or generic is None else _as_list(_as_list(fallbacks)[generic]["*"]) # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType, reportUnknownArgumentType] # the entry the lookup matched + ) + return format_no_fallback_group_message(lookup, _as_list(fallbacks)) if group is None else outcome(group) # pyright: ignore[reportArgumentType] # the fallbacks config shape + + def _fallback_detail(self, primary: BaseException) -> str: + """``run_async_fallback`` raises its last hop's error, or the primary's when no hop ran.""" + return truncate_fallback_error_detail(redact_string(str(self._level_errors.get(1, primary)))) + + +__all__ = [ + "Resolved", + "RustCore", + "RustCoreUnsupported", + "RustHealth", + "RustRegistry", + "RustSettings", + "attempt_failure", + "failure_kind", + "total_tokens", + "unsupported_arguments", +] diff --git a/litellm/router_rust/invoke/README.md b/litellm/router_rust/invoke/README.md new file mode 100644 index 00000000000..d8562f99359 --- /dev/null +++ b/litellm/router_rust/invoke/README.md @@ -0,0 +1,13 @@ +# invoke + +The host side of `Host::invoke`: running one attempt against the one deployment router-core picked. Nothing here selects, retries, falls back or records cooldowns. + +- `shaping`: the kwargs an attempt sends to its deployment +- `endpoints`: what each public router method does before, around and after the routed call +- `streaming`: stream wrappers, so a stream that fails before its first token fails the attempt +- `slots`: a deployment's `max_parallel_requests` +- `walk`: the retry and fallback metadata stamped on each attempt +- `prompt_management`: prompt-management groups +- `websocket`: admission for each turn of a Responses websocket + +A Rust host replaces this with native endpoint calls. diff --git a/litellm/router_rust/invoke/__init__.py b/litellm/router_rust/invoke/__init__.py new file mode 100644 index 00000000000..faa2c13198d --- /dev/null +++ b/litellm/router_rust/invoke/__init__.py @@ -0,0 +1 @@ +"""The host side of ``Host::invoke``: one router attempt against one deployment.""" diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py new file mode 100644 index 00000000000..c1dc5162ccf --- /dev/null +++ b/litellm/router_rust/invoke/endpoints.py @@ -0,0 +1,2454 @@ +""" +The Rust router core's endpoints: what each public router method does before, around and after +the routed call, as the Python router's endpoint methods do it. + +An endpoint is either routed (the Rust core picks the deployment, retries and falls back, and the +host runs each attempt) or served directly (the Python router calls litellm's function without +routing: no model group, an id that names no deployment, a fan-out over every deployment). Every +call out goes through the host: ``host.invoke(op)``, where an op is a litellm function name or a +dotted path to one (``vector_stores.main.asearch``), looked up when it is called. +""" + +import asyncio +import copy +import dataclasses +import threading +import time +import uuid +from collections import defaultdict +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_fallback_headers_to_response, + add_retry_headers_to_response, +) +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.core_helpers import without +from litellm.litellm_core_utils.routing_common_utils import without_router_only_kwargs +from litellm.responses.retrieve_stream_query import without_retrieve_stream_defaults +from litellm.router_api import RoutingContext +from litellm.router_rust.invoke.shaping import ( + ROUTER_CLIENTS_TARGET, + STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + Shaping, + as_dict, + deployment_params_with_request_reasoning_override, + drop_unsupported_classifier_reasoning_effort, + metadata_bucket, + stamp_deployment_num_retries, + stamp_failed_deployment_id, + stamp_retry_skip_deployment_id, + update_kwargs_with_deployment, +) +from litellm.router_strategy.complexity_router.context_compaction import ( + Surface, + compact_to_fit, + initialize_compaction_state, + surface_for_call, +) +from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.router_api import RouterRegistry + from litellm.router_rust.core import RustCore + from litellm.router_rust.steps.selection import StrategySelectors + from litellm.rust_bridge._native import RouterCoreHandle + from litellm.types.router import Deployment + + +class EndpointCore(Protocol): + """What the endpoints use of the core they serve: its host, Rust handle, settings, routing + cache, registry lookups and call drivers. ``RustCore`` provides it through ``_EndpointPort``; + a Rust host's endpoint layer would have to provide the same list.""" + + @property + def router(self) -> "RustCore": + """The core itself, for the provider handlers and helpers that take a router.""" + ... + + @property + def host(self) -> "PythonHost": ... + + @property + def handle(self) -> "RouterCoreHandle": ... + + @property + def resolved(self) -> "Resolved": ... + + @property + def shaping(self) -> Shaping: ... + + @property + def routing_cache(self) -> "DualCache": ... + + @property + def selectors(self) -> "StrategySelectors": ... + + @property + def endpoints(self) -> "RustEndpoints": ... + + async def drive( + self, + plan: "Plan", + request: Mapping[str, object], + overrides: Mapping[str, object] | None = None, + *, + passthrough: Callable[[Mapping[str, object], str], Awaitable[object]] | None = None, + prepared_already: bool = False, + ) -> object: ... + + def call_sync( + self, plan: "Plan", request: Mapping[str, object], overrides: Mapping[str, object] | None = None + ) -> object: ... + + def routing_callbacks(self) -> Iterable[object]: ... + + def resolve_alias(self, model: str) -> str | None: ... + + def group_deployments(self, model_name: str) -> Sequence[Mapping[str, object]] | None: + """``_core_get_model_list(model_name=...)`` on the host registry.""" + ... + + def routed_name(self, model: object) -> str: + """``RustCore.routed_name``: what a ``model`` that is no string raises, or the name it routes under.""" + ... + + def service_logged( + self, call_type: str, start_time: float, end_time: float, request: Mapping[str, object] + ) -> None: ... + + async def set_response_headers( + self, response: object, model_group: str | None, request_kwargs: Mapping[str, object] + ) -> object: ... + + def configured_deployment(self, model_id: str) -> "Deployment | None": + """``_core_get_deployment``: the deployment as the host registry holds it now.""" + ... + + def registry(self) -> "RouterRegistry": ... + + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the RouterCore signature + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: ... + + async def acompletion(self, *args: object, **kwargs: object) -> object: ... # kwargs-ok: endpoint payload + + +Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it + "chat", + "text", + "embedding", + "image", + "transcription", + "speech", + "rerank", + "adapter", + "generic", + "generic_sync", + "batch", + "cancel_batch", + "vector_store_create", + "moderation", + "create_file", + "search", +] +Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it +CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it + +# The text of the user message each attempt helper hands its pick in place of the request's own +# (`_aimage_generation`, `_aadapter_completion`, `_acreate_batch`, ...). +_PICK_TEXTS: Final[Mapping[str, str]] = MappingProxyType( + { + "image": "prompt", + "transcription": "prompt", + "speech": "prompt", + "adapter": "default text", + "batch": "files-api-fake-text", + "create_file": "files-api-fake-text", + "cancel_batch": "batch-api-fake-text", + "vector_store_create": "vector-store-api-fake-text", + } +) + + +def _user_message(content: object) -> list[dict[str, object]]: # mutable-ok: a fresh list, as each pick builds it + return [{"role": "user", "content": content}] + + +# Request keys the router reads to route the call; the deployment call never receives them. +ROUTING_KEYS: Final = frozenset( + { + "model", + "num_retries", + "disable_fallbacks", + "specific_deployment", + "_router_weights", + "_retry_skipped_deployment_ids", + "_excluded_deployment_ids", + "_target_order", + "original_function", + # The walk's own settings, which `async_function_with_fallbacks` pops before the attempt. + "model_group_retry_policy", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + } +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Plan: + """How one endpoint runs its routed attempts.""" + + endpoint: str + host_op: str + kind: Kind + sync: bool = False + # The bucket `_update_kwargs_before_fallbacks` writes the requested group into; None for the + # endpoints that write `metadata.model_group` alone (no trace id, no alias). + prepare_field: str | None = "metadata" + function_name: str | None = None + splits_stream_timeout: bool = False + stamp: Stamp = "retry_skip" + total_key: CountKey | None = "deployment" + success_key: CountKey | None = "deployment" + fail_key: CountKey | None = "deployment" + # One attempt, no retries and no fallbacks. + single_attempt: bool = False + # The named parameter the endpoint takes beside **kwargs, sent between the deployment params and + # the request (`{**data, name: value, **kwargs}`). + named: tuple[str, ...] = () + + @property + def on_its_own_loop(self) -> bool: + """``function_with_fallbacks`` runs these sync endpoints' whole walk on a private event loop.""" + return self.sync and self.kind in ("chat", "embedding", "image") + + def pick_input(self, request: Mapping[str, object]) -> tuple[object, object]: + """The ``messages`` and ``input`` the endpoint's attempt helper hands its pick, which every + selection stage reads: the pre-call checks run when either is set.""" + match self.kind: + case "chat": + return request.get("messages"), None + case "text": + return _user_message(request.get("prompt")), None + case "embedding": + return None, request.get("input") + case "rerank" | "moderation" | "search": + return None, None + case "generic" | "generic_sync": + return request.get("messages"), request.get("input") + case _: + return _user_message(_PICK_TEXTS[self.kind]), None + + @property + def sets_headers(self) -> bool: + """Whether the response gets the router's headers (the sync generic wrapper adds none).""" + return self.kind != "generic_sync" + + @property + def stamps_failed_deployment(self) -> bool: + """Whether the attempt stamps ``failed_deployment_id`` on its error, which weighted failover + reads: every endpoint but file and batch creation (whose result belongs to the deployment + that served it), moderation's pass-through and search.""" + return self.kind not in ("batch", "cancel_batch", "create_file", "moderation", "search") + + @property + def metadata_field(self) -> str: + """Where `_update_kwargs_with_deployment` writes the deployment it routed to.""" + return get_router_metadata_variable_name(function_name=self.function_name) + + +def _generic(endpoint: str, host_op: str | None = None, *, sync: bool = False) -> Plan: + if sync: + return Plan( + endpoint=endpoint, + host_op=host_op or endpoint, + kind="generic_sync", + sync=True, + prepare_field="litellm_metadata", + function_name="generic_api_call", + splits_stream_timeout=_handler_name(host_op or endpoint) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + stamp="none", + fail_key="group", + single_attempt=True, + ) + return Plan( + endpoint=endpoint, + host_op=host_op or endpoint, + kind="generic", + prepare_field="litellm_metadata", + function_name="_ageneric_api_call_with_fallbacks", + splits_stream_timeout=_handler_name(host_op or endpoint) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + stamp="deployment", + fail_key="group", + ) + + +def _handler_name(host_op: str) -> str: + return host_op.rsplit(".", 1)[-1] + + +def invoked_handler_name(core: EndpointCore, plan: Plan) -> str | None: + """The generic handler's ``__name__`` (`original_generic_function.__name__`): the function the + host serves for the op, which decides the stream-timeout split and the responses socket.""" + function: Final[object] = core.host.invoke(plan.host_op) # pyright: ignore[reportUnknownMemberType] # the core's host + name: Final[object] = getattr(function, "__name__", None) + return name if isinstance(name, str) else None + + +# The endpoints whose attempt helper runs inside `make_call`'s `endpoint_selection`, under the +# name of its `original_function`. +_SELECTION_CALLS: Final[Mapping[str, str]] = MappingProxyType( + { + "acompletion": "_acompletion", + "completion": "_completion", + "atext_completion": "_atext_completion", + } +) + + +def selection_endpoint(core: EndpointCore, plan: Plan) -> str | None: + """The endpoint ``endpoint_selection`` names around the plan's pick. A sync generic call never + reaches ``make_call``, and a generic attempt reads the handler's name and module.""" + call: Final = _SELECTION_CALLS.get(plan.endpoint) + if call is not None: + return endpoint_for_call(call) + if plan.kind != "generic": + return None + function: Final[object] = core.host.invoke(plan.host_op) # pyright: ignore[reportUnknownMemberType] # the core's host + return endpoint_for_call(getattr(function, "__name__", None), getattr(function, "__module__", None)) + + +PLANS: Final[Mapping[str, Plan]] = MappingProxyType( + { + "acompletion": Plan( + "acompletion", "acompletion", "chat", splits_stream_timeout=True, stamp="deployment", named=("messages",) + ), + "completion": Plan( + "completion", + "completion", + "chat", + sync=True, + splits_stream_timeout=True, + stamp="deployment", + total_key=None, + success_key=None, + fail_key=None, + named=("messages",), + ), + "atext_completion": Plan( + "atext_completion", + "atext_completion", + "text", + splits_stream_timeout=True, + fail_key="group", + named=("prompt",), + ), + # `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment + # picked; its helper picks among the router's search tools. + "asearch": Plan( + "asearch", + "search.asearch", + "search", + prepare_field="litellm_metadata", + stamp="none", + total_key=None, + success_key=None, + fail_key=None, + ), + "aembedding": Plan("aembedding", "aembedding", "embedding", named=("input",)), + "embedding": Plan("embedding", "embedding", "embedding", sync=True, named=("input",)), + "aimage_generation": Plan("aimage_generation", "aimage_generation", "image", named=("prompt",)), + "image_generation": Plan( + "image_generation", + "image_generation", + "image", + sync=True, + prepare_field=None, + total_key="empty", + success_key="empty", + fail_key="empty", + named=("prompt",), + ), + "atranscription": Plan( + "atranscription", + "atranscription", + "transcription", + total_key="group", + success_key="group", + fail_key="group", + named=("file",), + ), + "aspeech": Plan( + "aspeech", + "aspeech", + "speech", + total_key="group", + success_key="group", + fail_key="group", + named=("input",), + ), + "arerank": Plan("arerank", "arerank", "rerank"), + "aadapter_completion": Plan( + "aadapter_completion", + "aadapter_completion", + "adapter", + prepare_field=None, + splits_stream_timeout=True, + fail_key="group", + ), + "acreate_batch": Plan( + "acreate_batch", + "acreate_batch", + "batch", + prepare_field="litellm_metadata", + function_name="_acreate_batch", + fail_key="group", + ), + "acancel_batch": Plan( + "acancel_batch", + "acancel_batch", + "cancel_batch", + prepare_field="litellm_metadata", + function_name="_acancel_batch", + fail_key="group", + ), + "acreate_file": Plan( + "acreate_file", + "acreate_file", + "create_file", + function_name="acreate_file", + stamp="none", + total_key=None, + success_key=None, + fail_key="group", + ), + "avector_store_create": Plan( + "avector_store_create", + "vector_stores.acreate", + "vector_store_create", + prepare_field=None, + function_name="avector_store_create", + stamp="none", + fail_key="group", + single_attempt=True, + ), + "amoderation": Plan( + "amoderation", + "amoderation", + "moderation", + stamp="none", + total_key=None, + success_key=None, + fail_key=None, + single_attempt=True, + ), + # The Python router's anthropic messages host op is `anthropic_messages`. + "aanthropic_messages": _generic("aanthropic_messages", "anthropic_messages"), + "anthropic_messages": _generic("anthropic_messages", "anthropic_messages"), + "aresponses": _generic("aresponses"), + "responses": _generic("responses", sync=True), + } +) + +# --- the generic endpoints ------------------------------------------------------------------ + +# endpoint -> host op, for the async generic endpoints routed through +# `_ageneric_api_call_with_fallbacks` when they carry a model. +_HOST_GENERIC: Final[Mapping[str, str]] = MappingProxyType( + { + "afile_delete": "afile_delete", + "afile_content": "afile_content", + "afile_list": "afile_list", + "aget_responses": "aget_responses", + "acancel_responses": "acancel_responses", + "acompact_responses": "acompact_responses", + "adelete_responses": "adelete_responses", + "alist_input_items": "alist_input_items", + "_arealtime": "_arealtime", + "acreate_realtime_client_secret": "acreate_realtime_client_secret", + "arealtime_calls": "arealtime_calls", + "acreate_realtime_transcription_session": "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret": "acreate_realtime_translation_client_secret", + "arealtime_translation_calls": "arealtime_translation_calls", + "_aresponses_websocket": "_aresponses_websocket", + "acreate_fine_tuning_job": "acreate_fine_tuning_job", + "acancel_fine_tuning_job": "acancel_fine_tuning_job", + "alist_fine_tuning_jobs": "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job": "aretrieve_fine_tuning_job", + "aimage_edit": "aimage_edit", + "allm_passthrough_route": "allm_passthrough_route", + "acreate_skill": "acreate_skill", + "alist_skills": "alist_skills", + "aget_skill": "aget_skill", + "adelete_skill": "adelete_skill", + "agenerate_content": "google_genai.agenerate_content", + "agenerate_content_stream": "google_genai.agenerate_content_stream", + "aocr": "ocr.aocr", + "adecisions": "decisions.adecisions", + "asystemone": "systemone.asystemone", + "aextraction": "extraction.aextraction", + **{f"avideo_{name}": f"videos.avideo_{name}" for name in ("generation", "list", "status", "content", "remix")}, + **{ + name: f"containers.{name}" + for name in ("acreate_container", "alist_containers", "aretrieve_container", "adelete_container") + }, + **{ + name: f"containers.endpoint_factory._generated_endpoints.{name}" + for name in ( + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + ) + }, + **{ + f"avector_store_{name}": f"vector_stores.main.a{name}" + for name in ("search", "retrieve", "list", "update", "delete") + }, + **{ + f"avector_store_file_{name}": f"vector_store_files.main.a{target}" + for name, target in ( + ("create", "create"), + ("list", "list"), + ("retrieve", "retrieve"), + ("content", "retrieve_content"), + ("update", "update"), + ("delete", "delete"), + ) + }, + "acreate_interaction": "interactions.acreate", + "aget_interaction": "interactions.aget", + "adelete_interaction": "interactions.adelete", + "acancel_interaction": "interactions.acancel", + } +) + +_SYNC_GENERIC: Final[Mapping[str, str]] = MappingProxyType( + { + "generate_content": "google_genai.generate_content", + "generate_content_stream": "google_genai.generate_content_stream", + "ocr": "ocr.ocr", + # The sync wrapper routes a search tool name as a model group, unlike `asearch`. + "search": "search.search", + "decisions": "decisions.decisions", + "systemone": "systemone.systemone", + "extraction": "extraction.extraction", + **{f"video_{name}": f"videos.video_{name}" for name in ("generation", "list", "status", "content", "remix")}, + **{ + name: f"containers.{name}" + for name in ("create_container", "list_containers", "retrieve_container", "delete_container") + }, + **{ + f"vector_store_{name}": f"vector_stores.main.{name}" + for name in ("search", "create", "retrieve", "list", "update", "delete") + }, + } +) + +_CONTAINER_FILE_ROUTED: Final = frozenset( + { + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + } +) +_VIDEO_DIRECT: Final = frozenset({"avideo_create_character", "avideo_get_character", "avideo_edit", "avideo_extension"}) +_SYNC_VIDEO_UNBRANCHED: Final = frozenset( + {"video_create_character", "video_get_character", "video_edit", "video_extension"} +) +_SYNC_CONTAINER_FILES_UNBRANCHED: Final = frozenset( + { + "upload_container_file", + "list_container_files", + "retrieve_container_file", + "delete_container_file", + "retrieve_container_file_content", + } +) +_UNBRANCHED: Final = frozenset( + { + "aadapter_generate_content", + *_SYNC_VIDEO_UNBRANCHED, + *_SYNC_CONTAINER_FILES_UNBRANCHED, + "get_interaction", + "delete_interaction", + "cancel_interaction", + } +) +_ASSISTANTS: Final = frozenset( + { + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", + } +) +_MANAGED_AGENTS: Final[Mapping[str, str]] = MappingProxyType( + { # comprehension-ok: a flat walk of nested rows + f"{prefix}{name}": f"interactions.agents.{prefix}{target}" + for prefix in ("a", "") + for name, target in ( + ("create_agent", "create"), + ("list_agents", "list"), + ("get_agent", "get"), + ("delete_agent", "delete"), + ("list_agent_versions", "list_versions"), + ) + } +) +_SYNC_VECTOR_STORE_FILES: Final[Mapping[str, str]] = MappingProxyType( + { + f"vector_store_file_{name}": f"vector_store_files.main.{target}" + for name, target in ( + ("create", "create"), + ("list", "list"), + ("retrieve", "retrieve"), + ("content", "retrieve_content"), + ("update", "update"), + ("delete", "delete"), + ) + } +) + + +def plan_for(endpoint: str) -> Plan: + plan: Final = generic_plan(endpoint) + if plan is None: + raise KeyError(endpoint) + return plan + + +def generic_plan(endpoint: str) -> Plan | None: + if endpoint in PLANS: + return PLANS[endpoint] + if endpoint in _HOST_GENERIC: + return _generic(endpoint, _HOST_GENERIC[endpoint]) + if endpoint in _VIDEO_DIRECT: + return _generic(endpoint, f"videos.{endpoint}") + if endpoint in _SYNC_GENERIC: + return _generic(endpoint, _SYNC_GENERIC[endpoint], sync=True) + return None + + +# --- preparing the request ------------------------------------------------------------------- + + +def prepare_request( + plan: Plan, request: Mapping[str, object], model: str, aliases: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the call's kwargs + """What the router adds once before the first attempt (`_update_kwargs_before_fallbacks`): a + trace id and the requested group, in the caller's own metadata dict.""" + prepared: Final = dict(request) + if plan.kind == "vector_store_create": + return prepared # routed straight to a deployment, with no group recorded + if plan.prepare_field is None: + metadata_bucket(prepared, "metadata").update({"model_group": model}) + return prepared + prepared.setdefault("litellm_trace_id", str(uuid.uuid4())) + metadata_bucket(prepared, plan.prepare_field).update( + {"model_group": model, "model_group_alias": model if model in aliases else None} + ) + return prepared + + +# --- one attempt ---------------------------------------------------------------------------- + + +@dataclasses.dataclass(slots=True) +class AttemptCall: + """One attempt's host call: the op, its kwargs, and the request kwargs the router kept (the + ones its response headers are read from).""" + + host_op: str + kwargs: Mapping[str, object] + served: Mapping[str, object] + model_name: str + group: str + # `_embedding`'s batch-size refusal, raised by the driver in place of the host call. + refused: Exception | None = None + + +def reports_own_success(call: AttemptCall) -> bool: + """A realtime attempt's return says nothing about the session (a dropped one returns too), so + the host counts its success itself (`clean_realtime_session_deployment`).""" + return call.host_op == "_arealtime" + + +def clean_realtime_session_deployment(call: AttemptCall) -> str | None: + """`_ageneric_api_call_with_fallbacks`: the deployment a finished `_arealtime` session counts a + success against, when its success handlers ran and it did not end abnormally.""" + from litellm.constants import REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + session_logging: Final = call.kwargs.get("litellm_logging_obj") + if not reports_own_success(call) or not isinstance(session_logging, LiteLLMLogging): + return None + details: Final = session_logging.model_call_details + if details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is not True: + return None + if details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is True: + return None + deployment_id: Final = as_dict(call.kwargs.get("model_info")).get("id") + return None if deployment_id is None else str(deployment_id) + + +# `_scheduler` once `del router.scheduler` ran (None is a scheduler not built yet). +_NO_SCHEDULER: Final = object() + + +class Counters: + """`total_calls`, `success_calls` and `fail_calls`, keyed as each endpoint keys them.""" + + __slots__ = ("fail_calls", "success_calls", "total_calls") + + def __init__(self) -> None: + self.total_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: the router's live counters + self.success_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: same + self.fail_calls: defaultdict[str, int] = defaultdict(int) # mutable-ok: same + + @staticmethod + def _key(which: CountKey, model_name: str, group: str) -> str: + return model_name if which == "deployment" else group if which == "group" else "" + + def started(self, plan: Plan, call: AttemptCall) -> None: + if plan.total_key is not None: + self.total_calls[self._key(plan.total_key, call.model_name, call.group)] += 1 + + def succeeded(self, plan: Plan, call: AttemptCall) -> None: + if plan.success_key is not None: + self.success_calls[self._key(plan.success_key, call.model_name, call.group)] += 1 + + def retract_success(self, plan: Plan, call: AttemptCall) -> None: + """A served response the router then raised over: the attempt helper never reached its count.""" + if plan.success_key is not None: + self.success_calls[self._key(plan.success_key, call.model_name, call.group)] -= 1 + + def failed(self, plan: Plan, model_name: str | None, group: str) -> None: + if plan.fail_key is None: + return + if plan.fail_key == "deployment" and model_name is None: + return + self.fail_calls[self._key(plan.fail_key, model_name or "", group)] += 1 + + +def _compaction_surface(core: EndpointCore, plan: Plan) -> Surface | None: + """``surface_for_call`` over the function the Python router's fallback loop is handed.""" + if plan.kind == "generic" and surface_for_call(_handler_name(plan.host_op)) is not None: + return surface_for_call(invoked_handler_name(core, plan) or "") + return "chat" if plan.kind == "chat" and not plan.sync else None + + +def arm_compaction_state( + core: EndpointCore, + plan: Plan, + request: dict[str, object], # mutable-ok: the request kwargs, written in place as the Python router writes them +) -> None: + """``initialize_compaction_state`` at the top of ``async_function_with_fallbacks``.""" + surface: Final = _compaction_surface(core, plan) + if surface is not None: + request["_context_compaction_state"] = initialize_compaction_state(request, surface) # rebind-ok: its own + + +async def compacted_attempt( + core: EndpointCore, plan: Plan, deployment: Mapping[str, object], call: "AttemptCall" +) -> Mapping[str, object]: + """The attempt's kwargs after ``compact_to_fit``, which ``_acompletion`` and the generic helper + run on every attempt: it is also what refuses a compactor's own oversized child call.""" + if plan.kind != "generic" and (plan.kind != "chat" or plan.sync): + return call.kwargs + ctx: Final = RoutingContext(core.router) # pyright: ignore[reportArgumentType] # the checks read registry(), settings() and the token count + return await compact_to_fit(ctx, deployment, call.kwargs, _compaction_surface(core, plan)) + + +def attempt_call( + core: EndpointCore, + plan: Plan, + request: Mapping[str, object], + deployment: Mapping[str, object], + *, + group: str, + requested: str, + fallback_level: int = 0, +) -> AttemptCall: + """The host call for one attempt of ``plan`` against ``deployment``: the request (as the attempt + helper receives it), shaped for the deployment and merged with its litellm params.""" + shaping: Final = core.shaping + kwargs: Final = without(request, *ROUTING_KEYS) + named: Final = {name: kwargs.pop(name) for name in plan.named if name in kwargs} + data: Final = as_dict(deployment.get("litellm_params")) + model_name: Final = str(data.get("model")) + cache: Final = core.routing_cache + caching: Final = shaping.cache_responses + # A fallback hop routes the target group in a copy of the metadata (`run_async_fallback`). + if fallback_level > 0: + field: Final = plan.prepare_field or "metadata" + kwargs[field] = {**as_dict(kwargs.get(field)), "model_group": requested} + match plan.kind: + case "chat": + host = core.host # rebind-ok: rebound as the Python router rebinds it + drop_unsupported_classifier_reasoning_effort(deployment, requested, kwargs, host) + params = ( # rebind-ok: rebound as the Python router rebinds it + deployment_params_with_request_reasoning_override(data, kwargs) + ) + silent_model = params.pop("silent_model", None) # rebind-ok: rebound as the Python router rebinds it + start_silent_experiments(core, plan, silent_model, named.get("messages"), kwargs) + if "messages" in named: + kwargs.setdefault("messages", named["messages"]) + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, splits_stream_timeout=True) + kwargs.pop("silent_model", None) + final = without_router_only_kwargs( # rebind-ok: rebound as the Python router rebinds it + { + **params, + **named, + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + ) + return AttemptCall(plan.host_op, final, kwargs, str(params.get("model")), group) + case "text" | "embedding" | "image" | "transcription" | "rerank" | "adapter": + update_kwargs_with_deployment( + shaping, deployment, kwargs, cache=cache, splits_stream_timeout=plan.splits_stream_timeout + ) + refused: Final = embedding_batch_refusal(plan, deployment, named.get("input"), kwargs, requested) + if refused is not None: + return AttemptCall(plan.host_op, kwargs, kwargs, model_name, group, refused) + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + **named, + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, without_router_only_kwargs(final), kwargs, model_name, group) + case "speech": + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + voice = kwargs.pop("voice", None) # rebind-ok: rebound as the Python router rebinds it + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + **named, + "voice": data.get("voice") if voice is None else voice, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + case "batch" | "cancel_batch" | "vector_store_create": + if plan.kind == "cancel_batch": + data.update(_resolved_credentials(core, deployment, requested)) + data.pop("litellm_credential_name", None) + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, function_name=plan.function_name) + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + "custom_llm_provider": _provider(core, data), + "caching": caching, + "client": model_client(core, plan, deployment, kwargs), + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, str(data.get("model")), group) + case "generic": + return _generic_attempt(core, plan, kwargs, deployment, data, requested, group) + case "generic_sync": + update_kwargs_with_deployment( + shaping, + deployment, + kwargs, + cache=cache, + function_name="generic_api_call", + splits_stream_timeout=invoked_handler_name(core, plan) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + ) + endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it + if endpoint and isinstance(endpoint, str): + kwargs["endpoint"] = endpoint.replace(requested, model_name) + kwargs["model"] = model_name + final = { # rebind-ok: rebound as the Python router rebinds it + **data, + "custom_llm_provider": _generic_provider(data), + "caching": caching, + **kwargs, + } + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + case "moderation": + update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + kwargs["model"] = model_name + kwargs.update(data) + return AttemptCall(plan.host_op, kwargs, kwargs, model_name, group) + case "search": + return _search_attempt(core, plan, kwargs, group) + case "create_file": + # `_acreate_file` creates the file on every healthy deployment of the hop's name, not + # only the picked one: the driver hands the request to `create_file_everywhere`. + return AttemptCall(plan.host_op, kwargs, kwargs, requested, group) + + +def _generic_attempt( + core: EndpointCore, + plan: Plan, + kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs + deployment: Mapping[str, object], + data: dict[str, object], # mutable-ok: the attempt's copy of the deployment params + requested: str, + group: str, +) -> AttemptCall: + """`_ageneric_api_call_with_fallbacks_helper` for one deployment.""" + from litellm.llms.base_llm.passthrough.transformation import replace_path_segment + from litellm.router_rust.invoke.websocket import turn_admission + + handler: Final = invoked_handler_name(core, plan) + socket: Final = handler == "_aresponses_websocket" + update_kwargs_with_deployment( + core.shaping, + deployment, + kwargs, + cache=core.routing_cache, + function_name="_ageneric_api_call_with_fallbacks", + splits_stream_timeout=handler in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=socket, + ) + model_name: Final = str(data.get("model")) + endpoint: Final = kwargs.get("endpoint") + if endpoint: + try: + stripped, _, _, _ = core.host.invoke("get_llm_provider")( + model=model_name, + custom_llm_provider=kwargs.get("custom_llm_provider"), + api_base=kwargs.get("api_base"), + ) + replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it + except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is + replacement = model_name # rebind-ok: rebound as the Python router rebinds it + kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + replace_path_segment(str(endpoint), requested, replacement) + ) + provider: Final = _generic_provider(data) + final: Final = { + **without_retrieve_stream_defaults(data, kwargs), + "caching": core.shaping.cache_responses, + **kwargs, + "model": model_name, + **_session_with_model(kwargs.get("session"), model_name), + **({"turn_admission": turn_admission(core, requested, deployment, kwargs)} if socket else {}), + } + if provider is not None: + final["custom_llm_provider"] = provider + return AttemptCall(plan.host_op, final, kwargs, model_name, group) + + +_SEARCH_PROTECTED_PARAMS: Final = frozenset({"search_provider", "api_key", "api_base"}) + + +def _search_attempt( + core: EndpointCore, + plan: Plan, + kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs + group: str, +) -> AttemptCall: + """`SearchAPIRouter.async_search_with_fallbacks_helper`: one of the search tools named + ``group`` at random, called with its provider, credentials and params under the request's.""" + import random + + tools: Final = [tool for tool in core.endpoints.search_tools if as_dict(tool).get("search_tool_name") == group] + if not tools: + raise ValueError(f"Search tool '{group}' not found in router.search_tools") + selected: Final = random.choice(tools) # load balancing, as the Python router picks + params: Final = as_dict(as_dict(selected).get("litellm_params")) + search_provider: Final = params.get("search_provider") + if not search_provider: + raise ValueError(f"search_provider not found in litellm_params for search tool '{group}'") + search_params: Final = { + key: value + for source in (params, kwargs) + for key, value in source.items() + if key not in _SEARCH_PROTECTED_PARAMS and value is not None + } # comprehension-ok: the tool's params, then the request's + final: Final = { + "search_provider": search_provider, + "api_key": params.get("api_key"), + "api_base": params.get("api_base"), + **search_params, + } + return AttemptCall(plan.host_op, final, kwargs, group, group) + + +def unrouted_fields(plan: Plan) -> Mapping[str, object]: + """A search tool name is no model group: the Rust core hands every attempt to the host as is.""" + return {"unrouted": True} if plan.kind == "search" else {} + + +def failover_fields(plan: Plan) -> Mapping[str, object]: + """An endpoint whose attempt stamps no ``failed_deployment_id`` never runs weighted failover.""" + return {} if plan.stamps_failed_deployment else {"skip_weighted_failover": True} + + +# The Python router's handler names for the plans whose `function_name` differs from it. +_PYTHON_HANDLER_NAMES: Final[Mapping[str, str]] = MappingProxyType({"acreate_file": "_acreate_file"}) + + +def scoped_fallback_fields(plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """A request that creates or names a resource only one provider holds (a file upload, a batch + on an uploaded file, a fine-tuning job) falls back within its own model group only + (``run_async_fallback``'s ``same_model_group_only``), decided by the Python router's own rules.""" + from litellm.router_utils.fallback_event_handlers import ( + creates_provider_scoped_resource, + references_provider_scoped_resource, + ) + + function_name: Final = plan.function_name or plan.host_op + handlers: Final = { + "original_function": _Handler(_PYTHON_HANDLER_NAMES.get(function_name, function_name)), + "original_generic_function": _Handler(plan.host_op), + } + kwargs: Final = {**request, **handlers} + scoped: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(kwargs) + return {"same_group_fallbacks_only": True} if scoped else {} + + +@dataclasses.dataclass(frozen=True, slots=True) +class _Handler: + """A stand-in for a handler function: the scoped-resource rules read only its name.""" + + __name__: str + + +def _session_with_model(session: object, model_name: str) -> Mapping[str, object]: + if not isinstance(session, Mapping): + return {} + typed: Final = as_dict(session) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + if "model" not in typed and typed.get("type") != "transcription": + return {} + from litellm.realtime_api.main import with_resolved_session_model + + return {"session": with_resolved_session_model(typed, model_name)} + + +def _generic_provider(data: Mapping[str, object]) -> str | None: + from litellm.litellm_core_utils.routing_common_utils import provider_for_generic_call + + return provider_for_generic_call(data) # pyright: ignore[reportArgumentType] # deployment params + + +def model_client( + core: EndpointCore, plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> object: + """The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client` + with the sync keys for sync completion and embedding), dropped when the request carries + another api key.""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + + if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True: + return None + model_id: Final = as_dict(deployment.get("model_info")).get("id") + sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding") + stream: Final = "stream_" if kwargs.get("stream") is True else "" + key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client" + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + client: Final = _cached_client(core, key, span, local_only=not sync_keys) + dynamic_api_key: Final = kwargs.get("api_key") + if dynamic_api_key is not None and client is not None and dynamic_api_key != getattr(client, "api_key", None): + return None + return client + + +def _cached_client(core: EndpointCore, key: str, span: object, *, local_only: bool) -> object: + cache: Final = core.routing_cache + with service_target(ROUTER_CLIENTS_TARGET): + if local_only: + return cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # the cache holds untyped values + return cache.get_cache(key=key, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # see above + + +def _provider(core: EndpointCore, data: Mapping[str, object]) -> object: + """The deployment's declared provider, else the one its model names (`get_llm_provider`).""" + declared: Final = data.get("custom_llm_provider") + _, inferred, _, _ = core.host.invoke("get_llm_provider")(model=data["model"], custom_llm_provider=declared) + return declared or inferred + + +def _resolved_credentials(core: EndpointCore, deployment: Mapping[str, object], requested: str) -> Mapping[str, object]: + """`get_deployment_credentials_with_provider` for the picked deployment (`_acancel_batch`).""" + registry: Final = core.registry() + model_id: Final = as_dict(deployment.get("model_info")).get("id") + resolved: Final = registry.credentials(str(model_id or requested), form="with_provider") + return as_dict(resolved) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + + +def embedding_batch_refusal( + plan: Plan, deployment: Mapping[str, object], input: object, kwargs: Mapping[str, object], model: str +) -> Exception | None: + """``_embedding``'s check, past the deployment's kwargs shaping: an input the deployment admits + drops the mark an earlier rejected attempt left on the shared logging object.""" + if plan.kind != "embedding": + return None + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import ( + clear_rejected_before_provider_call, + embedding_batch_too_large_error, + ) + + error: Final = embedding_batch_too_large_error( + deployment=deployment, + input=input, + request_kwargs=kwargs, + model=model, # pyright: ignore[reportArgumentType] # opaque values + ) + logging_obj: Final = kwargs.get("litellm_logging_obj") + if error is None and isinstance(logging_obj, LiteLLMLogging): + clear_rejected_before_provider_call(logging_obj) + return error + + +def log_refused_call_sync(call: AttemptCall, refused: Exception) -> None: + """``_embedding`` logs its refusal before raising it: no provider call will.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import log_rejection_before_provider_call + + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLogging): + log_rejection_before_provider_call(logging_obj, refused) + + +async def log_refused_call(call: AttemptCall, refused: Exception) -> None: + """``log_refused_call_sync`` for ``_aembedding``.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.router_utils.embedding_batch_size_check import async_log_rejection_before_provider_call + + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLogging): + await async_log_rejection_before_provider_call(logging_obj, refused) + + +def stamp_error( + plan: Plan, error: BaseException, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> None: + """What each attempt helper stamps on the error it raises.""" + match plan.stamp: + case "deployment": + stamp_deployment_num_retries(error, deployment) + stamp_failed_deployment_id(error, kwargs.get("model_info") or deployment.get("model_info")) + case "retry_skip": + stamp_retry_skip_deployment_id(error, kwargs) + case "none": + pass + if plan.stamp != "deployment" and plan.stamps_failed_deployment: + # `_stamp_failed_attempt`: the deployment `_update_kwargs_with_deployment` put on the kwargs. + stamp_failed_deployment_id(error, kwargs.get("model_info")) + + +# --- silent experiments --------------------------------------------------------------------- + + +_SilentModel: TypeAlias = str | list[str] +_SILENT_MODEL_ADAPTER: Final[TypeAdapter[_SilentModel]] = TypeAdapter(_SilentModel) + + +def _silent_targets(silent_model: object) -> tuple[str, ...]: + """`_silent_experiment_targets`: none inside a mirrored call or for a malformed value.""" + from litellm._internal_context import in_mirrored_call + + if silent_model is None or in_mirrored_call(): + return () + try: + targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) + except ValidationError: + verbose_router_logger.warning( + "silent_model must be a model name or a list of model names, got %r; skipping shadow traffic", + silent_model, + ) + return () + return (targets,) if isinstance(targets, str) else tuple(targets) + + +def _silent_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]: # mutable-ok: the experiment's own kwargs + """`_get_silent_experiment_kwargs`.""" + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + + silent: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + original_metadata: Final = kwargs.get("metadata") + if original_metadata is not None and silent.get("metadata") is original_metadata: # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent["metadata"] = dict(original_metadata) # pyright: ignore[reportArgumentType, reportCallIssue] # the request's metadata dict + if "metadata" not in silent: + silent["metadata"] = {} + metadata: Final = silent["metadata"] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + if isinstance(metadata, dict): + metadata.pop("litellm_parent_otel_span", None) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + metadata["is_silent_experiment"] = True + silent.pop("litellm_call_id", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent.pop("litellm_logging_obj", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + silent.pop("standard_logging_object", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + return silent # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + + +def _snapshot(kwargs: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a snapshot handed to a background task + snapshot: Final = dict(kwargs) + metadata: Final = snapshot.get("metadata") + if isinstance(metadata, Mapping): + snapshot["metadata"] = dict(as_dict(metadata)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + return snapshot + + +async def _run_silent(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + from litellm._internal_context import mirrored_call + + remaining: Final = without(kwargs, "stream") + with mirrored_call(): + response: Final = await core.acompletion( + model=silent_model, messages=messages, stream=bool(kwargs.get("stream", False)), **remaining + ) + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass + + +async def _silent_async(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + try: + if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): + return + silent: Final = _silent_kwargs(kwargs) + as_dict_metadata: Final = silent["metadata"] + if isinstance(as_dict_metadata, dict): + as_dict_metadata["model_group"] = silent_model + await _run_silent(core, silent_model, copy.deepcopy(messages), silent) + except Exception: # noqa: BLE001, S110 # a silent experiment never fails the primary call + pass + + +def _silent_thread(core: EndpointCore, silent_model: str, messages: object, kwargs: Mapping[str, object]) -> None: + try: + if as_dict(kwargs.get("metadata")).get("is_silent_experiment", False): + return + silent: Final = _silent_kwargs(kwargs) + metadata: Final = silent["metadata"] + if isinstance(metadata, dict): + metadata["model_group"] = silent_model + loop: Final = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + + async def run() -> None: + await _run_silent(core, silent_model, copy.deepcopy(messages), silent) + pending: Final = asyncio.all_tasks() + current: Final = asyncio.current_task() + if current is not None: + pending.discard(current) + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + loop.run_until_complete(run()) + finally: + loop.close() + except Exception: # noqa: BLE001, S110 # a silent experiment never fails the primary call + pass + + +def start_silent_experiments( + core: EndpointCore, plan: Plan, silent_model: object, messages: object, kwargs: Mapping[str, object] +) -> None: + """Mirror the call to each silent model in the background: a task on the async path, a daemon + thread on the sync one.""" + for target in _silent_targets(silent_model): + if plan.sync: + threading.Thread( + target=_silent_thread, args=(core, target, messages, _snapshot(kwargs)), daemon=True + ).start() + else: + _ = asyncio.create_task( + _silent_async(core, target, messages, _snapshot(kwargs)) + ) # fire and forget, as the Python router + + +# --- the endpoints ---------------------------------------------------------------------------- + +_KEEPS_CLIENT: Final = frozenset( + { + "acreate_realtime_client_secret", + "arealtime_calls", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", + } +) +_KEEPS_PROVIDER_AND_CLIENT: Final = frozenset({"afile_delete", "afile_content"}) +_RESPONSE_ID_ENDPOINTS: Final = frozenset( + {"aget_responses", "acancel_responses", "acompact_responses", "adelete_responses", "alist_input_items"} +) +_CONTAINER_ENDPOINTS: Final = frozenset( + {"acreate_container", "alist_containers", "aretrieve_container", "adelete_container", *_CONTAINER_FILE_ROUTED} +) +_VECTOR_STORE_ENDPOINTS: Final = frozenset( + { + *(f"avector_store_{name}" for name in ("search", "create", "retrieve", "list", "update", "delete")), + *(f"avector_store_file_{name}" for name in ("create", "list", "retrieve", "content", "update", "delete")), + } +) +_INTERACTION_ENDPOINTS: Final = frozenset( + {"acreate_interaction", "aget_interaction", "adelete_interaction", "acancel_interaction"} +) +_MISSING: Final = object() + + +def pinned_video_deployment(plan: Plan, request: Mapping[str, object]) -> Mapping[str, object]: + """`_video_call_pin`: the deployment a video id names, for a call on that video only.""" + from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES, decode_video_id_with_provider + + video_id: Final = request.get("video_id") + if plan.endpoint not in VIDEO_ID_CALL_TYPES or not isinstance(video_id, str): + return {} + deployment_id: Final = decode_video_id_with_provider(video_id).get("deployment_id") + return {"pinned_deployment_id": deployment_id} if deployment_id else {} + + +async def _await(value: object) -> object: + """``await value``, raising the TypeError Python raises for a value that is not awaitable.""" + return await value # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType] # deliberately awaited as the Python router does + + +def _missing_model() -> TypeError: + return TypeError("_ageneric_api_call_with_fallbacks() missing 1 required positional argument: 'model'") + + +class RustEndpoints: + """The public endpoint methods of one Rust core, beyond the routed call loop the core drives.""" + + def __init__(self, core: EndpointCore, config: "RouterConfig") -> None: + from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout + from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority + + self._core: Final = core + self.counters: Final = Counters() + explicit: Final = config.get("timeout") + self._explicit_timeout: Final = explicit + self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None + self._stream_timeout: Final = config.get("stream_timeout") + # apply() flips it after construction, as on the Python core + self._cache_responses: bool = bool(config.get("cache_responses")) + self._assistants_config: Final = ( + as_dict(config.get("assistants_config")) if config.get("assistants_config") else None + ) + # These three are assigned by apply() after construction, as on the Python core. + tools: Final = list(_as_items(config.get("search_tools"))) + self._search_tools: list[object] = tools # mutable-ok: the router's live list + self._guardrails: list[object] = list(_as_items(config.get("guardrail_list"))) # mutable-ok: same + parsed: Final = parse_default_priority(config.get("default_priority")) + self._default_priority: Final = None if isinstance(parsed, InvalidPriority) else parsed + self._polling_interval: Final = config.get("polling_interval") + self._caching_groups: Final = config.get("caching_groups") + self._default_max_parallel_requests: object = config.get("default_max_parallel_requests") + self._scheduler: object = None + + @property + def cache_responses(self) -> bool: + return self._cache_responses + + def set_cache_responses(self, value: bool) -> None: + """``cache_responses`` assigned after construction (the proxy turns it on once its cache + exists): later calls pass ``caching=value``, as the Python core's flag does.""" + self._cache_responses = value + + def assign_setting(self, name: str, value: object) -> None: + """``search_tools``, ``guardrail_list`` or ``default_max_parallel_requests`` assigned after + construction, kept as given: later searches, guardrail picks and slot limits built from + then on read it, as the Python core's attributes are read.""" + if name == "search_tools": + self._search_tools = value # pyright: ignore[reportAttributeAccessIssue] # stored as given, as the Python core stores it + elif name == "guardrail_list": + self._guardrails = value # pyright: ignore[reportAttributeAccessIssue] # same + elif name == "default_max_parallel_requests": + self._default_max_parallel_requests = value + else: + raise ValueError(f"not an endpoint setting: {name}") + + @property + def default_max_parallel_requests(self) -> object: + """The setting as given: ``settings()`` reports it and the slot limits read it.""" + return self._default_max_parallel_requests + + @property + def search_tools(self) -> list[object]: # mutable-ok: the router's live list, as the Python router reports it + return self._search_tools + + @property + def guardrail_list(self) -> list[object]: # mutable-ok: same + return self._guardrails + + @staticmethod + def prepare_host(config: "RouterConfig", host: "PythonHost") -> None: + """`cache_responses` sets the process cache, unless one is set already (the proxy's).""" + if not config.get("cache_responses") or host.global_cache() is not None: + return + from litellm.caching.caching import Cache + + redis_url: Final = config.get("redis_url") + redis_host: Final = config.get("redis_host") + redis_port: Final = config.get("redis_port") + cache_config: Final[dict[str, object]] = {} # mutable-ok: the request kwargs the Python router mutates in place + cache_type: str = "local" # rebind-ok: redis when the router is given one + if redis_url is not None or (redis_host is not None and redis_port is not None): + cache_type = "redis" # rebind-ok: rebound as the Python router rebinds it + if redis_url is not None: + cache_config["url"] = redis_url + if redis_host is not None: + cache_config["host"] = redis_host + if redis_port is not None: + cache_config["port"] = str(redis_port) + if config.get("redis_password") is not None: + cache_config["password"] = config.get("redis_password") + if config.get("redis_db") is not None: + cache_config["db"] = str(config.get("redis_db")) + cache_config.update(as_dict(config.get("cache_kwargs"))) + host.set_global_cache(Cache(type=cache_type, **cache_config)) # pyright: ignore[reportArgumentType] # the Python router's cache config + + def shaping(self, resolved: "Resolved") -> Shaping: + return Shaping( + default_litellm_params=self.default_litellm_params(resolved), + timeout=resolved.timeout, + explicit_timeout=self._explicit_timeout, + request_timeout=self._request_timeout, + stream_timeout=self._stream_timeout, + cache_responses=self._cache_responses, + ) + + def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]: + """The defaults the Python constructor completes before any call: the router ``timeout`` + argument, no client retries, and the caching groups in the default metadata.""" + configured: Final = resolved.default_litellm_params + metadata: Final = configured.get("metadata") + return MappingProxyType( + { + "timeout": self._explicit_timeout, + "max_retries": 0, + **configured, + "metadata": { + **(as_dict(metadata) if isinstance(metadata, Mapping) else {}), # pyright: ignore[reportUnknownArgumentType] # the configured metadata + "caching_groups": self._caching_groups, + }, + } + ) + + def prepare( + self, plan: Plan, request: Mapping[str, object], model: str + ) -> dict[str, object]: # mutable-ok: the call's kwargs + return prepare_request(plan, request, model, self._core.resolved.model_group_alias) + + # -- host and registry --------------------------------------------------------------------- + + def _invoke(self, op: str) -> Callable[..., object]: + return self._core.host.invoke(op) + + async def _direct(self, op: str, kwargs: Mapping[str, object]) -> object: + return await _await(self._invoke(op)(**kwargs)) + + def _group(self, name: str) -> list[Mapping[str, object]]: # mutable-ok: a fresh list + """`_core_get_model_list(model_name=name)`: the deployments of the group ``name``.""" + records: Final = self._core.handle.deployments() + return [record for record in (as_dict(r) for r in records) if record.get("model_name") == name] + + def _is_model_group(self, name: str) -> bool: + """`_core_get_model_list(model_name=name)`: an alias, a routing group or a wildcard counts.""" + return bool(self._core.group_deployments(name)) + + # -- members ------------------------------------------------------------------------------ + + def member(self, name: str) -> Callable[..., object] | None: + """The public endpoint ``name``, bound to this core, or None when the core has none.""" + if name in _UNBRANCHED: + return _answers_none + if name in _ASSISTANTS: + return self._assistants(name) + if name in _MANAGED_AGENTS: + return self._managed_agents(name) + if name in _SYNC_VECTOR_STORE_FILES: + return self._sync_vector_store_files(name) + if name == "create_interaction": + return self._factory("create_interaction", "interactions.create") + direct: Final = { + "avector_store_create": self.avector_store_create, + "acreate_file": self.acreate_file, + "acreate_batch": self.acreate_batch, + "acancel_batch": self.acancel_batch, + "aretrieve_batch": self.aretrieve_batch, + "alist_batches": self.alist_batches, + "aadapter_completion": self.aadapter_completion, + "abatch_completion": self.abatch_completion, + "abatch_completion_one_model_multiple_requests": self.abatch_completion_one_model_multiple_requests, + "abatch_completion_fastest_response": self.abatch_completion_fastest_response, + "schedule_acompletion": self.schedule_acompletion, + "priority_header_update": self.priority_header_update, + "aguardrail": self.aguardrail, + "asearch": self.asearch, + }.get(name) + if direct is not None: + return direct + if name == "responses": + return self._sync_factory(name, PLANS["responses"]) + plan: Final = generic_plan(name) + if plan is None or name in PLANS: + return None + if plan.sync: + return self._sync_factory(name, plan) + return self._factory(name, plan.host_op) + + # -- the factory wrappers ------------------------------------------------------------------ + + def _factory(self, name: str, host_op: str) -> Callable[..., Awaitable[object]]: + async def async_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + return await self._wrapped(name, host_op, custom_llm_provider, client, dict(kwargs)) + + async_wrapper.__name__ = name + return async_wrapper + + async def _wrapped( + self, + name: str, + host_op: str, + custom_llm_provider: str | None, + client: object, + kwargs: dict[str, object], # mutable-ok: the wrapper's own kwargs + ) -> object: + plan: Final = generic_plan(name) or _generic(name, host_op) + if name in _CONTAINER_ENDPOINTS: + return await self._containers(plan, custom_llm_provider, kwargs) + if name in _RESPONSE_ID_ENDPOINTS: + from litellm.responses.utils import ResponsesAPIRequestUtils + + model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(kwargs.get("response_id")) # pyright: ignore[reportArgumentType] # any id the caller sends + if model_id is not None: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "model" + ] = model_id + return await self._ageneric(plan, kwargs) + if name in _VECTOR_STORE_ENDPOINTS: + return await self._vector_stores(name, plan, custom_llm_provider, kwargs) + if name in _INTERACTION_ENDPOINTS or name == "create_interaction": + return await self._interactions(plan, custom_llm_provider, kwargs) + if name in _KEEPS_CLIENT: + return await self._ageneric(plan, {**kwargs, "client": client}) + if name in _KEEPS_PROVIDER_AND_CLIENT: + return await self._ageneric(plan, {**kwargs, "custom_llm_provider": custom_llm_provider, "client": client}) + if name == "allm_passthrough_route": + if client: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "client" + ] = client + return await self._ageneric(plan, kwargs, passthrough=True) + return await self._ageneric(plan, kwargs) + + async def generic(self, plan: Plan, request: Mapping[str, object]) -> object: + """A dispatched generic endpoint called with its whole request (the wrapper's + ``custom_llm_provider`` and ``client`` are consumed by the wrapper).""" + kwargs: Final = without(request, "custom_llm_provider", "client") + return await self._ageneric(plan, kwargs) + + async def _ageneric( + self, + plan: Plan, + kwargs: Mapping[str, object], + *, + passthrough: bool = False, + ) -> object: + """`_ageneric_api_call_with_fallbacks`.""" + if "model" not in kwargs: + raise _missing_model() + + async def passthrough_call(prepared: Mapping[str, object], name: str) -> object: + sent: Final = without(prepared, *ROUTING_KEYS) + return await self._direct(plan.host_op, {"model": name, **sent}) + + return await self._core.drive(plan, kwargs, passthrough=passthrough_call if passthrough else None) + + def _sync_factory(self, name: str, plan: Plan) -> Callable[..., object]: + is_vector_store: Final = name.startswith("vector_store_") + + def sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + if not is_vector_store: + if "model" not in kwargs: + raise TypeError( + "_generic_api_call_with_fallbacks() missing 1 required positional argument: 'model'" + ) + return self._core.call_sync(plan, kwargs, _SINGLE_ATTEMPT) + provider_kwargs: dict[ # mutable-ok: the request kwargs the Python router mutates in place # rebind-ok: rebound as the Python router rebinds it + str, object + ] = ( + {**kwargs, "custom_llm_provider": custom_llm_provider} + if custom_llm_provider and "custom_llm_provider" not in kwargs + else dict(kwargs) + ) + if name == "vector_store_search": + provider_kwargs = { # rebind-ok: rebound as the Python router rebinds it + **provider_kwargs, + "_direct_vector_store_embedding_executor": self._embedding_executor(kwargs), + } + if provider_kwargs.get("model"): + return self._core.call_sync(plan, provider_kwargs, _SINGLE_ATTEMPT) + if name == "vector_store_search": + return self._invoke(plan.host_op)(**{**provider_kwargs, "router": self._core.router}) + return self._invoke(plan.host_op)(**provider_kwargs) + + sync_wrapper.__name__ = name + return sync_wrapper + + def _embedding_executor(self, kwargs: Mapping[str, object]) -> object: + from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, + vector_store_request_metadata, + ) + + return RouterVectorStoreEmbeddingExecutor( + router=self._core.router, # pyright: ignore[reportArgumentType] # the router serves aembedding + metadata=vector_store_request_metadata(kwargs), + ) + + async def _containers( + self, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> object: + """`_init_containers_api_endpoints`.""" + from litellm.responses.utils import ResponsesAPIRequestUtils + + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + container_id: Final = kwargs.get("container_id") + forwarded: Final = kwargs.get("model_id") + if isinstance(container_id, str): + decoded: Final = ResponsesAPIRequestUtils.decode_container_id(container_id) + original_id: Final = decoded.get("response_id", container_id) + if original_id != container_id: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "container_id" + ] = original_id + decoded_provider: Final = decoded.get("custom_llm_provider") + if decoded_provider and kwargs.get("custom_llm_provider") == "openai": + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = decoded_provider + model_id: Final = decoded.get("model_id") or ( + forwarded.strip() if isinstance(forwarded, str) and forwarded.strip() else None + ) + if model_id: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "model" + ] = model_id + return await self._ageneric(plan, kwargs) + requested: Final = kwargs.get("model") + if isinstance(requested, str) and requested.strip(): + return await self._ageneric(plan, kwargs, passthrough=True) + return await self._direct(plan.host_op, kwargs) + + async def _vector_stores( + self, + name: str, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: see _wrapped + ) -> object: + """`_init_vector_store_api_endpoints`.""" + if name == "avector_store_search": + kwargs = { # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + **kwargs, + "_direct_vector_store_embedding_executor": self._embedding_executor(kwargs), + } + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if kwargs.get("model"): + return await self._ageneric(plan, kwargs) + if name == "avector_store_search": + return await self._direct(plan.host_op, {**kwargs, "router": self._core.router}) + return await self._direct(plan.host_op, kwargs) + + async def _interactions( + self, + plan: Plan, + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> object: + """`_init_interactions_api_endpoints`.""" + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = "gemini" + if kwargs.get("agent") and kwargs.get("model") == kwargs.get("agent"): + kwargs["model"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + None + ) + if kwargs.get("model"): + return await self._ageneric(plan, kwargs) + return await self._direct(plan.host_op, kwargs) + + def _managed_agents(self, name: str) -> Callable[..., object]: + op: Final = _MANAGED_AGENTS[name] + + def prepared( + custom_llm_provider: str | None, + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place + ) -> dict[str, object]: # mutable-ok: the call's kwargs + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = custom_llm_provider + if "custom_llm_provider" not in kwargs: + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + "custom_llm_provider" + ] = "gemini" + return kwargs + + if name.startswith("a"): + + async def managed_agents_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return await self._direct(op, prepared(custom_llm_provider, dict(kwargs))) + + return managed_agents_wrapper + + def managed_agents_sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return self._invoke(op)(**prepared(custom_llm_provider, dict(kwargs))) + + return managed_agents_sync_wrapper + + def _sync_vector_store_files(self, name: str) -> Callable[..., object]: + op: Final = _SYNC_VECTOR_STORE_FILES[name] + + def vector_store_file_sync_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return self._invoke(op)(custom_llm_provider=custom_llm_provider, client=client, **kwargs) + + return vector_store_file_sync_wrapper + + def _assistants(self, name: str) -> Callable[..., Awaitable[object]]: + async def assistants_wrapper( + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + """`_pass_through_assistants_endpoint_factory`.""" + provider: object = custom_llm_provider # rebind-ok: the assistants config's when none is given + sent: Final = dict(kwargs) + if provider is None: + if self._assistants_config is None: + raise Exception( # noqa: TRY002 # the Python router raises a bare Exception + "'custom_llm_provider' must be set. Either via:\n `Router(assistants_config={'custom_llm_provider': ..})` \nor\n `router.arun_thread(custom_llm_provider=..)`" + ) + provider = self._assistants_config[ # rebind-ok: rebound as the Python router rebinds it + "custom_llm_provider" + ] + sent.update(as_dict(self._assistants_config.get("litellm_params"))) + return await self._direct(name, {"custom_llm_provider": provider, "client": client, **sent}) + + return assistants_wrapper + + # -- completions, priority and prompt management ---------------------------------------- + + def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: + from litellm.litellm_core_utils.routing_common_utils import get_request_team_id + from litellm.router_utils.request_priority import request_drops_params + + def deployment_params() -> tuple[Mapping[str, object], ...]: + names: Final = {as_dict(r).get("model_name") for r in self._core.handle.deployments()} + pinned: Final = ( + self._core.handle.deployment(model) if model not in names and isinstance(model, str) else None # pyright: ignore[reportUnnecessaryIsInstance] # a caller's value, typed for the common case + ) + if pinned is not None: + return (as_dict(as_dict(pinned).get("litellm_params")),) + if kwargs.get("specific_deployment") is True: + candidates = [ # rebind-ok: rebound as the Python router rebinds it + as_dict(r) + for r in self._core.handle.deployments() + if as_dict(as_dict(r).get("litellm_params")).get("model") == model + ] + else: + # `_core_get_model_list`: an alias's target, a routing group's members and a + # wildcard's deployments list here too. + candidates = [ # rebind-ok: rebound as the Python router rebinds it + as_dict(record) + for record in self._core.registry().deployments(model, team_id=get_request_team_id(kwargs)) or () + ] + return tuple(as_dict(candidate.get("litellm_params")) for candidate in candidates) + + return request_drops_params( + kwargs, + self._core.resolved.default_litellm_params, + deployment_params(), + ) + + def _split_priority( + self, model: str, kwargs: Mapping[str, object] + ) -> tuple[int | None, dict[str, object]]: # mutable-ok: the call's kwargs + """`_core_split_request_priority`.""" + from litellm.router_utils.request_priority import InvalidPriority, priority_error, resolve_request_priority + + priority: Final = resolve_request_priority( + requested=kwargs.get("priority"), + default_priority=self._default_priority, + drops_params=lambda: self._drops_params(model=model, kwargs=kwargs), + ) + if isinstance(priority, InvalidPriority): + raise priority_error(model=model, message=priority.message) + return priority, without(kwargs, "priority") + + def priority_header_update( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: + from litellm.router_utils.request_priority import priority_header_fields + + return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs)) + + def set_scheduler(self, scheduler: object) -> None: + """``router.scheduler = ...``, which the next prioritized call waits on.""" + self._scheduler = scheduler + + def delete_scheduler(self) -> None: + """``del router.scheduler``: a prioritized call raises from then on, as on the Python core.""" + self._scheduler = _NO_SCHEDULER + + def _get_scheduler(self) -> object: + if self._scheduler is _NO_SCHEDULER: + raise AttributeError("'RustCore' object has no attribute 'scheduler'", name="scheduler") + if self._scheduler is None: + from litellm.scheduler import Scheduler + + self._scheduler = ( + Scheduler(polling_interval=self._polling_interval) # pyright: ignore[reportArgumentType] # as configured + if self._polling_interval is not None + else Scheduler() + ) + return self._scheduler + + async def _schedule( + self, model: str, priority: int, run: Callable[[], Awaitable[object]], request: Mapping[str, object] + ) -> object: + """`_schedule_factory`: wait for the scheduler's turn, run, mark the response.""" + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.scheduler import FlowItem, Scheduler + + scheduler: Final = self._get_scheduler() + assert isinstance(scheduler, Scheduler) + + async def healthy_deployments() -> Sequence[object]: + healthy: Final = await self._core.select(model, dict(request), kind="healthy") + return list(healthy) if isinstance(healthy, Sequence) else [healthy] + + await scheduler.wait_for_turn( + request=FlowItem(priority=priority, request_id=str(uuid.uuid4()), model_name=model), + timeout=self._core.resolved.timeout, # pyright: ignore[reportArgumentType] # the router timeout + get_healthy_deployments=healthy_deployments, + ) + try: + response: Final = await run() + except Exception as error: + setattr(error, "priority", priority) # noqa: B010 # the Python router stamps the same attribute + raise + hidden: Final = get_hidden_params(response) + if hidden is not None: + headers: Final = hidden.setdefault("additional_headers", {}) + if isinstance(headers, dict): + headers.update({"x-litellm-request-prioritization-used": True}) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + return response + + async def acompletion(self, request: Mapping[str, object]) -> object: + model: Final = request.get("model") + if model is None and "model" in request: + return await self._core.drive(PLANS["acompletion"], request) + if "model" not in request: + raise TypeError("acompletion() needs a model name") + if not isinstance(model, str): + _ = self._core.routed_name(model) + kwargs: Final = {**request, "stream": request.get("stream", False)} + start_time: Final = time.time() + response: Final = await self._routed_acompletion(model, kwargs) # pyright: ignore[reportArgumentType] # the name as given + self._core.service_logged("acompletion", start_time, time.time(), kwargs) + return response + + async def _routed_acompletion(self, model: str, kwargs: Mapping[str, object]) -> object: + plan: Final = PLANS["acompletion"] + priority, rest = self._split_priority(model, kwargs) + from litellm.router_rust.invoke.prompt_management import ( + is_prompt_management_model, + prompt_management_acompletion, + ) + + if is_prompt_management_model(self._core, model): + prepared: Final = self.prepare(plan, rest, model) + messages: Final = rest.get("messages") + + def factory() -> Awaitable[object]: + return prompt_management_acompletion(self._core, model, messages, prepared) + + if priority is None: + return await factory() + return await self._schedule(model, priority, factory, prepared) + if priority is None: + return await self._core.drive(plan, rest) + return await self._schedule(model, priority, lambda: self._core.drive(plan, rest), rest) + + async def atext_completion(self, request: Mapping[str, object]) -> object: + model: Final = request.get("model") + if model is None and "model" in request: + return await self._core.drive(PLANS["atext_completion"], request) + if "model" not in request: + raise TypeError("atext_completion() needs a model name") + if not isinstance(model, str): + _ = self._core.routed_name(model) + kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") + plan: Final = PLANS["atext_completion"] + priority, rest = self._split_priority(model, kwargs) # pyright: ignore[reportArgumentType] # the name as given + if priority is None: + return await self._core.drive(plan, rest) + return await self._schedule(model, priority, lambda: self._core.drive(plan, rest), rest) # pyright: ignore[reportArgumentType] # see above + + async def schedule_acompletion( + self, + model: str, + messages: object, + priority: int | None = None, + stream: bool = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + from litellm.router_utils.request_priority import priority_error, unqueueable_priority_message + + queued, _ = self._split_priority(model, {**kwargs, "priority": priority}) + if queued is None: + raise priority_error(model=model, message=unqueueable_priority_message(priority)) + return await self._core.acompletion(model=model, messages=messages, stream=stream, priority=queued, **kwargs) + + # -- batch completions ------------------------------------------------------------------- + + async def _no_exceptions(self, **kwargs: object) -> object: # kwargs-ok: the request + try: + return await self._core.acompletion(**kwargs) + except Exception as error: # noqa: BLE001 # returned in place, as the Python router does + return error + + async def abatch_completion( + self, + models: list[str], # mutable-ok: the request kwargs the Python router mutates in place + messages: object, + **kwargs: object, # kwargs-ok: the endpoint's open request + ) -> object: + if isinstance(messages, list) and all(isinstance(m, dict) for m in messages): # pyright: ignore[reportUnknownVariableType] # checked item by item + return await asyncio.gather( + *(self._no_exceptions(model=model, messages=messages, **kwargs) for model in models) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + ) + if isinstance(messages, list) and all(isinstance(m, list) for m in messages): # pyright: ignore[reportUnknownVariableType] # same + rows: Final[list[object]] = list(messages) # pyright: ignore[reportUnknownArgumentType] # same # mutable-ok: the request kwargs the Python router mutates in place + + async def indexed(model: str, message: object, idx: int) -> tuple[object, int]: + return await self._no_exceptions(model=model, messages=message, **kwargs), idx + + responses: Final = await asyncio.gather( + *( + indexed(model, message, idx) for idx, message in enumerate(rows) for model in models + ) # comprehension-ok: a flat walk of nested rows + ) + final: Final[list[list[object]]] = [ # mutable-ok: the request kwargs the Python router mutates in place + [] for _ in rows + ] + for response, idx in responses: + final[idx].append(response) + return final + return None + + async def abatch_completion_one_model_multiple_requests( + self, + model: str, + messages: list[object], # mutable-ok: the request kwargs the Python router mutates in place + **kwargs: object, # kwargs-ok: the endpoint's open request + ) -> object: + return await asyncio.gather(*(self._no_exceptions(model=model, messages=m, **kwargs) for m in messages)) + + async def abatch_completion_fastest_response( + self, + model: str, + messages: object, + stream: bool = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.types.utils import ModelResponse + + models: Final = [m.strip() for m in model.split(",")] + + async def no_exceptions(model_name: str) -> object: + try: + return await self._core.acompletion(model=model_name, messages=messages, stream=stream, **kwargs) + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 # see _no_exceptions + return error + + pending: set[ # mutable-ok: the request kwargs the Python router mutates in place # rebind-ok: rebound as the Python router rebinds it + asyncio.Task[object] + ] = {asyncio.create_task(no_exceptions(m)) for m in models} + while pending: + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + for task in done: + result = task.result() + if isinstance(result, (ModelResponse, CustomStreamWrapper)): + for other in pending: + other.cancel() + hidden = get_hidden_params(result) + if hidden is not None: + hidden["fastest_response_batch_completion"] = True + return result + raise Exception("All tasks failed") # noqa: TRY002 # the Python router raises a bare Exception + + # -- moderation, text completion and adapter -------------------------------------------- + + async def amoderation(self, request: Mapping[str, object]) -> object: + """`_pass_through_moderation_endpoint_factory`: one pick when the model is a group.""" + kwargs: Final = without(request, "custom_llm_provider", "client") + plan: Final = PLANS["amoderation"] + model: Final = kwargs.get("model", "") + if not isinstance(model, str): + _ = self._core.routed_name(model) + if model and isinstance(model, str) and self._is_model_group(model): + return await self._core.drive(plan, kwargs, _SINGLE_ATTEMPT) + prepared: Final = prepare_request(plan, kwargs, str(model), self._core.resolved.model_group_alias) + return await self._direct("amoderation", prepared) + + def text_completion(self, request: Mapping[str, object]) -> object: + """The sync `text_completion`: one pick, no retries, the group name as the model.""" + from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin + + kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") + model: Final = kwargs.get("model") + prompt: Final = kwargs.get("prompt") + if "model" not in kwargs: + raise TypeError("text_completion() missing 1 required positional argument: 'model'") + kwargs["num_retries"] = kwargs.get("num_retries", self._core.resolved.num_retries) + metadata_bucket(kwargs, "metadata").update({"model_group": model}) + with endpoint_selection("text_completion", model=model): + deployment: Final = self._core.router.get_available_deployment( + model=model, # pyright: ignore[reportArgumentType] # the name as given + messages=_user_message(prompt), + specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the request's value, as given + request_kwargs=kwargs, + ) + data: Final = as_dict(deployment.get("litellm_params")) + for key, value in self._core.resolved.default_litellm_params.items(): + if key not in kwargs: + kwargs[key] = value + elif key == "metadata": + bucket = kwargs[key] + if isinstance(bucket, dict): + bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + pinned: Final = deployment_model_id_pin(data) + if pinned is not None: + kwargs["model_id"] = pinned + return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs}) + + async def aadapter_completion( + self, + adapter_id: str, + model: str, + is_retry: bool | None = False, + is_fallback: bool | None = False, + is_async: bool | None = False, + **kwargs: object, # kwargs-ok: the request + ) -> object: + return await self._core.drive( + PLANS["aadapter_completion"], {**kwargs, "model": model, "adapter_id": adapter_id} + ) + + # -- files, batches and vector stores --------------------------------------------------- + + async def acreate_batch(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acreate_batch"], {**kwargs, "model": model}) + + async def acancel_batch(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acancel_batch"], {**kwargs, "model": model}) + + async def avector_store_create(self, model: str | None, **kwargs: object) -> object: # kwargs-ok: the request + if model is None: + return await self._factory("avector_store_create", "vector_stores.main.acreate")(**kwargs) + return await self._core.drive(PLANS["avector_store_create"], {**kwargs, "model": model}, _SINGLE_ATTEMPT) + + async def aretrieve_batch(self, model: str | None = None, **kwargs: object) -> object: # kwargs-ok: the request + """Ask every deployment of the group (or of the router) for the batch.""" + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.types.utils import LiteLLMBatch + + shaping: Final = self._core.shaping + metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch") + if model is not None: + specific: Final = kwargs.pop("specific_deployment", None) + healthy = await self._core.select( # rebind-ok: rebound as the Python router rebinds it + model, + dict(kwargs), + kind="healthy", + specific_deployment=bool(specific) or None, + messages=_user_message("retrieve-api-fake-text"), + ) + else: + healthy = [as_dict(r) for r in self._core.handle.deployments()] # rebind-ok: as Python rebinds it + errors: Final[list[BaseException]] = [] # mutable-ok: the request kwargs the Python router mutates in place + + async def try_retrieve(deployment: Mapping[str, object]) -> object: + try: + data: Final = as_dict(deployment.get("litellm_params")) + deployment_model: Final = data.get("model") + provider: object = data.get("custom_llm_provider") # rebind-ok: inferred when not declared + if deployment_model is None: + raise Exception(f"Model not found in litellm_params for deployment: {deployment}") # noqa: TRY002 # as the Python router + if not provider: + lookup: Final = self._core.host.invoke("get_llm_provider") + _, provider, _, _ = lookup(model=deployment_model) # rebind-ok: as Python rebinds it + new_kwargs: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + update_kwargs_with_deployment( + shaping, + deployment, + new_kwargs, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + cache=self._core.routing_cache, + function_name="aretrieve_batch", + ) + bucket: Final = metadata_bucket(new_kwargs, metadata_variable_name) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + if not bucket.get("model_group"): + bucket["model_group"] = model or deployment.get("model_name") + new_kwargs.pop("custom_llm_provider", None) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + data.pop("custom_llm_provider", None) + return await self._direct("aretrieve_batch", {**data, "custom_llm_provider": provider, **new_kwargs}) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + except Exception as error: # noqa: BLE001 # collected; the first one is raised when no deployment has the batch + errors.append(error) + return None + + deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] + if not deployments: + raise Exception("No healthy deployments found.") # noqa: TRY002 # as the Python router + results: Final = await asyncio.gather(*(try_retrieve(as_dict(d)) for d in deployments), return_exceptions=True) + for result in results: + if isinstance(result, LiteLLMBatch): + return result + if errors: + raise errors[0] + raise Exception(f"Unable to find batch in any model. Received errors - {errors}") # noqa: TRY002 # as the Python router + + async def alist_batches(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + deployments: Final = self._group(model) + + async def try_list(deployment: Mapping[str, object]) -> object: + try: + return await self._direct("alist_batches", {**as_dict(deployment.get("litellm_params")), **kwargs}) + except Exception: # noqa: BLE001 # a failing deployment is left out, as the Python router does + return None + + results: Final = await asyncio.gather(*(try_list(d) for d in deployments)) + final: Final[dict[str, object]] = { # mutable-ok: the request kwargs the Python router mutates in place + "object": "list", + "data": [], + "first_id": None, + "last_id": None, + "has_more": False, + } + data: Final[list[object]] = [] # mutable-ok: the request kwargs the Python router mutates in place + final["data"] = data + for result in results: + if result is None: + continue + if final["first_id"] is None and hasattr(result, "first_id"): + final["first_id"] = getattr(result, "first_id") # noqa: B009 # as the Python router + final["last_id"] = getattr(result, "last_id") # noqa: B009 # same + data.extend(getattr(result, "data")) # noqa: B009 # same # pyright: ignore[reportAny] # litellm's untyped helpers and request values + if getattr(result, "has_more", False) is True: + final["has_more"] = True + return final + + async def acreate_file(self, model: str, **kwargs: object) -> object: # kwargs-ok: the request + return await self._core.drive(PLANS["acreate_file"], {**kwargs, "model": model}) + + async def create_file_everywhere(self, kwargs: Mapping[str, object], requested: str, specific: object) -> object: + """`_acreate_file`: create the file on every healthy deployment of the group.""" + from litellm.litellm_core_utils import batch_utils + from litellm.litellm_core_utils.core_helpers import ( + safe_deep_copy, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.litellm_core_utils.routing_common_utils import ( + add_model_file_id_mappings, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + ) + from litellm.types.utils import LlmProviders + + healthy: Final = await self._core.select( + requested, + dict(kwargs), + kind="healthy", + specific_deployment=bool(specific) or None, + messages=_user_message(_PICK_TEXTS["create_file"]), + ) + deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] + + async def create_for(deployment: Mapping[str, object]) -> object: + kwargs_copy: Final = safe_deep_copy(dict(kwargs)) # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + update_kwargs_with_deployment( + self._core.shaping, + deployment, + kwargs_copy, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + cache=self._core.routing_cache, + function_name="acreate_file", + ) + data: Final = as_dict(deployment.get("litellm_params")) + model_name: Final = str(data.get("model")) + self.counters.total_calls[model_name] += 1 + declared: Final = data.get("custom_llm_provider") + stripped, inferred, _, _ = self._core.host.invoke("get_llm_provider")( + model=data["model"], custom_llm_provider=declared + ) + provider: Final = declared or inferred + purpose: Final = kwargs.get("purpose") + file: object = kwargs.get("file") # rebind-ok: rewritten for the deployment + if not file or not purpose: + raise Exception("file and file_purpose are required for create_file") # noqa: TRY002 # as the Python router + replaces: Final = batch_utils.should_replace_model_in_jsonl( + purpose=purpose, # pyright: ignore[reportArgumentType] # the caller's purpose + passthrough=kwargs.get("passthrough") is True, + ) + if replaces: + file = batch_utils.replace_model_in_jsonl( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # an untyped helper # rebind-ok: rewritten for the deployment + file_content=file, # pyright: ignore[reportArgumentType] # the caller's file + new_model_name=stripped, + keep_line_numbers=provider == LlmProviders.VERTEX_AI.value, + ) + kwargs_copy["file"] = file + if provider == LlmProviders.LITELLM_PROXY.value: + kwargs_copy["extra_body"] = MappingProxyType( + {**as_dict(kwargs_copy.get("extra_body")), "target_model_names": stripped} # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # litellm's untyped helpers and request values + ) + if "gcs_bucket_name" in data: + metadata_bucket(kwargs_copy, "litellm_metadata")["gcs_bucket_name"] = data["gcs_bucket_name"] # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + from litellm.router_rust.invoke.slots import deployment_slot + + with deployment_slot(self._core, deployment, kwargs_copy): # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + response: Final = await self._direct( + "acreate_file", + { + **data, + "custom_llm_provider": provider, + "caching": self._cache_responses, + "client": model_client(self._core, PLANS["acreate_file"], deployment, kwargs_copy), # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + **kwargs_copy, + }, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + ) + self.counters.success_calls[model_name] += 1 + return response + + responses: Final = await asyncio.gather(*(create_for(as_dict(d)) for d in deployments)) + if len(responses) == 0: + raise Exception("No healthy deployments found.") # noqa: TRY002 # as the Python router + mapping: Final = add_model_file_id_mappings(healthy_deployments=deployments, responses=responses) # pyright: ignore[reportArgumentType, reportUnknownVariableType] # deployment records + returned: Final = responses[0] + hidden: Final = get_hidden_params(returned) + if hidden is not None: + hidden["model_file_id_mapping"] = mapping + return returned + + # -- search tools ------------------------------------------------------------------------- + + async def asearch( + self, + custom_llm_provider: str | None = None, + client: object = None, + **kwargs: object, # kwargs-ok: the request + ) -> object: + """`_asearch_with_fallbacks`: the search tool name (``search_tool_name``, else ``model``) + routes as the model; the factory wrapper drops ``custom_llm_provider`` and ``client``.""" + request: Final = dict(kwargs) + search_tool_name: Final = request.get("search_tool_name", request.get("model")) + if not search_tool_name: + raise ValueError("search_tool_name or model parameter is required for search") + request["model"] = search_tool_name + return await self._core.drive(PLANS["asearch"], request) + + # -- guardrails --------------------------------------------------------------------------- + + async def aguardrail( + self, + guardrail_name: str, + original_function: Callable[..., Awaitable[object]], + **kwargs: object, # kwargs-ok: the request + ) -> object: + """Pick one of the guardrails named ``guardrail_name`` (weighted) and run it. A guardrail name + is no model group, so the router's retries never apply.""" + request: Final = dict(kwargs) + request.setdefault("litellm_trace_id", str(uuid.uuid4())) + metadata_bucket(request, "litellm_metadata").update( + { + "model_group": guardrail_name, + "model_group_alias": guardrail_name + if guardrail_name in self._core.resolved.model_group_alias + else None, + } + ) + verdict: Final = await original_function(**request, selected_guardrail=self.available_guardrail(guardrail_name)) + # `make_call`, `async_function_with_retries` and `async_function_with_fallbacks` stamp the result. + headed: Final = await self._core.set_response_headers(verdict, guardrail_name, request) + return add_fallback_headers_to_response( + response=add_retry_headers_to_response(response=headed, attempted_retries=0, max_retries=None), + attempted_fallbacks=0, + ) + + def available_guardrail(self, guardrail_name: str) -> object: + """``_core_get_available_guardrail``: the one guardrail configured under the name, else one of + them picked by weight (``simple_shuffle``, the name resolved through the alias map).""" + from litellm.router_strategy.simple_shuffle import simple_shuffle + + candidates: Final = [g for g in self._guardrails if as_dict(g).get("guardrail_name") == guardrail_name] + if not candidates: + raise ValueError(f"No guardrail found with name: {guardrail_name}") + if len(candidates) == 1: + return candidates[0] + return simple_shuffle( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helper + resolve_model_alias=self._core.resolve_alias, + healthy_deployments=candidates, # pyright: ignore[reportArgumentType] # guardrail configs + model=guardrail_name, + request_kwargs=None, + ) + + # -- attempts ----------------------------------------------------------------------------- + + def attempt_failed( + self, + plan: Plan, + error: BaseException, + deployment: Mapping[str, object], + call: AttemptCall | None, + requested: str, + ) -> None: + from litellm import exceptions as litellm_exceptions + from litellm.types.router import LiteLLM_Params + + # `_acompletion`'s Timeout branch counts no failure and adds the deployment's timeouts; the + # sync `_completion` has no such branch. + timed_out: Final = plan.kind == "chat" and not plan.sync and isinstance(error, litellm_exceptions.Timeout) + if not timed_out: + self.counters.failed(plan, call.model_name if call is not None else None, requested) + elif isinstance(error, litellm_exceptions.Timeout) and self._core.host.dynamic().expose_router_debug_in_errors: + params: Final = as_dict(deployment.get("litellm_params")) + try: + typed: Mapping[str, object] = ( # rebind-ok: rebound as the Python router rebinds it + LiteLLM_Params.model_validate(params).model_dump() + ) + except Exception: # noqa: BLE001 # the params as configured, when they do not validate + typed = params # rebind-ok: rebound as the Python router rebinds it + error.message += ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + f"\n\nDeployment Info: request_timeout: {typed.get('request_timeout')}" + f"\ntimeout: {typed.get('timeout')}" + f"\nstream_timeout: {typed.get('stream_timeout')}" + ) + + def check_response(self, plan: Plan, response: object, call: AttemptCall) -> None: + """What the attempt helper checks of a response before it counts it as served.""" + + def sync_pre_call_checks(self, plan: Plan, deployment: Mapping[str, object], request: Mapping[str, object]) -> None: + """`routing_strategy_pre_call_checks` on the sync path: the process callbacks' pre-call checks. + Sync completion runs them for a group but not for a deployment id.""" + from litellm.integrations.custom_logger import CustomLogger + + if plan.kind == "chat": + model: Final = str(request.get("model")) + names: Final = {as_dict(r).get("model_name") for r in self._core.handle.deployments()} + if model not in names and self._core.handle.deployment(model) is not None: + return + for callback in tuple(self._core.routing_callbacks()): + if isinstance(callback, CustomLogger): + callback.pre_call_check(dict(deployment)) # pyright: ignore[reportUnknownMemberType] # litellm's untyped helpers and request values + + +_SINGLE_ATTEMPT: Final[Mapping[str, object]] = MappingProxyType({"num_retries": 0, "disable_fallbacks": True}) + + +async def _answers_none(*_args: object, **_kwargs: object) -> None: # kwargs-ok: any request + """An endpoint the Python router's async wrapper has no branch for.""" + return + + +def _as_items(value: object) -> list[object]: # mutable-ok: a fresh list + return list(value) if isinstance(value, list) else [] # pyright: ignore[reportUnknownArgumentType] # a config list + + +if TYPE_CHECKING: + from litellm.router_config import RouterConfig + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import Resolved + + +__all__ = [ + "PLANS", + "ROUTING_KEYS", + "AttemptCall", + "Counters", + "Plan", + "RustEndpoints", + "Shaping", + "arm_compaction_state", + "attempt_call", + "clean_realtime_session_deployment", + "compacted_attempt", + "failover_fields", + "generic_plan", + "log_refused_call", + "log_refused_call_sync", + "pinned_video_deployment", + "plan_for", + "prepare_request", + "reports_own_success", + "selection_endpoint", + "stamp_error", + "unrouted_fields", +] diff --git a/litellm/router_rust/invoke/prompt_management.py b/litellm/router_rust/invoke/prompt_management.py new file mode 100644 index 00000000000..62bd77e1b48 --- /dev/null +++ b/litellm/router_rust/invoke/prompt_management.py @@ -0,0 +1,129 @@ +""" +Prompt-management groups on the Rust router core (``_is_prompt_management_model`` and +``_prompt_management_factory``): a group of exactly one deployment whose model names a prompt +management integration (``langfuse/...``) or ``litellm_agent/...``. The request is not routed with +retries and fallbacks: the deployment is picked, its prompt rendered by the request's logging +object, and the rendered model called directly, or routed when it names a group. +""" + +from collections.abc import Mapping +from typing import Final + +from litellm.litellm_core_utils.core_helpers import without +from litellm.router_rust.invoke.endpoints import PLANS, EndpointCore +from litellm.router_rust.invoke.shaping import as_dict, update_kwargs_with_deployment + +_PROMPT_MANAGEMENT_PARAMS: Final = frozenset( + {"bitbucket_config", "dotprompt_config", "prompt_id", "prompt_variables", "prompt_label", "prompt_version"} +) + + +def is_prompt_management_model(core: EndpointCore, model: str) -> bool: + """`_is_prompt_management_model`: one deployment, whose model's provider prefix is a known + custom-logger-compatible callback.""" + model_list: Final = core.group_deployments(model_name=model) + if model_list is None or len(model_list) != 1: + return False + litellm_model: Final = as_dict(model_list[0].get("litellm_params")).get("model") + if not isinstance(litellm_model, str) or "/" not in litellm_model: + return False + known: Final = core.host.dynamic().known_custom_logger_compatible_callbacks + return litellm_model.split("/")[0] in known + + +def _logging_and_kwargs( + kwargs: dict[str, object], # mutable-ok: the request kwargs the Python router mutates in place +) -> tuple[object, dict[str, object]]: # mutable-ok: same + """The request's logging object, else the one `function_setup` builds (with the kwargs it returns).""" + from litellm.utils import Rules, function_setup, get_utc_datetime + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if logging_obj is not None: + return logging_obj, kwargs + return function_setup( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers + **{ # pyright: ignore[reportArgumentType] # the request's keyword payload + "original_function": "acompletion", + "rules_obj": Rules(), + "start_time": get_utc_datetime(), + **kwargs, + } + ) + + +async def prompt_management_acompletion( + core: EndpointCore, + model: str, + messages: object, + request: Mapping[str, object], +) -> object: + """`_prompt_management_factory`, on the request as `acompletion` prepared it.""" + from litellm.utils import get_non_default_completion_params + + # `acompletion` hands the routing function along; function_setup consumes it. + logging_obj, kwargs = _logging_and_kwargs({**request, "original_function": core.router.acompletion}) + caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout") + caller_stream_timeout: Final = kwargs.get("stream_timeout") + specific_deployment: Final = kwargs.pop("specific_deployment", None) + picked: Final = await core.select( + model, + kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs + kind="pick", + specific_deployment=bool(specific_deployment), + messages=messages, + ) + deployment: Final = as_dict(picked) + update_kwargs_with_deployment( + core.shaping, + deployment, + kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs + cache=core.routing_cache, + splits_stream_timeout=True, + ) + params: Final = as_dict(deployment.get("litellm_params")) + data: Final = dict(params) + litellm_model: Final = data.get("model") + is_litellm_agent_model: Final = isinstance(litellm_model, str) and litellm_model.startswith("litellm_agent/") + prompt_id: Final = kwargs.get("prompt_id") or params.get("prompt_id") + prompt_variables: Final = kwargs.get("prompt_variables") or params.get("prompt_variables") + prompt_label: Final = kwargs.get("prompt_label") or params.get("prompt_label") + if not is_litellm_agent_model and (prompt_id is None or not isinstance(prompt_id, str)): + raise ValueError(f"Prompt ID is not set or not a string. Got={prompt_id}, type={type(prompt_id)}") + if prompt_variables is not None and not isinstance(prompt_variables, dict): + raise ValueError( + f"Prompt variables is set but not a dictionary. Got={prompt_variables}, type={type(prompt_variables)}" + ) + rendered_model, rendered_messages, optional_params = logging_obj.get_chat_completion_prompt( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue, reportOptionalMemberAccess] # the request's Logging object + model=litellm_model, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + messages=messages, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + non_default_params=get_non_default_completion_params(kwargs=kwargs), # pyright: ignore[reportArgumentType] # the request kwargs + prompt_id=prompt_id, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + prompt_variables=prompt_variables, + prompt_label=prompt_label, # pyright: ignore[reportArgumentType] # a request value, passed as the Python router passes it + request_kwargs=kwargs, + injected_for_every_deployment=True, + ) + filtered_data: Final = without(data, *_PROMPT_MANAGEMENT_PARAMS) + final: Final[dict[str, object]] = {**filtered_data, **kwargs, **optional_params} # pyright: ignore[reportUnknownVariableType] # mutable-ok: the call's kwargs + final["model"] = rendered_model + final["messages"] = rendered_messages + final["litellm_logging_obj"] = logging_obj + final["prompt_id"] = prompt_id + final["prompt_variables"] = prompt_variables + final["prompt_label"] = prompt_label + target_list: Final = core.group_deployments(model_name=str(rendered_model)) # pyright: ignore[reportUnknownArgumentType] # the core's host registry + if target_list is None or len(target_list) == 0: # a direct call to the rendered model + _ = final.pop("original_function") + return await core.host.invoke("acompletion")(**final) # pyright: ignore[reportUnknownMemberType] # the core's host + timeouts: Final = { + key: value + for key, value in ( + ("timeout", caller_timeout if caller_timeout is not None else filtered_data.get("timeout")), + ( + "stream_timeout", + caller_stream_timeout if caller_stream_timeout is not None else filtered_data.get("stream_timeout"), + ), + ) + if value is not None + } + routed: Final = without(final, "timeout", "stream_timeout") + return await core.drive(PLANS["acompletion"], {**routed, **timeouts}, prepared_already=True) diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py new file mode 100644 index 00000000000..258f919c9f1 --- /dev/null +++ b/litellm/router_rust/invoke/shaping.py @@ -0,0 +1,635 @@ +""" +The keyword arguments one router attempt sends to one deployment, as the Python router builds them. + +The Rust router core picks the deployment; the host shapes the request for it here +(``_update_kwargs_with_deployment``, ``_update_kwargs_with_default_litellm_params``, +``_merge_tools_from_deployment``, the timeout resolvers and the reasoning-effort override), with +the same precedence and the same oddities. Nothing here reads a Python router. +""" + +import dataclasses +from collections.abc import Mapping, MutableMapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import TypeAdapter, ValidationError + +from litellm import exceptions as litellm_exceptions +from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, +) +from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name +from litellm.litellm_core_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + is_clientside_credential, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values +) +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import without +from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider +from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout +from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin +from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy +from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults +from litellm.router_api import deployment_id as generate_deployment_id +from litellm.router_strategy.tag_based_routing import ( + baggage_stamp_for_serving_deployment, + get_tags_from_request_kwargs, + tags_with_serving_deployment_tags, +) +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + deployment_has_io_token_limits, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + refund_stale_reservation_before_retry, + set_io_token_rate_limit_request_kwargs, +) +from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD +from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.router_python_host import PythonHost + +# The SDK-client and max_parallel_requests reads of the routing cache: process-local, so the family +# only names a Redis span when a read misses through to Redis. +ROUTER_CLIENTS_TARGET: Final = "router_clients" +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthropic"}) +_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +# Generic handlers whose stream timeout is split from the request timeout. +STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Shaping: + """What the request shaping reads from the router, resolved once when the core is built. + + ``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``), + ``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the + router has a timeout of its own, as the Python router keeps them.""" + + default_litellm_params: Mapping[str, object] + timeout: object + explicit_timeout: object + request_timeout: float | None + stream_timeout: object + cache_responses: bool + + +def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping + try: + return dict(_MAPPING.validate_python(value)) + except ValidationError: + return {} + + +def _mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING.validate_python(value) + except ValidationError: + return None + + +def metadata_bucket( + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + name: str, +) -> dict[str, object]: # mutable-ok: the request's own bucket + """``kwargs.setdefault(name, {})``: the request's own metadata dict, created when absent.""" + bucket: Final = kwargs.get(name) + if isinstance(bucket, dict): + return bucket # pyright: ignore[reportUnknownVariableType] # the request's untyped metadata dict + if name not in kwargs: + fresh: Final[dict[str, object]] = {} # mutable-ok: the request kwargs the Python router mutates in place + kwargs[name] = fresh # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + return fresh + # A non-dict value under the key: `setdefault` returns it as is, and `.update` would raise. + raise AttributeError(f"'{type(bucket).__name__}' object has no attribute 'update'") + + +# --- request params the deployment sets -------------------------------------------------------- + + +def _drops_malformed_request_tools( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> bool: + if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"): + return False + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools") + + +def _malformed_request_params_the_deployment_sets( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> frozenset[str]: + overrides: Final = { + key: value + for key, value in request.items() + if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None + } + if not overrides: + return frozenset() + try: + validate_litellm_params_strictly(GenericLiteLLMParams, overrides) + except ValidationError as error: + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + policy: Final = drop_policy({**deployment_params, **set_defaults, **request}) + malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"]) + return frozenset(key for key in malformed if policy.drops(key)) + return frozenset() + + +def _merged_tools( + deployment_tools: Sequence[object], request_tools: object +) -> list[object] | None: # mutable-ok: sent as the request's tools + if not isinstance(request_tools, (list, tuple)): + return None + items: Final[Sequence[object]] = request_tools # pyright: ignore[reportUnknownVariableType] # a request's untyped tools + return [*deployment_tools, *items] if deployment_tools or items else None + + +def merge_tools_from_deployment( + deployment_params: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + defaults: Mapping[str, object], +) -> None: + """``_merge_tools_from_deployment``: deployment tools first, then the request's; the request's + ``tool_choice`` else the deployment's.""" + if _drops_malformed_request_tools(deployment_params, defaults, kwargs): + for name in TOOL_FIELDS: + kwargs.pop(name, None) + deployment_tools: Final = deployment_params.get("tools") or [] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + merged: Final = _merged_tools( + deployment_tools if isinstance(deployment_tools, list) else [], # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped tools + kwargs.get("tools") or [], + ) + if merged is not None: + kwargs["tools"] = merged # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + if "tool_choice" not in kwargs and deployment_params.get("tool_choice") is not None: + kwargs["tool_choice"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + deployment_params["tool_choice"] + ) + + +def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool: + value: Final = deployment_params.get(key) + if value is None: + return False + field: Final = LiteLLM_Params.model_fields.get(key) + return field is None or value != field.default # pyright: ignore[reportAny] # litellm's untyped helpers and request values + + +def _forwarded_alias_marker_keys_the_deployment_sets( + deployment_params: Mapping[str, object], forwarded_keys: object +) -> tuple[str, ...]: + if not isinstance(forwarded_keys, tuple): + return () + keys: Final[tuple[object, ...]] = forwarded_keys # pyright: ignore[reportUnknownVariableType] # the router's own marker + return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key)) + + +# --- reasoning effort ------------------------------------------------------------------------- + + +def _pop_effort_from_nested_carrier(params: dict[str, object], carrier: str) -> None: # mutable-ok: copy-on-write + nested: Final = _mapping(params.get(carrier)) if isinstance(params.get(carrier), Mapping) else None + if nested is None: + return + sanitized: Final = without(nested, "effort") + if sanitized: + params[carrier] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + sanitized + ) + else: + params.pop(carrier, None) + + +def deployment_params_with_request_reasoning_override( + deployment_params: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the call's own copy of the deployment params + """``_deployment_params_with_request_reasoning_override``: a request ``reasoning_effort`` drops + the deployment's ``thinking`` and nested ``effort`` carriers.""" + sanitized: Final = dict(deployment_params) + if request_kwargs.get("reasoning_effort") is None: + return sanitized + sanitized.pop("thinking", None) + _pop_effort_from_nested_carrier(sanitized, "output_config") + _pop_effort_from_nested_carrier(sanitized, "reasoning") + extra_body: Final = sanitized.get("extra_body") + if isinstance(extra_body, Mapping): + sanitized_extra_body: Final = as_dict(extra_body) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + sanitized_extra_body.pop("reasoning_effort", None) + sanitized_extra_body.pop("thinking", None) + _pop_effort_from_nested_carrier(sanitized_extra_body, "output_config") + _pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning") + if sanitized_extra_body: + sanitized["extra_body"] = sanitized_extra_body + else: + sanitized.pop("extra_body", None) + return sanitized + + +def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool: + return any( + isinstance(candidate, Mapping) + and (_mapping(candidate) or {}).get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + for candidate in (kwargs.get("metadata"), kwargs.get("litellm_metadata")) + ) + + +def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + if not isinstance(declared, (list, tuple, set, frozenset)): + return frozenset() + entries: Final[Sequence[object]] = list(declared) # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped allowlist + return frozenset(entry for entry in entries if isinstance(entry, str)) + + +def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if not params: + return True + model_info: Final = as_dict(deployment.get("model_info")) + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model_label: Final = str(params.get("model") or group) + provider_param: Final = params.get("custom_llm_provider") + authenticating: Final = declared_authenticating_provider( + model_label, provider_param if isinstance(provider_param, str) else None + ) + if authenticating is not None: + model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating + else: + model, custom_llm_provider, _, _ = host.invoke( # rebind-ok: rebound as the Python router rebinds it + "get_llm_provider" + )( + model=model_label, + custom_llm_provider=provider_param, + litellm_params=LiteLLM_Params.model_validate(params), + ) + supported: Final = host.invoke("get_supported_openai_params")( + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) + except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + return True + return supported is None or param in supported + + +def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if not params: + return True + if param in _declared_param_allowlist(params): + return True + provider_param: Final = params.get("custom_llm_provider") + if declared_authenticating_provider( + str(params.get("model") or ""), provider_param if isinstance(provider_param, str) else None + ): + return True + return _deployment_supports_param(deployment, group, param, host) + + +def drop_unsupported_classifier_reasoning_effort( + deployment: Mapping[str, object], + model: str, + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + host: "PythonHost", +) -> None: + """``_drop_unsupported_classifier_reasoning_effort``.""" + if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs): + return + if deployment_accepts_param(deployment, model, "reasoning_effort", host): + return + kwargs.pop("reasoning_effort", None) + proxy_server_request: Final = kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, dict): + return + body: Final[object] = proxy_server_request.get("body") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the proxy's untyped request + if isinstance(body, dict): + body.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # same + + +# --- timeouts --------------------------------------------------------------------------------- + + +def stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_stream_timeout``.""" + return ( + kwargs.get("stream_timeout", None) + or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") + or data.get("stream_timeout", None) + or shaping.stream_timeout + or shaping.request_timeout + or shaping.default_litellm_params.get("stream_timeout", None) + ) + + +def non_stream_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_non_stream_timeout``.""" + return ( + kwargs.get("timeout", None) + or kwargs.get("request_timeout", None) + or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") + or data.get("timeout", None) + or data.get("request_timeout", None) + or shaping.request_timeout + or shaping.timeout + or shaping.default_litellm_params.get("timeout", None) + ) + + +def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[str, object]) -> object: + """``_get_timeout``.""" + stream: Final = stream_timeout(shaping, kwargs, data) if kwargs.get("stream") else None + return stream if stream is not None else non_stream_timeout(shaping, kwargs, data) + + +# --- default litellm params ------------------------------------------------------------------- + + +def update_kwargs_with_default_litellm_params( + shaping: Shaping, + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + metadata_variable_name: str, +) -> None: + """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request; + the default metadata is merged over the request's.""" + defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)) + metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + for key, value in defaults.items(): + if value is None: + continue + kwargs.setdefault(key, value) + metadata_bucket(kwargs, metadata_variable_name).update(as_dict(metadata_defaults)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + + +# --- clientside credentials ------------------------------------------------------------------- + + +def _clientside_deployment( + deployment: Mapping[str, object], kwargs: Mapping[str, object], function_name: str | None +) -> tuple[Mapping[str, object], str, object]: + """``_handle_clientside_credential``: the request-scoped model_info, model and api_base.""" + model_info: Final = as_dict(deployment.get("model_info")) + litellm_params: Final = as_dict(deployment.get("litellm_params")) + dynamic_litellm_params: Final = get_dynamic_litellm_params( # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + litellm_params=litellm_params, request_kwargs=dict(kwargs) + ) + metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name) + model_group: Final = as_dict(kwargs.get(metadata_variable_name)).get("model_group") + model_id: Final = generate_deployment_id(model_group, dynamic_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # the group as the request carries it + original_model_id: Final = model_info.get("id") + model_info["id"] = model_id + model_info["original_model_id"] = original_model_id + built: Final = Deployment( + model_name=model_group, # pyright: ignore[reportArgumentType] # the group as the request carries it + litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params), + model_info=model_info, # pyright: ignore[reportArgumentType] # model_info as configured + ) + return built.model_info.model_dump(), built.litellm_params.model, built.litellm_params.api_base + + +# --- the deployment -------------------------------------------------------------------------- + + +def update_kwargs_with_deployment( + shaping: Shaping, + deployment: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + *, + cache: "DualCache | None", + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, +) -> None: + """``_update_kwargs_with_deployment``, in place on the attempt's kwargs.""" + deployment_params: Final = as_dict(deployment.get("litellm_params")) + for key in _forwarded_alias_marker_keys_the_deployment_sets( + deployment_params, kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) + ): + kwargs.pop(key, None) + kwargs.pop("daily_reset_time", None) + defaults: Final = shaping.default_litellm_params + merge_tools_from_deployment(deployment_params, kwargs, defaults) + + model_info: Mapping[str, object] = as_dict( # rebind-ok: rebound as the Python router rebinds it + deployment.get("model_info") + ) + deployment_model: object = deployment_params.get("model") # rebind-ok: same + deployment_api_base: object = deployment_params.get("api_base") # rebind-ok: same + deployment_model_name: Final = deployment.get("model_name") + from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils + + for key in _malformed_request_params_the_deployment_sets(deployment_params, defaults, kwargs): + kwargs.pop(key) + declared_provider: Final = deployment_params.get("custom_llm_provider") + declared_model: Final = deployment_params.get("model") + deployment_provider: Final = ( + declared_provider + if isinstance(declared_provider, str) + else inferred_provider(declared_model if isinstance(declared_model, str) else None) + ) + forwarded_authorization: Final = ( + ProviderSpecificHeaderUtils.forwarded_authorization(kwargs.get("provider_specific_header"), deployment_provider) + if deployment_provider in _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS + else None + ) + if is_clientside_credential(request_kwargs=dict(kwargs)) or forwarded_authorization is not None: + model_info, deployment_model, deployment_api_base = ( # rebind-ok: rebound as the Python router rebinds it + _clientside_deployment( + deployment, + {**kwargs, **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {})}, + function_name, + ) + ) + + metadata_variable_name: Final = get_router_metadata_variable_name(function_name=function_name) + routing: Final = deployment.get(KUBERNETES_POD_ROUTING_KEY) + metadata: Final = metadata_bucket(kwargs, metadata_variable_name) + metadata.update( + { + "deployment": deployment_model, + "model_info": model_info, + "api_base": deployment_api_base, + "deployment_model_name": deployment_model_name, + KUBERNETES_POD_ROUTING_KEY: routing, + } + ) + other_bucket: Final = "metadata" if metadata_variable_name == "litellm_metadata" else "litellm_metadata" + other_bucket_metadata: Final = kwargs.get(other_bucket) + if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata: + kwargs[other_bucket] = {**as_dict(other_bucket_metadata), KUBERNETES_POD_ROUTING_KEY: routing} # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + + if cache is not None: + refund_stale_reservation_before_retry(cache, kwargs) # pyright: ignore[reportArgumentType] # the request's kwargs dict + set_io_token_rate_limit_request_kwargs( + kwargs, # pyright: ignore[reportArgumentType] # the request's kwargs dict + store_in_context=deployment_has_io_token_limits(deployment), # pyright: ignore[reportArgumentType] # a deployment record + opens_responses_socket=opens_responses_socket, + metadata_channel=metadata_variable_name, + ) + metadata.setdefault( + ROUTING_REQUEST_TAGS_METADATA_KEY, + tuple(get_tags_from_request_kwargs(kwargs, metadata_variable_name=metadata_variable_name)), # pyright: ignore[reportArgumentType] # the request's kwargs dict + ) + deployment_tags: Final = deployment_params.get("tags") + merged_tags: Final = tags_with_serving_deployment_tags(metadata, deployment_tags) + if merged_tags is not None: + metadata["tags"] = list(merged_tags) + baggage_stamp: Final = baggage_stamp_for_serving_deployment(metadata, deployment_tags) + if baggage_stamp is not None: + metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] = baggage_stamp + + credential_name: Final = deployment_params.get("litellm_credential_name") + if credential_name: + credential_tag: Final = f"Credential: {credential_name}" + existing_tags: Final = metadata.get("tags") or [] # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values + if isinstance(existing_tags, list) and credential_tag not in existing_tags: + existing_tags.append(credential_tag) # pyright: ignore[reportUnknownMemberType] # the request's own tags list + metadata["tags"] = existing_tags + + from litellm.llms.github_copilot.per_user_auth import github_copilot_per_user_credential_name + + configured_per_user_name: Final = github_copilot_per_user_credential_name(deployment_params) + caller_credential_name: Final = kwargs.get("litellm_credential_name") + if ( + configured_per_user_name is not None + and isinstance(caller_credential_name, str) + and caller_credential_name != configured_per_user_name + ): + raise litellm_exceptions.BadRequestError( + message="litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth", + model=str(deployment_model_name), + llm_provider="", + ) + + kwargs["model_info"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + model_info + ) + _set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout) + + if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)): + kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + STREAM_USAGE_INJECTED_FIELD + ] = True + update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name) + pinned_model_id: Final = deployment_model_id_pin(deployment_params) + if pinned_model_id is not None: + kwargs["model_id"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + pinned_model_id + ) + + +def _set_timeouts( + shaping: Shaping, + deployment_params: Mapping[str, object], + kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place + function_name: str | None, + splits_stream_timeout: bool, +) -> None: + if function_name == "_ageneric_api_call_with_fallbacks": + from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout + + router_timeout: Final = ( + shaping.request_timeout + if shaping.request_timeout is not None + else float(shaping.explicit_timeout) + if isinstance(shaping.explicit_timeout, (int, float)) + else None + ) + router_stream_timeout: Final = ( + shaping.stream_timeout + if shaping.stream_timeout is not None + else shaping.request_timeout + if shaping.request_timeout is not None + else shaping.default_litellm_params.get("stream_timeout") + ) + passthrough_timeout: Final = partial( + resolve_llm_passthrough_timeout, + litellm_params=deployment_params, + router_timeout=router_timeout, + router_stream_timeout=router_stream_timeout, # pyright: ignore[reportArgumentType] # a configured default, passed as the Python router passes it + ) + timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else dict(kwargs) + kwargs.update( + CompletionTimeout.stream_request_kwargs( + passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request) + ) + if splits_stream_timeout and timeout_request.get("stream") + else {"timeout": passthrough_timeout(kwargs=timeout_request)} + ) + elif splits_stream_timeout: + kwargs.update( + CompletionTimeout.stream_request_kwargs( + non_stream_timeout(shaping, kwargs, deployment_params), + stream_timeout(shaping, kwargs, deployment_params) if kwargs.get("stream") else None, + ) + ) + else: + kwargs["timeout"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + call_timeout(shaping, kwargs, deployment_params) + ) + + +def stamp_deployment_num_retries(error: BaseException, deployment: Mapping[str, object]) -> None: + """``_set_deployment_num_retries_on_exception``.""" + if getattr(error, "num_retries", None) is not None: + return + value: Final = as_dict(deployment.get("litellm_params")).get("num_retries") + if value is None: + return + try: + error.num_retries = int(str(value)) if not isinstance(value, int) else value # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + except (ValueError, TypeError): + pass + + +def stamp_failed_deployment_id(error: BaseException, model_info: object) -> None: + """``_set_failed_deployment_id_on_exception`` with the effective model_info.""" + if getattr(error, "failed_deployment_id", None): + return + deployment_id: Final = as_dict(model_info).get("id") + if deployment_id: + try: + error.failed_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # the router stamps the same attribute # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + except Exception: # noqa: BLE001, S110 # as the Python router: a read-only error keeps its attributes + pass + + +def stamp_retry_skip_deployment_id(error: BaseException, kwargs: Mapping[str, object]) -> None: + """``_stamp_retry_skip_deployment_id``.""" + model_info: Final = kwargs.get("model_info") + if not isinstance(model_info, Mapping): + return + info: Final = as_dict(model_info) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + configured: Final = ( + info.get("original_model_id") if isinstance(error, litellm_exceptions.UnsupportedEndpointError) else None + ) + deployment_id: Final = configured or info.get("id") + if isinstance(deployment_id, str) and deployment_id: + error.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read on retry # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them + + +NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) + +__all__ = [ + "STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS", + "Shaping", + "as_dict", + "call_timeout", + "deployment_params_with_request_reasoning_override", + "drop_unsupported_classifier_reasoning_effort", + "merge_tools_from_deployment", + "metadata_bucket", + "stamp_deployment_num_retries", + "stamp_failed_deployment_id", + "stamp_retry_skip_deployment_id", + "update_kwargs_with_default_litellm_params", + "update_kwargs_with_deployment", +] diff --git a/litellm/router_rust/invoke/slots.py b/litellm/router_rust/invoke/slots.py new file mode 100644 index 00000000000..08e88b5b2f1 --- /dev/null +++ b/litellm/router_rust/invoke/slots.py @@ -0,0 +1,107 @@ +""" +A deployment's ``max_parallel_requests`` slot on the Rust router core (``_deployment_slot`` and +``_get_client(client_type="max_parallel_requests")``): the attempt holds one of the deployment's +slots around its provider call, and an open chat stream keeps it until the stream is exhausted, +closed, or falls back. A caller arriving while every slot is taken gets a 429, which the core +records as that attempt's failure. + +The limit object is the one the Python router caches (``MaxParallelRequestsLimit``), kept in the +core's routing cache under the same key, so its in-flight count is shared by every attempt. +""" + +import contextlib +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +from litellm._internal_context import with_service_target +from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET, as_dict + +if TYPE_CHECKING: + from litellm.router_rust.invoke.endpoints import EndpointCore, Plan + +# The attempt helpers that run inside `_deployment_slot` (`_acompletion` holds the same slot +# itself). Sync helpers, rerank, moderation and the search helper hold none. +_SLOTTED_KINDS: Final = frozenset( + { + "chat", + "text", + "embedding", + "image", + "transcription", + "speech", + "adapter", + "generic", + "batch", + "cancel_batch", + "vector_store_create", + } +) + + +@with_service_target(ROUTER_CLIENTS_TARGET) +def max_parallel_requests_limit( + core: "EndpointCore", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> object: + """`_get_client(client_type="max_parallel_requests")`: the cached limit, built on first use + (`InitalizeCachedClient.set_max_parallel_requests_client`).""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + from litellm.utils import calculate_max_parallel_requests + + model_id: Final = as_dict(deployment.get("model_info")).get("id") + key: Final = f"{model_id}_max_parallel_requests_client" + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + cache: Final = core.routing_cache + cached: Final[object] = cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # the cache holds untyped values + if cached is not None: + return cached # pyright: ignore[reportUnknownVariableType] # the cache holds untyped values + params: Final = as_dict(deployment.get("litellm_params")) + calculated: Final = calculate_max_parallel_requests( + rpm=params.get("rpm"), # pyright: ignore[reportArgumentType] # the configured value, as the Python router reads it + max_parallel_requests=params.get("max_parallel_requests"), # pyright: ignore[reportArgumentType] # same + tpm=params.get("tpm"), # pyright: ignore[reportArgumentType] # same + default_max_parallel_requests=core.endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given, as the Python router passes it + ) + if calculated: + cache.set_cache( # pyright: ignore[reportUnknownMemberType] # the routing cache + key=key, + value=MaxParallelRequestsLimit( + max_parallel_requests=calculated, + model_id=str(model_id), + model_group=str(deployment.get("model_name", "")), + ), + local_only=True, + ) + return cache.get_cache(key=key, local_only=True, parent_otel_span=span) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # see above + + +def enter_deployment_slot( + core: "EndpointCore", plan: "Plan", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> contextlib.ExitStack: + """The attempt's slot, entered (a full deployment raises its 429 here); an empty stack when + the plan's helper holds none or the deployment has no limit.""" + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + + slot: Final = contextlib.ExitStack() + if plan.sync or plan.kind not in _SLOTTED_KINDS: + return slot + limit: Final = max_parallel_requests_limit(core, deployment, kwargs) + if isinstance(limit, MaxParallelRequestsLimit): + slot.enter_context(limit) + return slot + + +def deployment_slot( + core: "EndpointCore", deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> contextlib.ExitStack: + """`_deployment_slot` outside a plan's attempt (a file created on every deployment).""" + from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit + + slot: Final = contextlib.ExitStack() + limit: Final = max_parallel_requests_limit(core, deployment, kwargs) + if isinstance(limit, MaxParallelRequestsLimit): + slot.enter_context(limit) + return slot + + +__all__ = ["deployment_slot", "enter_deployment_slot", "max_parallel_requests_limit"] diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py new file mode 100644 index 00000000000..ca70143072f --- /dev/null +++ b/litellm/router_rust/invoke/streaming.py @@ -0,0 +1,1839 @@ +""" +The stream wrappers of the Rust router core: host-side objects around a streamed response. + +Every attempt the Rust core runs that returns a stream is wrapped here before the core sees the +outcome, the way the Python router wraps each attempt's stream: + +- a chat stream (``CustomStreamWrapper``) has its first chunk read inside the attempt, so a stream + that fails before any token fails the attempt and the core retries or falls back as for any error +- a chat, ``/v1/messages`` or Responses stream that fails later, before the caller received content, + hands the failure back to the core: the core decides whether a same-group retry or a fallback takes + over (``RouterCoreHandle.stream_policy`` / ``retry_stream`` / ``fallback_stream``), and the wrapper + splices what that call returns into the stream the caller already holds + +Each wrapper also reports its stream's outcome to the core when the stream ends, as litellm's stream +logging reports it to the Python router's callbacks: the attempt that returned the stream counted +only its request (the attempt answer is ``deferred``), so the success with the stream's final tokens, +or a failure the stream raised after it opened, is recorded then (``StreamAttempt.report_success`` / +``report_failure``). A stream closed before it ends reports nothing. + +The wrappers decide nothing about routing: what may take over, the retry budget and every +deployment pick come from the core. They hold live Python iterators, frames and hidden params, +which is why they live in the host. Ported from the Python router's ``_acompletion_streaming_iterator``, +``_completion_streaming_iterator``, ``_aanthropic_messages_streaming_iterator`` and +``_aresponses_streaming_iterator``, without importing it. +""" + +import asyncio +import contextlib +import dataclasses +import json +import weakref +from collections.abc import ( + AsyncGenerator, + AsyncIterable, + AsyncIterator, + Awaitable, + Callable, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + Sequence, +) +from typing import ( + TYPE_CHECKING, + Final, + Generic, + NoReturn, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # untyped stream iterators and provider chunk lists +) + +import anyio + +import litellm.exceptions as litellm_exceptions +from litellm._logging import verbose_router_logger +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + add_retry_headers_to_response, + get_hidden_params_dict, + replace_complexity_router_headers, +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, without +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_rust.surface.health import failure_already_logged, logged_call +from litellm.types.utils import ModelResponseStream, Usage + +if TYPE_CHECKING: + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import RustCore + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuationOutput, ContinuedPartial + from litellm.types.llms.openai import ResponseAPIUsage, ResponseIncompleteEvent + +# A stream that never emits content or an error could grow a held-frame buffer without bound, so +# hitting either cap commits the stream instead. +MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS: Final = 200 +MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 + +# The hidden params and headers a fallback's response carries, as dicts the proxy writes into. +_Prepared: TypeAlias = "tuple[dict[str, object], dict[str, object]]" +_RestT: Final = TypeVar("_RestT", Iterator[object], AsyncIterator[object]) + + +# --- the attempt a stream came from ------------------------------------------------------------- + + +@dataclasses.dataclass(slots=True) +class StreamAttempt: + """The attempt that returned a stream, and the core that can still recover it. + + ``request`` is the call's prepared request, the one every attempt of the call is built from; + ``kwargs`` are this attempt's own kwargs. The origin fields are what the core reads to decide + what may take over a failure of this stream. + """ + + core: "RustCore" + op: str + request: Mapping[str, object] + kwargs: Mapping[str, object] + group: str + order: int | None + depth: int + level: int + deployment_id: str | None + attempted_retries: int + committed_budget: int | None = None + # The deployment's max_parallel_requests slot the attempt holds; an open chat stream takes it. + slot: contextlib.ExitStack | None = None + # Set by the wrapper that reports the stream's outcome when it ends; the attempt's answer is then + # deferred and counts only the request and the tokens read so far (``counted_tokens``). + reports_outcome: bool = False + counted_tokens: int = 0 + outcome_reported: bool = False + # The background recording of the stream's failure, which a same-group retry waits for so it + # picks with the failed deployment already cooled down. + failure_recording: "asyncio.Task[None] | None" = None + + @property + def model_group(self) -> str: + return self.group + + def origin(self) -> Mapping[str, object]: + return { + "group": self.group, + "order": self.order, + "depth": self.depth, + "level": self.level, + "deployment_id": self.deployment_id, + "attempted_retries": self.attempted_retries, + "committed_budget": self.committed_budget, + } + + def policy(self, trigger: BaseException | None = None) -> Mapping[str, object]: + return self.core.stream_policy(self.request, self.origin(), trigger) + + def can_retry(self) -> bool: + return self.policy().get("can_retry") is True + + def can_fall_back(self) -> bool: + return self.policy().get("can_fall_back") is True + + def refusal_fallback_available(self) -> bool: + return self.policy().get("refusal_fallback") is True + + def retry_budget(self, trigger: BaseException) -> tuple[int, bool]: + policy: Final = self.policy(trigger) + budget: Final = policy.get("retry_budget") + return (budget if isinstance(budget, int) else 0), policy.get("policy_applies") is True + + def should_retry_status(self, status_code: int) -> bool: + return bool(self.core._host.invoke("should_retry")(status_code)) # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # the wrapper serves its own core's host + + async def fallback(self, trigger: BaseException, request: Mapping[str, object] | None = None) -> object: + """The fallback walk for this stream's failure; raises when no fallback answers.""" + return await self.core.recover_stream("fallback", self.op, request or self.request, self.origin(), trigger) + + def fallback_sync(self, trigger: BaseException) -> object: + return self.core.recover_stream_sync("fallback", self.op, self.request, self.origin(), trigger) + + async def retry(self, trigger: BaseException) -> "tuple[object, int, int] | BaseException": + """A same-group retry: the retried response with the retry count and budget, or the newest failure.""" + return await self.core.retry_stream(self.op, self.request, self.origin(), trigger) + + # -- the stream's outcome, reported once when it ends -------------------------------------- + + def logged(self) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + """The logging kwargs litellm's stream logging hands the outcome callbacks, for this attempt.""" + from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY + + kwargs: Final = self.kwargs + metadata: Final = as_str_mapping(kwargs.get("litellm_metadata")) or as_str_mapping(kwargs.get("metadata")) or {} + return { + "call_type": self.op, + "litellm_params": { + "metadata": {**metadata, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: self.counted_tokens}, + "model_info": kwargs.get("model_info") or {}, + "api_base": kwargs.get("api_base"), + "client_side_timeout": kwargs.get("client_side_timeout"), + "timeout": kwargs.get("timeout"), + **({"cooldown_time": kwargs["cooldown_time"]} if "cooldown_time" in kwargs else {}), + }, + } + + def _failure_logged(self, error: BaseException) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + return {**logged_call(self.op, self.kwargs), **self.logged(), "exception": _provider_error(error)} + + def _callbacks(self, *, sync: bool) -> Mapping[str, bool]: + """The outcome callbacks that run for this attempt when its stream ends.""" + return self.core.outcome_callbacks(self.kwargs, sync=sync) + + def _claim(self) -> bool: + if not self.reports_outcome or self.outcome_reported: + return False + self.outcome_reported = True + return True + + def report_success(self, total_tokens: int, last: object = None) -> None: + """The stream ended: count its success and the tokens the attempt did not count, and re-pin + the session on ``last``, the stream's final item (``deployment_callback_on_success``), in the + background.""" + if self._claim(): + logged: Final = {**self.logged(), "standard_logging_object": {"total_tokens": total_tokens}} + _record_in_background(self.core.report_stream_success(logged, last, self._callbacks(sync=False))) + + def report_failure(self, error: BaseException) -> None: + """The stream failed after it opened: count the failure, cool down by the rules and count one + more request (``async_deployment_callback_on_failure``), in the background. A logging object + shared with an earlier failure blocks it: the stream logs through ``attempt_scoped_copy``, read + here, when the stream fails.""" + if self._claim() and not failure_already_logged(self.kwargs, "async_failure"): + self.failure_recording = _record_in_background( + self.core.report_stream_failure(self._failure_logged(error), self._callbacks(sync=False)) + ) + + async def failure_recorded(self) -> None: + """Wait for the stream's failure to be recorded, if one is being recorded.""" + recording: Final = self.failure_recording + if recording is not None and not recording.done(): + _ = await asyncio.wait({recording}) + + def report_first_chunk_failure(self, error: BaseException) -> None: + """The stream failed before its attempt returned it. ``_dispatch_stream_failure`` logs that in + a task of its own, so the record reads the error as the router left it by then, with its + routing note when no real suspension came first. The attempt checked the logging object.""" + _record_in_background(self.core.report_stream_failure(self._failure_logged(error), self._callbacks(sync=False))) + + def report_success_sync(self) -> None: + """A sync stream ended: a sync call's success callback counts the success alone.""" + if self._claim(): + _record_now(lambda: self.core.report_stream_success_sync(self.logged(), self._callbacks(sync=True))) + + def report_failure_sync(self, error: BaseException) -> None: + """A sync stream failed after it opened: counted and cooled down, with no request counted.""" + if self._claim() and not failure_already_logged(self.kwargs, "sync_failure"): + logged: Final = self._failure_logged(error) + _record_now(lambda: self.core.report_stream_failure_sync(logged, self._callbacks(sync=True))) + + +# --- reporting a stream's outcome --------------------------------------------------------------- + +# The background recordings still running, held so none is collected before it finishes. +_RECORDINGS: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: tasks leave it when they finish + + +def _provider_error(error: BaseException) -> BaseException: + """The provider's own error, which the failure rules classify, rather than the router's envelope.""" + if isinstance(error, MidStreamFallbackError) and error.original_exception is not None: + return error.original_exception + return error + + +def _record_in_background(recording: Coroutine[object, object, object]) -> "asyncio.Task[None]": + """Run a recording through the core's async cache driver without holding up the stream, as litellm + schedules its stream logging.""" + + async def run() -> None: + try: + _ = await recording + except Exception as error: # noqa: BLE001 # a recording never fails the stream + verbose_router_logger.warning("a stream's outcome was not recorded: %s", error) + + task: Final = asyncio.get_running_loop().create_task(run()) + _RECORDINGS.add(task) + task.add_done_callback(_RECORDINGS.discard) + return task + + +def _record_now(recording: Callable[[], object]) -> None: + try: + _ = recording() + except Exception as error: # noqa: BLE001 # see _record_in_background + verbose_router_logger.warning("a stream's outcome was not recorded: %s", error) + + +_ItemT: Final = TypeVar("_ItemT") + + +async def _areported( + attempt: StreamAttempt, + items: AsyncIterable[_ItemT], + total_tokens: Callable[[object], int], + observe: Callable[[object], None] | None = None, +) -> AsyncGenerator[_ItemT, None]: + """``items``, reporting the attempt's stream once they end: a success with the tokens + ``total_tokens`` reads off the last item, or the failure they raise. Closed early, nothing is + reported. The wrapper marks the attempt (``reports_outcome``) when it wraps the stream, before + the attempt answers.""" + last: object = None # rebind-ok: the newest item + try: + async for item in items: + last = item + if observe is not None: + observe(item) + yield item + except Exception as error: + attempt.report_failure(error) + raise + attempt.report_success(total_tokens(last), last) + + +def _reported(attempt: StreamAttempt, items: Iterable[_ItemT]) -> Generator[_ItemT, None, None]: + """``_areported`` for a sync stream, whose success counts no tokens.""" + try: + yield from items + except Exception as error: + attempt.report_failure_sync(error) + raise + attempt.report_success_sync() + + +def _usage_total_tokens(usage: object) -> int: + mapping: Final = as_str_mapping(usage) + tokens: Final[object] = mapping.get("total_tokens") if mapping is not None else getattr(usage, "total_tokens", None) + return tokens if isinstance(tokens, int) and not isinstance(tokens, bool) and tokens > 0 else 0 + + +def _chat_total_tokens(last: object) -> int: + """The total tokens of a finished chat stream: its usage chunk's, else the final usage its last + chunk's hidden params carry once the stream assembled its response.""" + return _usage_total_tokens(getattr(last, "usage", None)) or _usage_total_tokens( + (get_hidden_params(last) or {}).get("usage") + ) + + +def _responses_total_tokens(last: object, source: object) -> int: + """The total tokens of a finished Responses stream: its terminal event's response usage.""" + for event in (last, getattr(source, "completed_response", None)): + tokens = _usage_total_tokens(getattr(getattr(event, "response", None), "usage", None)) + if tokens: + return tokens + return 0 + + +class _AnthropicStreamUsage: + """The usage a ``/v1/messages`` stream reports in its frames, as litellm logs it: the prompt's input, + cache-read and cache-write tokens on ``message_start`` (a ``message_delta`` may restate them), and the + cumulative output tokens on each ``message_delta``. A frame split across byte chunks is read once its + line completes.""" + + __slots__ = ("_open_line", "_output", "_prompt") + + def __init__(self) -> None: + self._open_line = b"" + self._prompt: dict[str, int] = {} # mutable-ok: the newest count per prompt field + self._output = 0 + + def observe(self, chunk: object) -> None: + if not isinstance(chunk, (bytes, bytearray)): + return + *lines, self._open_line = (self._open_line + bytes(chunk)).split(b"\n") + for line in lines: + if line.startswith(b"data:") and b'"usage"' in line: + self._read(line[len(b"data:") :]) + + def _read(self, payload: bytes) -> None: + try: + event: Final[object] = json.loads(payload) # pyright: ignore[reportAny] # plain JSON + except ValueError: + return + frame: Final = as_str_mapping(event) or {} + kind: Final = frame.get("type") + usage: Final = as_str_mapping( + (as_str_mapping(frame.get("message")) or {}).get("usage") if kind == "message_start" else frame.get("usage") + ) + if usage is None or kind not in ("message_start", "message_delta"): + return + for field in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"): + count = usage.get(field) + if isinstance(count, int) and not isinstance(count, bool): + self._prompt[field] = count + output: Final = usage.get("output_tokens") + if isinstance(output, int) and not isinstance(output, bool): + self._output = output + + def total_tokens(self, _last: object = None) -> int: + return sum(self._prompt.values()) + self._output + + +# --- the first chunk ---------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True, slots=True) +class _FirstChunk(Generic[_RestT]): + """The chunk an attempt already read off a stream (None: the provider closed it empty) and the + iterator it was read from, which carries on after it.""" + + chunk: object + rest: _RestT + + +def _mid_stream_retry_trigger(error: MidStreamFallbackError) -> Exception: + """The provider's own error, which is what the retry policy and the retry rules classify.""" + return error.original_exception if error.original_exception is not None else error + + +async def _aread_first_chunk(stream: CustomStreamWrapper) -> _FirstChunk[AsyncIterator[object]]: + """Read the first chunk while the attempt still holds the request, so a stream that fails before + any token fails the attempt. Iterates the way ``async for`` does: one ``__aiter__`` call, then + ``__anext__`` on what it returned.""" + rest: Final = stream.__aiter__() + try: + return _FirstChunk(chunk=await rest.__anext__(), rest=rest) + except StopAsyncIteration: + return _FirstChunk(chunk=None, rest=rest) + except MidStreamFallbackError as e: + raise _mid_stream_retry_trigger(e) from e + + +def _read_first_chunk(stream: CustomStreamWrapper) -> _FirstChunk[Iterator[object]]: + rest: Final = iter(stream) + try: + return _FirstChunk(chunk=next(rest), rest=rest) + except StopIteration: + return _FirstChunk(chunk=None, rest=rest) + except MidStreamFallbackError as e: + raise _mid_stream_retry_trigger(e) from e + + +def _chunks_with_first(model_response: Iterable[object], taken: _FirstChunk[Iterator[object]]) -> Iterator[object]: + if taken.chunk is None: + return + yield taken.chunk + yield from taken.rest + + +async def _achunks_with_first( + model_response: AsyncIterable[object], taken: _FirstChunk[AsyncIterator[object]] +) -> AsyncIterator[object]: + if taken.chunk is None: + return + yield taken.chunk + async for item in taken.rest: + yield item + + +# --- shared helpers ----------------------------------------------------------------------------- + + +_CONTENT_DELTA_KEYS: Final = ( + "content", + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", +) + + +def _delta_has_content(delta: object) -> bool: + get: Final[object] = getattr(delta, "get", None) + if not callable(get): + return False + for key in _CONTENT_DELTA_KEYS: + value: object = get(key) + if value: + return True + return False + + +def _stream_chunks_have_generated_content(chunks: Sequence[object]) -> bool: + for chunk in chunks: + choices: object = getattr(chunk, "choices", None) + if not isinstance(choices, Sequence) or not choices: + continue + delta: object = getattr(choices[0], "delta", None) # pyright: ignore[reportUnknownArgumentType] # a chunk choice + if _delta_has_content(delta): + return True + return False + + +def _raise_original_mid_stream_error(e: MidStreamFallbackError) -> NoReturn: + """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError.""" + if e.original_exception is not None: + raise e.original_exception from e + raise e + + +def _raise_fallback_failure(error: Exception) -> NoReturn: + """What a stream's recovery ends on: the error the fallback's own stream raised, or the failure + a walk that served nothing was handed; never the internal MidStreamFallbackError.""" + if isinstance(error, MidStreamFallbackError) and error.original_exception is not None: + raise error.original_exception from error + raise error + + +def _stream_pre_content_error(error: Exception, model: str, llm_provider: str = "anthropic") -> MidStreamFallbackError: + """The envelope the fallback chain judges a failure by when the client has received no content yet.""" + return MidStreamFallbackError( + message=str(error), + model=model, + llm_provider=llm_provider, + original_exception=error, + is_pre_first_chunk=True, + ) + + +def _context_window_stream_fallback_error( + error: "MidStreamFallbackError | litellm_exceptions.ContextWindowExceededError", + model: object, + llm_provider: str, + has_output: bool, +) -> MidStreamFallbackError: + if isinstance(error, litellm_exceptions.ContextWindowExceededError): + if has_output: + raise error + return _stream_pre_content_error(error, model if isinstance(model, str) else "", llm_provider) + if isinstance(error.original_exception, litellm_exceptions.ContextWindowExceededError) and ( + has_output or error.generated_content or error.emitted_disqualifying_content + ): + raise error.original_exception + return error + + +def _is_context_window(error: "MidStreamFallbackError | litellm_exceptions.ContextWindowExceededError") -> bool: + return isinstance(error, litellm_exceptions.ContextWindowExceededError) or isinstance( + error.original_exception, litellm_exceptions.ContextWindowExceededError + ) + + +def _fallback_trigger(e: MidStreamFallbackError, unwrapped: tuple[type[Exception], ...]) -> Exception: + """The error the fallback walk dispatches on: the provider's own for the kinds with a list of their own.""" + return e.original_exception if isinstance(e.original_exception, unwrapped) else e + + +def _prepare_fallback_hidden_params( + fallback_response: object, +) -> _Prepared: + fallback_hidden_params: Final = get_hidden_params_dict(fallback_response) + fallback_headers: Final = fallback_hidden_params.get("additional_headers") + if not isinstance(fallback_headers, dict): + return fallback_hidden_params, {} + return fallback_hidden_params, cast("dict[str, object]", fallback_headers) # cast-ok: a dict of headers + + +def _apply_fallback_hidden_params_to_item( + fallback_item: object, + prepared_fallback_hidden_params: _Prepared, +) -> None: + if fallback_item is None or not hasattr(fallback_item, HIDDEN_PARAMS_ATTR): + return + fallback_hidden_params, fallback_headers = prepared_fallback_hidden_params + item_hidden_params: Final = get_hidden_params_dict(fallback_item) + item_headers: object = item_hidden_params.get("additional_headers") # rebind-ok: replaced when not a dict + if not isinstance(item_headers, dict): + item_headers = {} # rebind-ok: rebound as the stream advances + set_hidden_params( + fallback_item, + { + **item_hidden_params, + **fallback_hidden_params, + "additional_headers": {**cast("dict[str, object]", item_headers), **fallback_headers}, # cast-ok: checked + }, + ) + + +def _combine_fallback_usage(fallback_item: ModelResponseStream, complete_response_object_usage: Usage | None) -> None: + """Merge partial-stream usage with fallback-stream usage on the chunk. A usage given as a plain + dict comes out all-zero: ``combine_usage_objects`` reads token counts as attributes.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + + usage: Final = cast("Usage | None", getattr(fallback_item, "usage", None)) # cast-ok: a chunk's usage + usage_objects: Final = [usage] if usage is not None else [] + if complete_response_object_usage is not None: + usage_objects.append(complete_response_object_usage) + fallback_item.usage = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects) # pyright: ignore[reportAttributeAccessIssue] # usage is not a declared chunk field # rebind-ok: the Python router rewrites the chunk's usage in place + + +# --- mid-stream continuation -------------------------------------------------------------------- + +CONTINUATION_FLAG: Final = "enable_mid_stream_fallback_continuation" +NO_FLAG: Final = object() +# The routing-body key naming the deployments a continuation may not land on (``continuation_deployments``). +CONTINUATION_EXCLUDED_IDS: Final = "continuation_excluded_ids" + + +def continuation_flag(value: object) -> bool: + """``_is_flag_enabled``: True, or a string ``str_to_bool`` reads as True.""" + from litellm.secret_managers.main import str_to_bool + + return value is True or (isinstance(value, str) and str_to_bool(value) is True) + + +def register_continuation_prefill_check(host: "PythonHost") -> None: + """``_register_continuation_prefill_check``: process-global on purpose, as in the Python router.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuationPrefillDeploymentCheck + + host.callbacks.add_litellm_callback(ContinuationPrefillDeploymentCheck()) + + +def continuation_routing(core: "RustCore", request: Mapping[str, object]) -> Mapping[str, object]: + """The routing fields of a continuation request: the deployments that cannot continue it, which the + core leaves out of selection the way ``continuation_deployments`` filters the healthy list.""" + from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + deployment_can_continue, + ) + + if request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER: + return {} + registry: Final = core.registry() + excluded: Final = [ + deployment.model_info.id + for record in registry.all() + if (deployment := registry.deployment(str(_record_id(record)))) is not None + and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + ] + return {"body": {CONTINUATION_EXCLUDED_IDS: excluded}} + + +def _record_id(record: Mapping[str, object]) -> object: + model_info: Final = record.get("model_info") + return model_info.get("id") if isinstance(model_info, Mapping) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a plain record + + +def _names_deployment_that_cannot_continue(core: "RustCore", target: object) -> bool: + from litellm.router_utils.fallback_event_handlers import fallback_target_name + from litellm.router_utils.pre_call_checks.continuation_prefill_check import deployment_can_continue + + name: Final = fallback_target_name(target) + registry: Final = core.registry() + model_names: Final = {record.get("model_name") for record in registry.all()} + if name is None or name in model_names or not registry.has_deployment(name): + return False + deployment: Final = registry.deployment(name) + return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + + +def _mid_stream_continuation_eligible( + attempt: StreamAttempt, e: MidStreamFallbackError, cut_deployment_id: object +) -> bool: + """``_mid_stream_continuation_eligible``: whether a stream that broke after plain assistant text may + continue via prefill.""" + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.router_utils.fallback_event_handlers import ( + continuation_fallback_targets, + fallback_lookup_groups, + fallback_target_overrides_request_params, + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import constrains_continuation + from litellm.types.guardrails import GuardrailEventHooks + + core: Final = attempt.core + if not core.enable_mid_stream_fallback_continuation: + return False + if not e.generated_content or e.emitted_disqualifying_content: + return False + if isinstance(e.original_exception, litellm_exceptions.ContentPolicyViolationError): + return False + if isinstance(cut_deployment_id, str) and not core.registry().has_deployment(cut_deployment_id): + return False + if constrains_continuation(_params_sent_to_deployment(attempt, cut_deployment_id)): + return False + request: Final = attempt.request + requested_group: Final = request.get("model") + lookup_groups: Final = fallback_lookup_groups( + request, requested_group if isinstance(requested_group, str) else None + ) + targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them + if any(fallback_target_overrides_request_params(target) for target in targets): + return False + if any(_names_deployment_that_cannot_continue(core, target) for target in targets): + return False + return not any( + isinstance(callback, CustomGuardrail) + and callback._event_hook_is_event_type( # pyright: ignore[reportPrivateUsage] # the only check of a guardrail's hooks + GuardrailEventHooks.post_call + ) + for callback in core._host.callback_list("callbacks") # pyright: ignore[reportPrivateUsage, reportArgumentType] # the core's own host; read as getattr(litellm, name) + ) + + +def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]: + """Continue the last text block in place so the prefill reads as the text the caller already received.""" + last: Final = blocks[-1] if blocks else None + if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str): + return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}] + return [*blocks, {"type": "text", "text": text}] + + +def _build_completion_continuation_input( + messages: Sequence[Mapping[str, object]], generated_content: str +) -> Sequence[Mapping[str, object]] | None: + """The partial output as an assistant prefill, or a trailing ``prefix: True`` assistant turn extended in + place. None when the request ends in a plain assistant turn, in a prefill turn whose content this cannot + extend, or when the cut came mid-echo of the prefill.""" + from pydantic import TypeAdapter, ValidationError + + from litellm.router_utils.pre_call_checks.continuation_prefill_check import continued_text + + last: Final = messages[-1] if messages else None + if last is None or last.get("role") != "assistant": + return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}] + if last.get("prefix") is not True: + return None + new_text: Final = continued_text(last, generated_content) + if new_text is None: + return None + content: Final = last.get("content") + if content is None or isinstance(content, str): + return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}] + try: + blocks: Final = TypeAdapter(list[dict[str, object]]).validate_python(content) + except ValidationError: + return None + return [*messages[:-1], {**last, "content": _extend_trailing_text_block(blocks, new_text), "prefix": True}] + + +def _price_cut_partial(cut_stream: CustomStreamWrapper, partial_response: object) -> "ContinuedPartial | None": + """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ContinuedPartial + + cut_logging: Final = cut_stream.logging_obj + partial_usage: Final = getattr(partial_response, "usage", None) + if partial_response is None or not isinstance(partial_usage, Usage): + return None + details: Final = cut_logging.model_call_details + inherited_usage: Final = details.get("continued_partial_usage") + inherited_cost: Final = details.get("continued_partial_cost") + inherited_id: Final = details.get("continued_response_id") + priced_partial: Final = partial_response.model_copy( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # a ModelResponse + update={"model": cut_stream.model} if cut_stream.model else {} # pyright: ignore[reportUnknownMemberType] # untyped attribute + ) + priced_partial._hidden_params = without( # pyright: ignore[reportUnknownMemberType] # no public accessor + get_hidden_params_dict(partial_response), "response_cost" + ) + return ContinuedPartial( + usage=BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[partial_usage, inherited_usage] if isinstance(inherited_usage, Usage) else [partial_usage] + ), + response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0) # pyright: ignore[reportUnknownArgumentType] # a ModelResponse + + (inherited_cost if isinstance(inherited_cost, float) else 0.0), + response_id=inherited_id if isinstance(inherited_id, str) else str(getattr(partial_response, "id", "")), + ) + + +def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None: + """Bill what the cut legs consumed on the continuation's success log.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + fallback_logging: Final = getattr(fallback_response, "logging_obj", None) + if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging): + return + fallback_logging.record_continued_partial( + usage=cut_partial.usage, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # a Usage + response_cost=cut_partial.response_cost, + response_id=cut_partial.response_id, + ) + + +@dataclasses.dataclass(frozen=True, slots=True) +class _Continuation: + """What a chat continuation sends the fallback and splices back.""" + + request: Mapping[str, object] + output: "ContinuationOutput" + partial: "ContinuedPartial | None" + usage: Usage | None + + +async def _chat_continuation( + attempt: StreamAttempt, e: MidStreamFallbackError, model_response: CustomStreamWrapper +) -> _Continuation: + """The request a fallback continues a chat stream from, once it broke after content. Raises the + provider's error when the continuation is declined or the output ceiling is spent.""" + from litellm.constants import MID_STREAM_FAILURE_LOG_WAIT_SECONDS + from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG + from litellm.main import ( + stream_chunk_builder, # pyright: ignore[reportUnknownVariableType] # untyped chunk list argument + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + continuation_output_ceilings, + ) + + request: Final = attempt.request + raw_messages: Final = request.get("messages") + messages: Final = cast("list[Mapping[str, object]]", raw_messages or []) # cast-ok: chat messages + complete: Final = stream_chunk_builder(chunks=model_response.chunks, messages=messages) # pyright: ignore[reportUnknownArgumentType] # provider chunk list + usage: Final = cast("Usage | None", getattr(complete, "usage", None)) # cast-ok: a built response's usage + continuation_messages: Final = _build_completion_continuation_input(messages, e.generated_content) + output: Final = ( + ContinuationOutput( + e.generated_content, + continuation_messages[-1], + str(request.get("model") or ""), + complete, + checks_post_call_rules=request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER, + ) + if continuation_messages is not None + else None + ) + partial: Final = _price_cut_partial(model_response, complete) if output is not None else None + if partial is not None: + model_response.logging_obj.mark_stash_billed_by_continuation() + emitted_tokens: Final = int(getattr(usage, "completion_tokens", 0) or 0) + reduced: Final = continuation_output_ceilings(request, emitted_tokens) + if reduced is None or continuation_messages is None or output is None: + _raise_original_mid_stream_error(e) + await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS) + return _Continuation( + request={ + **request, + **reduced, + "messages": continuation_messages, + MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER, + }, + output=output, + partial=partial, + usage=usage, + ) + + +# --- chat completions --------------------------------------------------------------------------- + + +class FallbackAwareStreamWrapper(CustomStreamWrapper): + """A chat stream built around the attempt the core picked first, which repoints itself when a + fallback takes over.""" + + fallback_headers_adopted: bool = False + + def adopt_fallback_response_headers( + self, + fallback_response: object, + prepared_fallback_hidden_params: _Prepared, + ) -> None: + """Repoint this wrapper at the deployment that served the stream. Replaces rather than merges, + so the failed attempt's request id, counters, ``model_id`` and ``api_base`` cannot leak.""" + self._response_headers = getattr(fallback_response, "_response_headers", None) + fallback_hidden_params, fallback_headers = prepared_fallback_hidden_params + if fallback_hidden_params: + self._hidden_params = {**fallback_hidden_params, "additional_headers": dict(fallback_headers)} + self._base_hidden_params = {**self._hidden_params, "response_cost": None} + self.fallback_headers_adopted = True + + +def _adopt_fallback_response_headers( + wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object +) -> _Prepared: + prepared: Final = _prepare_fallback_hidden_params(fallback_response) + adopting_wrapper: Final = wrapper_ref() + if adopting_wrapper is not None: + adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared) + return prepared + + +def _chat_wrapper_init( + wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper +) -> None: + CustomStreamWrapper.__init__( # pyright: ignore[reportUnknownMemberType] # the base constructor is untyped + wrapper, + completion_stream=stream, + model=model_response.model, # pyright: ignore[reportUnknownMemberType] # an untyped attribute + custom_llm_provider=model_response.custom_llm_provider, + logging_obj=model_response.logging_obj, + _response_headers=getattr(model_response, "_response_headers", None), + ) + inner_chunks: Final[object] = getattr(model_response, "chunks", None) + if isinstance(inner_chunks, list): + chunk_list: Final = cast("list[ModelResponseStream]", inner_chunks) # cast-ok: the inner stream's chunks + wrapper.chunks = chunk_list # rebind-ok: initializes the wrapper being built + hidden_params: Final = get_hidden_params(model_response) + if hidden_params is not None: + wrapper._hidden_params = dict(hidden_params) # pyright: ignore[reportPrivateUsage] # the wrapper's own field # rebind-ok: initializes the wrapper being built + + +class _AsyncChatStream(FallbackAwareStreamWrapper): + def __init__(self, generator: AsyncGenerator[object, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it + _chat_wrapper_init(self, generator, model_response) + self._async_generator: Final = generator + + def __aiter__(self) -> "_AsyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator + return self + + async def __anext__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + return await self._async_generator.__anext__() + + +class _SyncChatStream(FallbackAwareStreamWrapper): + def __init__(self, generator: Generator[object, None, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it + _chat_wrapper_init(self, generator, model_response) + self.chunks = [] # the sync twin keeps no inner chunk list + hidden_params: Final = get_hidden_params(model_response) + if hidden_params is not None: + self._hidden_params = dict(hidden_params) + self._sync_generator: Final = generator + + def __iter__(self) -> "_SyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator + return self + + def __next__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + return next(self._sync_generator) + + +def _chat_has_output(model_response: CustomStreamWrapper) -> bool: + return bool(getattr(model_response, "response_uptil_now", "")) or _stream_chunks_have_generated_content( + model_response.chunks + ) + + +def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports + attempt: StreamAttempt, + model_response: CustomStreamWrapper, + first_chunk: _FirstChunk[AsyncIterator[object]], + held_slot: contextlib.ExitStack | None = None, +) -> FallbackAwareStreamWrapper: + """``_acompletion_streaming_iterator``: a mid-stream error before content falls back through the core; + after content, the provider's error is raised. ``held_slot`` (the deployment's max_parallel_requests + slot) is released when the stream is exhausted, closed, or falls back.""" + slot: Final = held_slot if held_slot is not None else contextlib.ExitStack() + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + async def close_model_response() -> None: + if not hasattr(model_response, "aclose"): + return + try: + await model_response.aclose() + except BaseException as e: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks: error closing model_response: %s", e) + + async def stream_with_fallbacks() -> AsyncGenerator[object, None]: + fallback_response: object = None # rebind-ok: tracked for cleanup + try: + async for item in _areported(attempt, _achunks_with_first(model_response, first_chunk), _chat_total_tokens): + yield item + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + _chat_has_output(model_response), + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + with anyio.CancelScope(shield=True): + await close_model_response() + slot.close() + committed: Final = bool( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ) + cut_deployment_id: Final = (get_hidden_params(model_response) or {}).get("model_id") + if committed and not _mid_stream_continuation_eligible(attempt, e, cut_deployment_id): + _raise_original_mid_stream_error(e) + trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,)) + try: + continuation: Final = await _chat_continuation(attempt, e, model_response) if committed else None + fallback_response = await attempt.fallback( # rebind-ok: rebound as the stream advances + trigger, continuation.request if continuation is not None else None + ) + if hasattr(fallback_response, "__aiter__"): + if continuation is not None: + _carry_partial_spend_to_fallback(fallback_response, continuation.partial) + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted + settled = False # rebind-ok: rebound as the stream advances + fallback_stream: Final = cast("AsyncIterable[object]", fallback_response) # cast-ok: checked + fallback_items: Final = ( + continuation.output.splice(fallback_stream) if continuation is not None else fallback_stream + ) + async for fallback_item in fallback_items: + if not settled: + settled = True + # a fallback that failed over again only repoints itself once it yields + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if ( + fallback_item + and isinstance(fallback_item, ModelResponseStream) + and hasattr(fallback_item, "usage") + ): + _combine_fallback_usage( + fallback_item, + continuation.usage if continuation is not None else None, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # a Usage + ) + yield fallback_item + else: + # A fallback that answers without a stream yields None (an oddity kept as is). + yield None + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + finally: + with anyio.CancelScope(shield=True): + await close_model_response() + slot.close() + if fallback_response is not None and hasattr(fallback_response, "aclose"): + try: + await fallback_response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except BaseException as close_error: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug( + "stream_with_fallbacks: error closing fallback_response: %s", close_error + ) + + wrapped_response: Final = _AsyncChatStream(stream_with_fallbacks(), model_response) + # weak, so the generator closing over it does not keep the wrapper alive + wrapper_ref: Final[weakref.ref[FallbackAwareStreamWrapper]] = weakref.ref(wrapped_response) + return wrapped_response + + +def wrap_chat_stream_sync( + attempt: StreamAttempt, model_response: CustomStreamWrapper, first_chunk: _FirstChunk[Iterator[object]] +) -> FallbackAwareStreamWrapper: + """``_completion_streaming_iterator``: the sync twin, which never continues after content.""" + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + def stream_with_fallbacks() -> Generator[object, None, None]: + fallback_response: object = None # rebind-ok: tracked for cleanup + try: + yield from _reported(attempt, _chunks_with_first(model_response, first_chunk)) + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + _chat_has_output(model_response), + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + if attempt.request.get("disable_fallbacks") is True or ( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ): + _raise_original_mid_stream_error(e) + trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,)) + try: + fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances + if hasattr(fallback_response, "__iter__"): + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted + settled = False # rebind-ok: rebound as the stream advances + for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked + if not settled: + settled = True + prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if ( + fallback_item + and isinstance(fallback_item, ModelResponseStream) + and hasattr(fallback_item, "usage") + ): + _combine_fallback_usage(fallback_item, None) + yield fallback_item + else: + yield None + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + finally: + for closable in (model_response, fallback_response): + close = getattr(closable, "close", None) + if callable(close): + try: + close() + except BaseException as close_error: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks: error closing: %s", close_error) + + wrapped_response: Final = _SyncChatStream(stream_with_fallbacks(), model_response) + wrapper_ref: Final[weakref.ref[FallbackAwareStreamWrapper]] = weakref.ref(wrapped_response) + return wrapped_response + + +# --- /v1/messages ------------------------------------------------------------------------------- + + +class FallbackAwareAnthropicMessagesStream: + """A ``/v1/messages`` byte stream that carries the ``_hidden_params`` the proxy reads its response + headers off, through retries and fallbacks of the stream it wraps.""" + + def __init__( + self, async_generator: AsyncGenerator[bytes, None], source_iterator: object, attempt: StreamAttempt + ) -> None: + self._async_generator: Final = async_generator + self._source_iterator: object = source_iterator + self.attempt: Final = attempt + self.fallback_headers_adopted = False + self._hidden_params: dict[str, object] = dict( # mutable-ok: the proxy writes headers into it + getattr(source_iterator, "_hidden_params", None) or {} + ) + self._followed_source_params: object = None + + @property + def has_buffered_provider_output(self) -> bool: + return getattr(self._source_iterator, "has_buffered_provider_output", False) is True + + @property + def chunks(self) -> object: + return getattr(self._source_iterator, "chunks", None) + + @property + def messages(self) -> object: + return getattr(self._source_iterator, "messages", None) + + @property + def model(self) -> object: + return getattr(self._source_iterator, "model", None) + + def adopt_fallback_source(self, fallback_response: object) -> None: + self._source_iterator = fallback_response + self.fallback_headers_adopted = True + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + return partial_responses_incomplete_event_on_disconnect(self._source_iterator) + + def follow_source_attribution(self) -> None: + """Mirror the source's current attribution onto this wrapper whenever the source has adopted a + new one: a retry's or a fallback's stream carries a wrapper of its own.""" + source: Final = self._source_iterator + if not getattr(source, "fallback_headers_adopted", False): + return + source_params: Final = getattr(source, "_hidden_params", None) + if source_params is None or source_params is self._followed_source_params: + return + self._followed_source_params = source_params + hidden_params, headers = _prepare_fallback_hidden_params(source) + self.merge_fallback_hidden_params(hidden_params, headers) + + def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": + return self + + async def __anext__(self) -> bytes: + return await self._async_generator.__anext__() + + async def aclose(self) -> None: + await self._async_generator.aclose() + + def merge_fallback_hidden_params( + self, fallback_hidden_params: Mapping[str, object], fallback_headers: Mapping[str, object] + ) -> None: + existing_headers: Final = cast( # cast-ok: additional_headers is a dict when present + "dict[str, object]", self._hidden_params.get("additional_headers") or {} + ) + self._hidden_params = { + **self._hidden_params, + **fallback_hidden_params, + "additional_headers": dict(replace_complexity_router_headers(existing_headers, fallback_headers)), + } + + +def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: bool) -> bool: + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_ping_chunk + + return not has_generated_content and is_anthropic_ping_chunk(chunk) + + +def _is_retriable_anthropic_status(status_code: int) -> bool: + return status_code == 429 or status_code >= 500 + + +def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool: + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import is_server_fulfilled_tool_leak_error + + return is_server_fulfilled_tool_leak_error(chunk) + + +def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error: MidStreamFallbackError) -> bool: + return has_generated_content or not error.is_pre_first_chunk + + +def _stream_raised_error_status(error: Exception) -> int | None: + raw_status: Final = getattr(error, "status_code", None) + if isinstance(raw_status, int): + return raw_status + if isinstance(raw_status, str) and raw_status.isdigit(): + return int(raw_status) + response_status: Final = getattr(getattr(error, "response", None), "status_code", None) + return response_status if isinstance(response_status, int) else None + + +def _stream_fallback_error_for_raised( + attempt: StreamAttempt, error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" +) -> MidStreamFallbackError | None: + """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + if has_generated_content: + return None + status_code: Final = _stream_raised_error_status(error) + if status_code is None or attempt.should_retry_status(status_code): + return _stream_pre_content_error(error, model, llm_provider) + return None + + +def _stream_commits_now(chunk: object, has_generated_content: bool, buffered_chunk_count: int) -> bool: + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_content_delta_chunk + + if has_generated_content: + return False + return is_anthropic_content_delta_chunk(chunk) or buffered_chunk_count >= MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS + + +def _bridged_stream_failure(source: object) -> Exception | None: + """The provider's error behind an ``event: error`` frame the completion or Responses bridge wrote + in its place, which litellm's stream logging logged. A native frame has none: litellm logs what the + frame itself maps to (``anthropic_error_frame_exception``), once a takeover closes the stream.""" + failure: Final[object] = getattr(source, "stream_failure", None) + return failure if isinstance(failure, Exception) else None + + +def _anthropic_recoverable_frame_error( + attempt: StreamAttempt, + error_event: tuple[str, str, int] | None, + chunk: object, + has_generated_content: bool, + model_group: str, +) -> Exception | None: + """The exception a provider ``event: error`` frame before content recovers through when a retry of + its class or a fallback can still take over; None forwards the frame as the provider sent it.""" + from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception + + if has_generated_content or error_event is None: + return None + error_type, message, status_code = error_event + if not _is_retriable_anthropic_status(status_code) or _anthropic_stream_error_is_gateway_verdict(chunk): + return None + frame_error: Final = anthropic_error_frame_exception(error_type, message, status_code, model_group) + budget, _ = attempt.retry_budget(frame_error) + if budget > attempt.attempted_retries: + return frame_error + if attempt.can_fall_back(): + return frame_error + return None + + +def wrap_anthropic_stream( + attempt: StreamAttempt, response: AsyncIterator[bytes] +) -> FallbackAwareAnthropicMessagesStream: + """``_aanthropic_messages_streaming_iterator``: lifecycle frames are held while a retry or a fallback + can still take over; an error frame, a raised error or a safeguard refusal before content is + retried in the group, then falls back, as the core decides.""" + from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + aclose_if_supported, + aclose_superseded_attempt, + parse_anthropic_error_event, + parse_anthropic_refusal_stop_details, + ) + from litellm.llms.anthropic.messages.native.utils import safeguard_refusal_error + + source_iterator: Final = response + replays_a_finished_agentic_turn: Final = isinstance(source_iterator, FakeAnthropicMessagesStreamIterator) + model: Final = attempt.group + usage: Final = _AnthropicStreamUsage() + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + + async def close_source() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_if_supported(source_iterator) + + async def close_superseded() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_superseded_attempt(source_iterator) + + async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: + policy: Final = attempt.policy() + has_generated_content = not ( # rebind-ok: set once content is seen, the buffer cap is hit, or nothing can take over + policy.get("can_retry") is True or policy.get("can_fall_back") is True + ) + buffered: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline + try: + async for chunk in _areported(attempt, source_iterator, usage.total_tokens, usage.observe): + if _anthropic_stream_forwards_ping_live(chunk, has_generated_content): + yield chunk + continue + if _stream_commits_now(chunk, has_generated_content, len(buffered)): + has_generated_content = True + # A transport can split one SSE data line across byte chunks, so pre-content detection + # parses the held frames plus the current chunk. + parse_window = ( + b"".join(c for c in (*buffered, chunk) if isinstance(c, (bytes, bytearray))) # pyright: ignore[reportUnnecessaryIsInstance] # bridge-path chunks are not always bytes + if not has_generated_content and isinstance(chunk, (bytes, bytearray)) # pyright: ignore[reportUnnecessaryIsInstance] # see above + else chunk + ) + error_event = parse_anthropic_error_event(parse_window) + bridged_failure = _bridged_stream_failure(source_iterator) if error_event is not None else None + if bridged_failure is not None: + attempt.report_failure(bridged_failure) + recoverable_frame_error = _anthropic_recoverable_frame_error( + attempt, error_event, chunk, has_generated_content, model + ) + refusal_stop_details = ( + parse_anthropic_refusal_stop_details(parse_window) + if not has_generated_content and error_event is None + else None + ) + if ( + refusal_stop_details is not None + and not replays_a_finished_agentic_turn + and attempt.refusal_fallback_available() + ): + refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) + raise MidStreamFallbackError( + message=refusal_error.message, + model=model, + llm_provider="anthropic", + original_exception=refusal_error, + is_pre_first_chunk=True, + ) + if not has_generated_content and error_event is None: + buffered = (*buffered, chunk) + continue + if recoverable_frame_error is not None and error_event is not None: + raise MidStreamFallbackError( + message=error_event[1], + model=model, + llm_provider="anthropic", + original_exception=recoverable_frame_error, + is_pre_first_chunk=True, + ) + for buffered_chunk in buffered: + yield buffered_chunk + buffered = () + yield chunk + for buffered_chunk in buffered: + yield buffered_chunk + except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + async for item in _anthropic_recover_stream_error( + attempt, stream_error, has_generated_content, buffered, model, wrapper, close_superseded + ): + yield item + finally: + await close_source() + + wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator, attempt) + return wrapper + + +async def _anthropic_recover_stream_error( + attempt: StreamAttempt, + stream_error: Exception, + has_generated_content: bool, + buffered: tuple[bytes, ...], + model: str, + wrapper: FallbackAwareAnthropicMessagesStream, + close_superseded: Callable[[], Awaitable[None]], +) -> AsyncGenerator[bytes, None]: + """Turns a source-iterator failure into a retry, a fallback, or the error reaching the caller.""" + if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, stream_error + ): + for buffered_chunk in buffered: + yield buffered_chunk + if stream_error.original_exception is not None: + raise stream_error.original_exception from stream_error + raise stream_error + fallback_error: Final = ( + stream_error + if isinstance(stream_error, MidStreamFallbackError) + else _stream_fallback_error_for_raised(attempt, stream_error, model, has_generated_content) + ) + if fallback_error is None: + raise stream_error + # A native error frame or a refusal the wrapper raised fails the attempt too (a raised error or a + # bridged frame already did). + attempt.report_failure(fallback_error) + await close_superseded() + await attempt.failure_recorded() + retried: Final = await attempt.retry(_mid_stream_retry_trigger(fallback_error)) + if isinstance(retried, BaseException): + last_error: Final = ( + fallback_error + if retried is _mid_stream_retry_trigger(fallback_error) + else _stream_pre_content_error( + retried if isinstance(retried, Exception) else fallback_error, model, fallback_error.llm_provider + ) + ) + async for item in _anthropic_fallback_attempt(attempt, last_error, wrapper): + yield item + return + response, attempted, budget = retried + inner: Final = response + if isinstance(inner, FallbackAwareAnthropicMessagesStream): + inner.attempt.attempted_retries = attempted + inner.attempt.committed_budget = budget + async for item in _anthropic_yield_recovered(inner, wrapper, (attempted, budget)): + yield item + + +async def _anthropic_yield_recovered( + recovered: object, + wrapper: FallbackAwareAnthropicMessagesStream, + retry_counters: tuple[int, int] | None = None, +) -> AsyncGenerator[bytes, None]: + """Hands a retry's or a fallback's response to the client through the wrapper, closing it after.""" + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + aclose_if_supported, + anthropic_messages_response_as_sse_events, + ) + + hidden_params, headers = _prepare_fallback_hidden_params(recovered) + wrapper.merge_fallback_hidden_params(hidden_params, headers) + wrapper.adopt_fallback_source(recovered) + if retry_counters is not None: + add_retry_headers_to_response( + response=wrapper, attempted_retries=retry_counters[0], max_retries=retry_counters[1] + ) + try: + if hasattr(recovered, "__aiter__"): + async for item in cast("AsyncIterator[bytes]", recovered): # cast-ok: __aiter__ checked above + wrapper.follow_source_attribution() + yield item + return + # A complete response for a streaming request is replayed as the SSE lifecycle a stream sends. + for event in anthropic_messages_response_as_sse_events(recovered): # pyright: ignore[reportArgumentType] # the non-streaming shape by elimination + yield event + finally: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_if_supported(recovered) + + +async def _anthropic_fallback_attempt( + attempt: StreamAttempt, e: MidStreamFallbackError, wrapper: FallbackAwareAnthropicMessagesStream +) -> AsyncGenerator[bytes, None]: + """Re-enters the fallback walk for a ``/v1/messages`` stream error and yields what the fallback produces.""" + trigger: Final = _fallback_trigger( + e, (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError) + ) + try: + fallback_response: Final = await attempt.fallback(trigger) + async for fallback_item in _anthropic_yield_recovered(fallback_response, wrapper): + yield fallback_item + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Anthropic messages streaming fallback also failed: %s", fallback_error) + _raise_fallback_failure(fallback_error) + + +# --- Responses API ------------------------------------------------------------------------------ + + +def _responses_stream_holds_event(item: object, held_event_count: int) -> bool: + from litellm.responses.streaming_iterator import PRE_OUTPUT_LIFECYCLE_EVENT_TYPES + + if held_event_count >= MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: + return False + return getattr(item, "type", None) in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES + + +def _responses_event_announces_background_run(item: object) -> bool: + return getattr(getattr(item, "response", None), "background", None) is True + + +def _mid_stream_error_before_output(error: MidStreamFallbackError) -> bool: + return not error.generated_content or error.is_pre_first_chunk + + +def _extract_partial_responses_usage(source_iterator: "BaseResponsesAPIStreamingIterator") -> "ResponseAPIUsage | None": + """Partial token usage of a Responses stream that failed, normalized to ``ResponseAPIUsage``.""" + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ) + + if isinstance(source_iterator, LiteLLMCompletionStreamingIterator): + chunks: Final = source_iterator.collected_chat_completion_chunks + if chunks: + try: + from litellm.main import ( + stream_chunk_builder, # pyright: ignore[reportUnknownVariableType] # untyped chunk list + ) + + built: Final = stream_chunk_builder(chunks=chunks) + chat: Final[object | None] = getattr(built, "usage", None) if built is not None else None + if chat is not None: + prompt: Final = int(getattr(chat, "prompt_tokens", 0) or 0) + completion: Final = int(getattr(chat, "completion_tokens", 0) or 0) + total: Final = int(getattr(chat, "total_tokens", prompt + completion) or (prompt + completion)) + return ResponseAPIUsage(input_tokens=prompt, output_tokens=completion, total_tokens=total) + except Exception as error: # noqa: BLE001 # the builder is best-effort + verbose_router_logger.debug("partial usage of a failed Responses stream: %s", error) + completed: Final = source_iterator.completed_response + if isinstance(completed, (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent)): + return completed.response.usage + return None + + +def _combine_responses_fallback_usage(fallback_item: object, partial_usage: "ResponseAPIUsage") -> None: + from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ) + + if not isinstance(fallback_item, (ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent)): + return + response: Final = fallback_item.response + if response.usage is None: + return + fb: Final = response.usage + response.usage = ResponseAPIUsage( + input_tokens=(partial_usage.input_tokens or 0) + (fb.input_tokens or 0), + output_tokens=(partial_usage.output_tokens or 0) + (fb.output_tokens or 0), + total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0), + ) + + +def _params_sent_to_deployment(attempt: StreamAttempt, deployment_id: object) -> Mapping[str, object]: + """The request as the deployment received it, merged in the order the attempt applies.""" + return attempt.core.params_sent_to_deployment(attempt.request, deployment_id) + + +def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it ports + attempt: StreamAttempt, response: "BaseResponsesAPIStreamingIterator" +) -> "BaseResponsesAPIStreamingIterator": + """``_aresponses_streaming_iterator``: pre-output lifecycle events are held while a fallback can take + over; a mid-stream error falls back through the core, continuing after text.""" + import time + from datetime import datetime, timezone + + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, get_openai_response_types + + source_iterator: Final = response + attempt.reports_outcome = True # rebind-ok: the wrapper reports this attempt's outcome + openai_types: Final = get_openai_response_types() + terminal_event_types: Final = ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, + ) + + class FallbackResponsesStreamWrapper(BaseResponsesAPIStreamingIterator): + """Subclasses the base iterator only for isinstance checks; iteration is the generator's.""" + + fallback_headers_adopted: bool = False + + def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own + self._async_generator: Final = async_generator + self._billing_source: object = source_iterator + self._last_sequence_number = -1 + self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source + self.model = getattr(source_iterator, "model", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.logging_obj = getattr( + source_iterator, "logging_obj", getattr(source_iterator, "litellm_logging_obj", None) + ) # pyright: ignore[reportAttributeAccessIssue] # see above + self.finished = False + self.responses_api_provider_config = getattr(source_iterator, "responses_api_provider_config", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.completed_response = None + self.start_time = getattr(source_iterator, "start_time", None) or datetime.now(timezone.utc).replace( + tzinfo=None + ) + self._failure_handled = False + self._yielded_first_chunk = False + self._generated_content = "" + self._completed_response_cached = False + self._completed_response_logged = False + self._completed_response_cache_hit = None + self._persist_completed_response_before_logging = True + self._stream_created_time = time.time() + self.litellm_metadata = getattr(source_iterator, "litellm_metadata", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.custom_llm_provider = getattr(source_iterator, "custom_llm_provider", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self.request_data = getattr(source_iterator, "request_data", {}) or {} + self.call_type = getattr(source_iterator, "call_type", None) # pyright: ignore[reportAttributeAccessIssue] # see above + self._hidden_params = dict(getattr(source_iterator, "_hidden_params", None) or {}) + + def adopt_fallback_headers(self, fallback_response: object) -> _Prepared: + prepared: Final = _prepare_fallback_hidden_params(fallback_response) + self._billing_source = fallback_response + self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} + self.fallback_headers_adopted = True + return prepared + + def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None": + from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect + + current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source) + if self._billing_source is source_iterator: + return current + primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator) + if current is None or primary is None: + return current or primary + if primary.response.usage is not None: + _combine_responses_fallback_usage(current, primary.response.usage) + return current + + def __aiter__(self) -> "FallbackResponsesStreamWrapper": + return self + + @property + def yielded_first_chunk(self) -> bool: + return self._yielded_first_chunk + + async def __anext__(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # yields what the stream yields + try: + chunk: Final = await self._async_generator.__anext__() + except StopAsyncIteration: + if self.completed_response is None: + self.completed_response = getattr(source_iterator, "completed_response", None) + raise + if self.completed_response is None and getattr(chunk, "type", None) in terminal_event_types: + self.completed_response = chunk # pyright: ignore[reportAttributeAccessIssue] # the terminal event + self._yielded_first_chunk = True + sequence_number: Final = getattr(chunk, "sequence_number", None) + if not isinstance(sequence_number, int): + return chunk + if not self.fallback_headers_adopted: + self._last_sequence_number = max(self._last_sequence_number, sequence_number) + return chunk + self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number) + if self._last_sequence_number == sequence_number: + return chunk + return chunk.model_copy(update={"sequence_number": self._last_sequence_number}) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # a response event, the only item with a sequence_number + + async def aclose(self) -> None: + await self._async_generator.aclose() + + def background_stream_can_release(item: object) -> bool: + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + if ( + type(source_iterator) is not ResponsesAPIStreamingIterator + or source_iterator.custom_llm_provider != "openai" + ): + return False + if not _responses_event_announces_background_run(item): + return False + return not attempt.can_fall_back() + + async def stream_with_fallbacks() -> AsyncGenerator[object, None]: + held: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback + response_id_exposed = False # rebind-ok: set once, before an early-released event is yielded + try: + async for item in _areported( + attempt, + cast("AsyncIterator[object]", source_iterator), # cast-ok: an async stream + lambda last: _responses_total_tokens(last, source_iterator), + ): + if not response_id_exposed and background_stream_can_release(item): + response_id_exposed = True + if not response_id_exposed and _responses_stream_holds_event(item, len(held)): + held = (*held, item) + continue + for held_event in held: + yield held_event + held = () + yield item + for held_event in held: + yield held_event + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + source_iterator.model or "", + source_iterator.custom_llm_provider or "", + wrapper.yielded_first_chunk, + ) + if _is_context_window(stream_error) + else cast("MidStreamFallbackError", stream_error) # cast-ok: not a context window error + ) + if response_id_exposed: + _raise_original_mid_stream_error(e) + async with contextlib.aclosing( + _responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held) + ) as fallback_stream: + async for fallback_item in fallback_stream: + yield fallback_item + except Exception: + for held_event in held: + yield held_event + raise + finally: + with anyio.CancelScope(shield=True): + if hasattr(source_iterator, "aclose"): + try: + await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing source: %s", exc) + + wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks()) + return wrapper + + +def _responses_continuation_request( + attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator" +) -> Mapping[str, object] | None: + """The request a fallback continues from once a Responses stream broke after text, or None before + output. Raises the primary's error when the output is constrained, a tool call began, a content + filter cut it, or the output ceiling is spent.""" + from litellm.litellm_core_utils.token_counter import ( + token_counter, # pyright: ignore[reportUnknownVariableType] # untyped tokenizer argument + ) + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + RESPONSES_CONTINUATION_KWARG, + ResponsesContinuation, + constrains_responses_continuation, + continuation_output_ceilings, + responses_continuation_input, + ) + + if _mid_stream_error_before_output(e): + return None + if source_iterator.started_tool_call or isinstance( + e.original_exception, litellm_exceptions.ContentPolicyViolationError + ): + _raise_original_mid_stream_error(e) + cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") + if constrains_responses_continuation(_params_sent_to_deployment(attempt, cut_deployment_id)): + _raise_original_mid_stream_error(e) + request: Final = attempt.request + earlier: Final = request.get(RESPONSES_CONTINUATION_KWARG) + continuation: Final = ( + earlier.extended(e.generated_content) + if isinstance(earlier, ResponsesContinuation) + else ResponsesContinuation(request.get("input"), e.generated_content) + ) + emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) + reduced: Final = continuation_output_ceilings(request, emitted_tokens) + if reduced is None: + _raise_original_mid_stream_error(e) + return { + **request, + **reduced, + RESPONSES_CONTINUATION_KWARG: continuation, + "input": responses_continuation_input(continuation), + } + + +async def _responses_fallback_attempt( + attempt: StreamAttempt, + e: MidStreamFallbackError, + source_iterator: "BaseResponsesAPIStreamingIterator", + adopt_headers: Callable[[object], _Prepared], + held: tuple[object, ...], +) -> AsyncGenerator[object, None]: + """Re-enters the fallback walk for a mid-stream Responses error. The primary's held lifecycle events + reach the client only when no fallback lands.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ResponsesContinuationSeam + + continued: Final = _responses_continuation_request(attempt, e, source_iterator) + seam: Final = ResponsesContinuationSeam(e.generated_content if continued is not None else "") + partial_usage: Final = _extract_partial_responses_usage(source_iterator) + trigger: Final = _fallback_trigger( + e, (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError) + ) + fallback_response: object = None # rebind-ok: set once a fallback answered, closed at the end + fallback_yielded = False # rebind-ok: flipped on the first fallback item + try: + fallback_response = await attempt.fallback(trigger, continued) # rebind-ok: rebound as the stream advances + prepared: Final = adopt_headers(fallback_response) + if hasattr(fallback_response, "__aiter__"): + spliced: Final = cast( # cast-ok: the fallback's events + "AsyncIterator[object]", + seam.splice(fallback_response), # pyright: ignore[reportArgumentType] # an async stream + ) + async for fallback_item in spliced: + _apply_fallback_hidden_params_to_item(fallback_item, prepared) + if partial_usage is not None: + _combine_responses_fallback_usage(fallback_item, partial_usage) + fallback_yielded = True + yield fallback_item + else: + fallback_yielded = True # rebind-ok: rebound as the stream advances + yield fallback_response + except Exception as fallback_error: # noqa: BLE001 # as the Python wrapper: logged, then the stream's error raises + verbose_router_logger.error("Responses streaming fallback also failed: %s", fallback_error) + if not fallback_yielded: + for held_event in held: + yield held_event + _raise_fallback_failure(fallback_error) + finally: + if fallback_response is not None and hasattr(fallback_response, "aclose"): + with anyio.CancelScope(shield=True): + try: + await fallback_response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing fallback: %s", exc) + + +# --- the attempt hook --------------------------------------------------------------------------- + + +async def open_attempt_stream(attempt: StreamAttempt, response: object) -> object: + """What an attempt returns once its stream is wrapped: a chat stream's first chunk is read here, so + a stream that fails before any token raises inside the attempt.""" + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + if attempt.op == "acompletion" and isinstance(response, CustomStreamWrapper): + if response.completion_stream is None and response.make_call is not None: # pyright: ignore[reportUnknownMemberType] # untyped attributes + await response.fetch_stream() + first_chunk: Final = await _aread_first_chunk(response) + held_slot: Final = attempt.slot.pop_all() if attempt.slot is not None else contextlib.ExitStack() + return wrap_chat_stream(attempt, response, first_chunk, held_slot) + if not attempt.kwargs.get("stream"): + return response + if attempt.op == "aresponses" and isinstance(response, BaseResponsesAPIStreamingIterator): + return wrap_responses_stream(attempt, response) + if attempt.op in ("aanthropic_messages", "anthropic_messages") and hasattr(response, "__aiter__"): + return wrap_anthropic_stream(attempt, cast("AsyncIterator[bytes]", response)) # cast-ok: a byte stream + return response + + +def open_attempt_stream_sync(attempt: StreamAttempt, response: object) -> object: + if attempt.op == "completion" and isinstance(response, CustomStreamWrapper): + if response.completion_stream is None and response.make_call is not None: # pyright: ignore[reportUnknownMemberType] # untyped attributes + response.fetch_sync_stream() + return wrap_chat_stream_sync(attempt, response, _read_first_chunk(response)) + return response + + +def shape_attempt_kwargs(kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> Mapping[str, object]: + """A Responses continuation is shaped for the deployment the fallback landed on.""" + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + RESPONSES_CONTINUATION_KWARG, + responses_continuation_input_for, + ) + + if RESPONSES_CONTINUATION_KWARG not in kwargs: + return kwargs + shaped: Final = without(kwargs, RESPONSES_CONTINUATION_KWARG) + shaped["input"] = responses_continuation_input_for(kwargs, deployment) + return shaped + + +__all__ = [ + "MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS", + "MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS", + "FallbackAwareAnthropicMessagesStream", + "FallbackAwareStreamWrapper", + "StreamAttempt", + "open_attempt_stream", + "open_attempt_stream_sync", + "shape_attempt_kwargs", +] diff --git a/litellm/router_rust/invoke/walk.py b/litellm/router_rust/invoke/walk.py new file mode 100644 index 00000000000..4dab8edc07f --- /dev/null +++ b/litellm/router_rust/invoke/walk.py @@ -0,0 +1,398 @@ +""" +The request metadata the Python router stamps as it walks retries and fallbacks, kept for a +``RustCore`` call. router-core decides the walk and reports it (each attempt's ``scope``, +``max_retries`` and ``model_group_size``, and the ``fallback`` and ``retry_logged`` events); this +module only holds what each level's metadata dict holds in the Python router and lays it over the +attempt's kwargs. + +In the Python router one metadata dict serves a level's retries, and every fallback hop copies +its parent's (``run_async_fallback``'s ``_hop_metadata``), so a hop's breadcrumbs never reach +its parent. Each scope here is one such dict. The root level's dict is the caller's own (a proxy +reads a failed request's walk off it), so a fresh call's root writes reach that dict as well. +""" + +from collections.abc import Callable, Coroutine, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.router_api import RoutingContext + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + +# `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps. +RETRY_BREADCRUMB_LIMIT: Final = 4 + +# The keys the walk stamps on each attempt's metadata. +_WALK_KEYS: Final = ( + "original_model_group", + "attempted_fallbacks", + "attempted_retries", + "max_retries", + "model_group_size", + "previous_models", + "request_retry_count", +) +# What a breadcrumb reads from the last attempt a scope made. +_SERVED_KEYS: Final = ("model_info", "deployment") +# The walk state `run_async_fallback` hands every hop as kwargs. +_HOP_KWARGS: Final = ("fallback_depth", "max_fallbacks", "attempted_targets", "include_fallback_errors") + + +class _Scope: + __slots__ = ("kwargs", "metadata") + + def __init__( + self, + metadata: dict[str, object], # mutable-ok: the level's metadata, stamped as the walk runs + kwargs: Mapping[str, object], + ) -> None: + self.metadata: Final = metadata + self.kwargs: Final = kwargs + + +def _breadcrumb( + metadata: Mapping[str, object], model: str, error: BaseException, params_base_model: Callable[[str], str | None] +) -> dict[str, object]: # mutable-ok: a fresh RetryAttemptRecord + """``_core_log_retry``'s record of the failure.""" + from litellm.litellm_core_utils.get_litellm_params import get_base_model_from_litellm_call_metadata + + model_info: Final = as_str_mapping(metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + known_id: Final = deployment_id if isinstance(deployment_id, str) else None + base_model: Final = get_base_model_from_litellm_call_metadata(metadata=metadata) or ( + params_base_model(known_id) if known_id is not None else None + ) + deployment: Final = metadata.get("deployment") + attempted: Final = metadata.get("attempted_retries") + return { + "model_group": model, + "deployment_id": known_id, + "model": deployment if isinstance(deployment, str) else None, + "base_model": base_model, + "exception_type": type(error).__name__, + "exception_string": str(error), + "attempted_retries": attempted if type(attempted) is int else None, + } + + +def _kept_breadcrumbs(earlier: object) -> tuple[object, ...]: + """The breadcrumbs a new one joins: the latest ``RETRY_BREADCRUMB_LIMIT - 1``.""" + if not isinstance(earlier, list | tuple): + return () + stamped: Final[tuple[object, ...]] = tuple(earlier) # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the breadcrumbs as stamped + return stamped[-(RETRY_BREADCRUMB_LIMIT - 1) :] + + +def log_retry_into( + kwargs: dict[str, object], # mutable-ok: the request kwargs whose metadata the record joins + error: Exception, + params_base_model: Callable[[str], str | None], +) -> dict[str, object]: # mutable-ok: the same kwargs, as ``_core_log_retry`` returns them + """ + ``_core_log_retry`` (``RoutingContext.log_retry``): the failure joins the request metadata's + ``previous_models`` and counts toward ``request_retry_count``, with a ``litellm.routing.retry`` + phase event when a pre-routing strategy picked the group. + """ + from litellm.integrations.otel.runtime import phase_event + + field: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + metadata: Final = kwargs[field] + view: Final = as_str_mapping(metadata) + if view is None: + # Python reads it with ``.get``. + raise AttributeError(f"{type(metadata).__name__!r} object has no attribute 'get'") + model_group: Final = kwargs.get("model") + record: Final = _breadcrumb(view, "", error, params_base_model) + record["model_group"] = model_group if isinstance(model_group, str) else None + deployment_id: Final = record["deployment_id"] + breadcrumbs: Final = (*_kept_breadcrumbs(view.get("previous_models")), record) + earlier: Final = view.get("request_retry_count") + request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1 + decision: Final = view.get("routing_decision") + router_name: Final = (as_str_mapping(decision) or {}).get("router_model_name") + if isinstance(router_name, str): + phase_event( + "litellm.routing.retry", + { + "litellm.routing.router_model_name": router_name, + "litellm.retry.count": request_retry_count, + "error.type": type(error).__name__, + **({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}), + **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}), + }, + ) + metadata["previous_models"] = breadcrumbs # pyright: ignore[reportIndexIssue] # the logging object already holds this dict + metadata["request_retry_count"] = request_retry_count # pyright: ignore[reportIndexIssue] # same dict, read by the cap + return kwargs + + +class WalkMetadata: + """The metadata each level of one call's walk holds, by scope.""" + + __slots__ = ("_attempted", "_field", "_live", "_params_base_model", "_request", "_scopes") + + def __init__( + self, + request: Mapping[str, object], + field: str, + params_base_model: Callable[[str], str | None], + *, + live: bool = False, + ) -> None: + model: Final = request.get("model") + root: Final[dict[str, object]] = {"attempted_fallbacks": 0} # mutable-ok: the root level's metadata + if model is not None: + root["original_model_group"] = model + self._request: Final = request + self._field: Final = field + self._params_base_model: Final = params_base_model + self._scopes: Final[dict[int, _Scope]] = {0: _Scope(root, {})} # mutable-ok: filled as the walk hops + self._attempted: AttemptedFallbackTargets | None = None + # A call whose retry and fallback wrappers stamp the request itself; a stream's recovery + # re-enters below them. + self._live: Final = live + if live: + self._stamp_fallback_entry() + + def _stamp_fallback_entry(self) -> None: + """The head of ``async_function_with_fallbacks``, in the request's own buckets. ``run_async_fallback``'s + hops carry their ``attempted_targets`` and never reach it.""" + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + + request: Final = self._request + if isinstance(request.get("attempted_targets"), AttemptedFallbackTargets): + return + sibling: Final = request.get("metadata" if self._field == "litellm_metadata" else "litellm_metadata") + if isinstance(sibling, dict): + sibling.pop("attempted_fallbacks", None) # pyright: ignore[reportUnknownMemberType] # the caller's metadata dict + sibling.pop("original_model_group", None) # pyright: ignore[reportUnknownMemberType] # see above + bucket: Final = request.get(self._field) + if not isinstance(bucket, dict): + return + bucket["attempted_fallbacks"] = 0 + model: Final = request.get("model") + if model is not None: + bucket["original_model_group"] = model + + def _retry_bucket(self, scope: _Scope) -> dict[str, object] | None: # mutable-ok: the request's own dict + """``_metadata`` of ``async_function_with_retries`` when it is a dict of the request's: the + root level's, or a hop's when the bucket is not the one ``run_async_fallback`` copies.""" + if not self._live: + return None + name: Final = "litellm_metadata" if "litellm_metadata" in self._request else "metadata" + if scope is not self._scopes[0] and name == self._field: + return None + bucket: Final = self._request.get(name) + # `... or {}`: an empty bucket leaves the stamps on a dict of their own. + return bucket if isinstance(bucket, dict) and bucket else None # pyright: ignore[reportUnknownVariableType] # the request's untyped metadata dict + + def _write_through(self, scope: _Scope, values: Mapping[str, object]) -> None: + bucket: Final = self._retry_bucket(scope) + if bucket is not None: + bucket.update(values) + + def retries_opened(self, event: Mapping[str, object]) -> None: + """The head of ``async_function_with_retries``: the group's size and the retry tracking, before any pick.""" + scope: Final = self._scope(event.get("scope")) + stamped: Final = {"attempted_retries": 0, "max_retries": event.get("max_retries")} + scope.metadata.update(stamped) + self._write_through(scope, stamped) + bucket: Final = self._retry_bucket(scope) + if bucket is not None and isinstance(bucket.get("model_group"), str): + bucket["model_group_size"] = event.get("model_group_size") + + def retry_budget(self, event: Mapping[str, object]) -> None: + """``_metadata["max_retries"] = num_retries``, once a level's first failure settled its overrides.""" + scope: Final = self._scope(event.get("scope")) + stamped: Final = {"max_retries": event.get("max_retries")} + scope.metadata.update(stamped) + self._write_through(scope, stamped) + + def _scope(self, value: object) -> _Scope: + found: Final = self._scopes.get(value) if isinstance(value, int) else None + return found if found is not None else self._scopes[0] + + def targets(self) -> "AttemptedFallbackTargets": + """The walk's ``attempted_targets``: the targets tried and the ones skipped, and why.""" + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + + if self._attempted is None: + self._attempted = AttemptedFallbackTargets() + return self._attempted + + def hop(self, event: Mapping[str, object]) -> None: + """A ``fallback`` event: the new level copies its parent's metadata and kwargs.""" + parent: Final = self._scope(event.get("parent")) + attempted: Final = self.targets() + for key in (event.get("from"), event.get("to")): + if isinstance(key, str): + attempted.record(key) + metadata: Final = dict(parent.metadata) + original: Final = metadata.pop("original_model_group", self._request.get("model")) + metadata.pop("model_group", None) + metadata.pop("attempted_fallbacks", None) + metadata["original_model_group"] = original + # A dict target's non-string `model` is what `kwargs["model"]` holds on the hop. + metadata["model_group"] = event["unroutable"] if "unroutable" in event else event.get("to") + metadata["attempted_fallbacks"] = event.get("depth") + kwargs: Final = { + **parent.kwargs, + "fallback_depth": event.get("depth"), + "max_fallbacks": event.get("max_fallbacks"), + "attempted_targets": attempted, + } + if self._request.get("include_fallback_errors") is True: + kwargs["include_fallback_errors"] = True + scope: Final = event.get("scope") + if isinstance(scope, int): + self._scopes[scope] = _Scope(metadata, kwargs) + + def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a fresh dict for the loggers + """The kwargs the level held after a hop (``kwargs["model"] = target``), as the + fallback loggers see them.""" + scope: Final = self._scope(event.get("scope")) + model: Final = scope.metadata.get("model_group", event.get("to")) + return {**self._request, **scope.kwargs, "model": model, self._field: dict(scope.metadata)} + + def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None: + """A ``retry_logged`` event: the failure joins the scope's ``previous_models``.""" + scope: Final = self._scope(event.get("scope")) + metadata: Final = scope.metadata + attempted: Final = event.get("attempted_retries") + if isinstance(attempted, int): + metadata["attempted_retries"] = attempted + self._write_through(scope, {"attempted_retries": attempted}) + if error is None: + return + record: Final = _breadcrumb(metadata, str(event.get("model")), error, self._params_base_model) + earlier: Final = metadata.get("previous_models") + kept: Final = _kept_breadcrumbs(earlier) + count: Final = metadata.get("request_retry_count") + metadata["previous_models"] = (*kept, record) + metadata["request_retry_count"] = (count if type(count) is int and count >= 0 else 0) + 1 + self._write_through( + scope, + {"previous_models": metadata["previous_models"], "request_retry_count": metadata["request_retry_count"]}, + ) + + def deployment_that_just_failed(self, step: Mapping[str, object]) -> str | None: + """``_deployment_that_just_failed``: the deployment the metadata an attempt inherits names, + before the attempt stamps its own; ``None`` when the walk's level has run none yet.""" + model_info: Final = as_str_mapping(self._scope(step.get("scope")).metadata.get("model_info")) + model_id: Final = model_info.get("id") if model_info is not None else None + return model_id if isinstance(model_id, str) else None + + def request_retry_count(self, step: Mapping[str, object]) -> object: + """The level's ``request_retry_count``, as the retry loop has counted it so far.""" + return self._scope(step.get("scope")).metadata.get("request_retry_count") + + def stamp(self, step: Mapping[str, object], kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The attempt's kwargs with its level's walk metadata and hop kwargs laid over them.""" + scope: Final = self._scope(step.get("scope")) + metadata: Final = scope.metadata + metadata["attempted_retries"] = step.get("retry") + metadata["max_retries"] = step.get("max_retries") + metadata["model_group_size"] = step.get("model_group_size") + self._write_through( + scope, {"attempted_retries": metadata["attempted_retries"], "max_retries": metadata["max_retries"]} + ) + served: Final = dict(as_str_mapping(kwargs.get(self._field)) or {}) + served.update({key: metadata[key] for key in _WALK_KEYS if key in metadata}) + metadata.update({key: served.get(key) for key in _SERVED_KEYS}) + return {**kwargs, **{key: scope.kwargs[key] for key in _HOP_KWARGS if key in scope.kwargs}, self._field: served} + + +async def _log_failed_fallbacks(ctx: "RoutingContext", detail: str, parent_otel_span: "Span | None") -> None: + from litellm._logging import verbose_router_logger + from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments_with_debug_info + + cooldown_info: Final = await async_get_cooldown_deployments_with_debug_info( + ctx=ctx, parent_otel_span=parent_otel_span + ) + verbose_router_logger.error( + "litellm.router.py::async_function_with_fallbacks() - " + "Error occurred while trying to do fallbacks - %s\n" + "Debug Information:\nCooldown Deployments=%s", + detail, + cooldown_info, + ) + + +class FallbackOutcomes: + """What ``run_async_fallback`` does when a hop returns: it logs the hop to the process's + custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own + first, then each hop that failed) in the response's ``x-litellm-fallback-errors``.""" + + __slots__ = ("_failed", "_header_errors", "_include", "_pending") + + def __init__(self, request: Mapping[str, object]) -> None: + self._include: Final = request.get("include_fallback_errors", False) is True + self._failed: Final[dict[object, list[object]]] = {} # mutable-ok: each level's failed hops, in order + self._header_errors: Final[list[list[object]]] = [] # mutable-ok: each successful hop's errors, innermost first + self._pending: Final[list[Coroutine[object, object, None]]] = [] # mutable-ok: logger calls not yet awaited + + def done( + self, + event: Mapping[str, object], + level_error: BaseException | None, + hop_error: BaseException | None, + kwargs: Mapping[str, object], + ) -> None: + from litellm.litellm_core_utils.add_retry_fallback_headers import get_fallback_error_info + from litellm.router_utils import fallback_event_handlers as handlers + + if level_error is None or not isinstance(level_error, Exception): + return + parent: Final = event.get("parent") + failed: Final = self._failed.setdefault(parent, []) + original: Final = str(event.get("from")) + if event.get("succeeded") is True: + if self._include: + self._header_errors.append([get_fallback_error_info(level_error), *failed]) + self._pending.append( + handlers.log_success_fallback_event( # pyright: ignore[reportUnknownMemberType] # takes the kwargs as an untyped dict + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + ) + ) + return + if isinstance(hop_error, Exception): + failed.append(get_fallback_error_info(hop_error)) + self._pending.append( + handlers.log_failure_fallback_event( # pyright: ignore[reportUnknownMemberType] # see above + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + ) + ) + + def failed_fallbacks(self, ctx: "RoutingContext", detail: str, parent_otel_span: "Span | None") -> None: + """The ``except`` around the regular fallbacks of ``async_function_with_fallbacks_common_utils``: + ``detail`` is the failure's text as it read then, the cooldowns are read when the pending calls run.""" + self._pending.append(_log_failed_fallbacks(ctx, detail, parent_otel_span)) + + def take_pending(self) -> tuple[Coroutine[object, object, None], ...]: + pending: Final = tuple(self._pending) + self._pending.clear() + return pending + + def add_headers(self, response: object, attempted_fallbacks: int) -> object: + """``add_fallback_headers_to_response`` for each successful hop, innermost first.""" + from litellm.litellm_core_utils.add_retry_fallback_headers import add_fallback_headers_to_response + + headed: object = response # rebind-ok: gains each level's errors + for errors in self._header_errors: + headed = add_fallback_headers_to_response( + response=headed, + attempted_fallbacks=attempted_fallbacks, + fallback_errors=errors, # pyright: ignore[reportArgumentType] # FallbackErrorInfo dicts + ) + return headed + + +async def await_all(pending: Sequence[Coroutine[object, object, None]]) -> None: + for call in pending: + await call + + +__all__ = ["RETRY_BREADCRUMB_LIMIT", "FallbackOutcomes", "WalkMetadata", "await_all"] diff --git a/litellm/router_rust/invoke/websocket.py b/litellm/router_rust/invoke/websocket.py new file mode 100644 index 00000000000..e7baeb92345 --- /dev/null +++ b/litellm/router_rust/invoke/websocket.py @@ -0,0 +1,127 @@ +""" +Responses websocket turns on the Rust router core (``async_admit_deployment_turn``): the socket is +opened on one deployment, and the core hands the handler a ``turn_admission`` callable that re-runs +that deployment's admission before every later turn: the io-token turn budget, the deployment +budget limiter, the routing strategy (on that deployment alone) and the pre-call checks. + +Which strategy and selector apply is the Rust core's routing context (``routing_context``); the +selectors, limiters and checks are the live Python objects the host keeps. +""" + +from collections.abc import Callable, Mapping +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import without +from litellm.router_rust.invoke.shaping import as_dict + +if TYPE_CHECKING: + from litellm.router_rust.invoke.endpoints import EndpointCore + + +def turn_admission( + core: "EndpointCore", model_group: str, deployment: Mapping[str, object], kwargs: Mapping[str, object] +) -> Callable[[], object]: + """The ``turn_admission`` the handler calls before each later turn.""" + return partial(admit_deployment_turn, core, model_group, deployment, kwargs) + + +def _deployment_as_configured_now(core: "EndpointCore", deployment: Mapping[str, object]) -> Mapping[str, object]: + model_info: Final = deployment.get("model_info") + model_id: Final = as_dict(model_info).get("id") if isinstance(model_info, dict) else None # pyright: ignore[reportUnknownArgumentType] # the configured model_info dict + current: Final = core.configured_deployment(model_id) if isinstance(model_id, str) else None + return deployment if current is None else current.model_dump(exclude_none=True) + + +def _budget_limiter(core: "EndpointCore") -> object: + """The router's deployment-budget callback; the Rust core declines every config that has one.""" + from litellm.router_rust.core import RustCoreUnsupported + + try: + return core.registry().plugins("budget_limiter") + except RustCoreUnsupported: + return None + + +async def admit_deployment_turn( + core: "EndpointCore", model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> Mapping[str, object]: + """`async_admit_deployment_turn`.""" + from litellm import exceptions as litellm_exceptions + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_admit_io_token_turn, + without_io_token_limits, + ) + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + from litellm.utils import get_utc_datetime + + deployment: Final = _deployment_as_configured_now(core, opened_on) + rate_limit_check: Final = next( + (callback for callback in core.routing_callbacks() if isinstance(callback, ModelRateLimitingCheck)), + None, + ) + io_token_admission: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if rate_limit_check is None + else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) # pyright: ignore[reportArgumentType] # the configured deployment dict + ) + budget_limiter: Final[object] = _budget_limiter(core) + if budget_limiter is not None: + try: + await budget_limiter.async_filter_deployments( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # RouterBudgetLimiting + model=model_group, + healthy_deployments=[dict(deployment)], + messages=None, + request_kwargs=dict(request_kwargs), + ) + except ValueError as e: + raise litellm_exceptions.RateLimitError( + message=str(e), + llm_provider="", + model=model_group, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from e + await _admit_turn_through_routing_strategy(core, model_group, deployment, request_kwargs) + await core.router.async_routing_strategy_pre_call_checks( + deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None + ) + return io_token_admission + + +async def _admit_turn_through_routing_strategy( + core: "EndpointCore", model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object] +) -> None: + """`_admit_turn_through_routing_strategy`: the group's strategy picks among this deployment alone.""" + from litellm import exceptions as litellm_exceptions + from litellm.router_api import RoutingContext + from litellm.router_rust.steps.selection import pick_async + from litellm.router_utils.handle_error import async_raise_no_deployment_exception + + routing_kwargs: Final = without(request_kwargs, "litellm_logging_obj") + override: Final = routing_kwargs.get("routing_strategy") + strategy, scope, group = core.handle.routing_context(model_group, override if isinstance(override, str) else None) + selectors: Final = core.selectors + selector: Final = selectors.selector_for(strategy, scope, group, routing_kwargs) + if strategy == "simple-shuffle" or selector is None: + return + selected: Final = await pick_async( + strategy, selector, model_group, [dict(deployment)], None, None, dict(routing_kwargs) + ) + if selected is None: + no_deployment: Final = await async_raise_no_deployment_exception( + ctx=RoutingContext(core), # pyright: ignore[reportArgumentType] # the error reads registry() and health() + model=model_group, + parent_otel_span=None, + ) + raise litellm_exceptions.RateLimitError( + message=str(no_deployment), + llm_provider="", + model=model_group, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from no_deployment + if selectors.is_override(strategy, selector): + await selector.async_pre_call_check(dict(deployment), None) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # every selector is a CustomLogger + + +__all__ = ["admit_deployment_turn", "turn_admission"] diff --git a/litellm/router_rust/steps/README.md b/litellm/router_rust/steps/README.md new file mode 100644 index 00000000000..9b5c68bdba9 --- /dev/null +++ b/litellm/router_rust/steps/README.md @@ -0,0 +1,12 @@ +# steps + +Answers to router-core's host steps that need live Python objects. router-core makes the routing decision; these modules supply the Python-only facts or run the Python-only objects it asks about. + +- `selection`: stateful strategies (latency, tpm/rpm, cost, usage) on `select` steps +- `prerouting`: auto, complexity, adaptive and quality routers, and session binding, on `pre_route` steps +- `precall`: token counts, context windows and param support for pre-call checks +- `plugins`: `Router(plugins=[...])` objects +- `callbacks`: the optional pre-call checks (budgets, affinity, rate limits) as `CustomLogger`s +- `access`: the caller facts read from auth metadata + +A Rust host answers these steps natively, or declines the config at build. diff --git a/litellm/router_rust/steps/__init__.py b/litellm/router_rust/steps/__init__.py new file mode 100644 index 00000000000..c47505e7468 --- /dev/null +++ b/litellm/router_rust/steps/__init__.py @@ -0,0 +1 @@ +"""Answers to router-core host steps that need live Python objects.""" diff --git a/litellm/router_rust/steps/access.py b/litellm/router_rust/steps/access.py new file mode 100644 index 00000000000..087a87e2c6b --- /dev/null +++ b/litellm/router_rust/steps/access.py @@ -0,0 +1,85 @@ +"""The caller facts the Rust core routes on, read from the request's auth metadata: whether the +caller is a proxy admin, the models its key and team allow (which narrow an access-group-only +caller to its groups' deployments) and, for a proxy admin, the model the request names.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request, request_user_api_key_auth + +_NAMES: Final = TypeAdapter(tuple[object, ...]) + + +def _names(value: object) -> tuple[str, ...]: + names: Final = _NAMES.validate_python(value) if isinstance(value, list | tuple | set | frozenset) else () + return tuple(str(name) for name in names) + + +def _admin_model(request: Mapping[str, object]) -> object: + """The model ``filter_team_based_models`` compares team deployments against.""" + metadata: Final = as_str_mapping(request.get("metadata")) or {} + litellm_metadata: Final = as_str_mapping(request.get("litellm_metadata")) or {} + return request.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group") + + +def caller_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """``proxy_admin``, ``allowed_models`` and ``admin_model`` for the Rust request document.""" + auth: Final[object] = request_user_api_key_auth(request) + if auth is None: + return {} + allowed: Final = sorted({*_names(getattr(auth, "models", None)), *_names(getattr(auth, "team_models", None))}) + admin: Final = is_proxy_admin_request(request) + model: Final = _admin_model(request) if admin else None + return { + **({"allowed_models": allowed} if allowed else {}), + **({"proxy_admin": True} if admin else {}), + **({"admin_model": model} if isinstance(model, str) else {}), + } + + +def _router_weights_error(raw: object) -> BaseException: + """The error ``validate_router_weights`` raises for weights the Rust core refused.""" + from pydantic import ValidationError + + from litellm.types.router_weights import validate_router_weights + + try: + _ = validate_router_weights(raw) + except ValidationError as error: + return error + return RuntimeError(f"router-core refused router weights the validator accepts: {raw!r}") + + +def selection_refusal(step: Mapping[str, object], model: str, request: Mapping[str, object]) -> BaseException | None: + """The error for a pick the Rust core refused over who the caller is or what it asked for: + a public name several teams share, deployments reserved for other teams, invalid request + router weights or a non-numeric deployment weight. None for any other failure.""" + import litellm.exceptions as litellm_exceptions + + kind: Final = step.get("error_kind") + named: Final = str(step.get("error_model") or model) + if kind == "multiple_teams": + return litellm_exceptions.BadRequestError( + message=( + f"Model name '{named}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=named, + llm_provider="", + ) + if kind == "reserved": + return litellm_exceptions.BadRequestError( + message=f"Deployment {named} is reserved for another team until {step.get('error_detail')}", + model=named, + llm_provider="", + ) + if kind == "invalid_router_weights": + return _router_weights_error(request.get("_router_weights")) + if kind == "non_numeric_weight": + return TypeError(f"Deployment {step.get('error_detail')} must be numeric") + return None diff --git a/litellm/router_rust/steps/callbacks.py b/litellm/router_rust/steps/callbacks.py new file mode 100644 index 00000000000..b906a496550 --- /dev/null +++ b/litellm/router_rust/steps/callbacks.py @@ -0,0 +1,650 @@ +""" +Host-side optional pre-call checks for ``RustCore``: the ``CustomLogger`` checks a router builds +from ``optional_pre_call_checks``, ``provider_budget_config``, per-deployment budgets and +``model_group_affinity_config``, registered in the host's ``callbacks`` list exactly as the Python +router's ``add_optional_pre_call_checks`` registers them, since they are features users configure. + +The Rust core asks the host to run every routing callback's ``async_filter_deployments`` through a +``callback_filters`` select stage (``async_callback_filter_deployments``) and decides itself what +the surviving deployments or the raised error mean: a pick with nothing left, a retry, a fallback. + +This module also keeps the process-wide holder rules (``litellm/router_utils/global_callback_holders``): +a retiring router hands the checks it vacates to a survivor, a runtime removal keeps a check +another live router still has, and ``observe(request_scoped=True)`` takes every routing callback +off the host lists while the router keeps running them (``_routing_callbacks``). It also answers +which outcome callbacks the Python router would still have running for a request +(``OptionalChecks.outcome_callbacks``), which router-core records by. It never reaches a Python +router core. +""" + +from collections.abc import Iterable, Mapping, Sequence +from itertools import chain +from typing import TYPE_CHECKING, Final +from weakref import WeakSet + +from litellm._logging import verbose_router_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import PROMPT_CACHE_PIN_CACHE_MAX_SIZE +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.routing_common_utils import get_request_team_id +from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_utils.global_callback_holders import ( + live_check_owners, + routers_holding_global_callbacks, + routers_holding_global_callbacks_lock, +) +from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( + DeploymentAffinityCheck, + completed_affinity_response, + warn_on_unknown_model_group_affinity_flags, +) +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.router_python_host import PythonHost + from litellm.router_rust.steps.selection import StrategySelectors + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.router_utils.global_callback_holders import GlobalCallbackHolder + from litellm.types.router import Deployment + +_AFFINITY_CHECKS: Final = ( + "deployment_affinity", + "responses_api_deployment_check", + "session_affinity", + "encrypted_content_affinity", +) +# `_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS`: the checks a runtime list drops when it leaves them out. +_RUNTIME_TOGGLEABLE: Final[Mapping[str, type[CustomLogger]]] = { + "prompt_caching": PromptCachingDeploymentCheck, + "enforce_model_rate_limits": ModelRateLimitingCheck, + "encrypted_content_affinity": EncryptedContentAffinityCheck, +} + + +def _callbacks(host: "PythonHost") -> list[object]: # mutable-ok: the process list itself + return host.callback_list("callbacks") # pyright: ignore[reportArgumentType] # not in the CallbackList literal + + +def register_router_selector(host: "PythonHost", selector: CustomLogger) -> None: + """``_register_router_selector``: least-busy also counts calls as they start.""" + if isinstance(selector, LeastBusyLoggingHandler): + if isinstance(host.callback_list("input_callback"), list): # pyright: ignore[reportUnnecessaryIsInstance] # a process list a user may have reassigned + host.callbacks.add_litellm_input_callback(selector) + else: + host.set_callback_list("input_callback", [selector]) + if isinstance(_callbacks(host), list): # pyright: ignore[reportUnnecessaryIsInstance] # see above + host.callbacks.add_litellm_callback(selector) + + +def _move_before_deployment_affinity( + callback_list: list[object], # mutable-ok: reordered in place + callback: object, +) -> None: + if callback not in callback_list: + return + callback_list.remove(callback) + index: Final = next( + (i for i, entry in enumerate(callback_list) if isinstance(entry, DeploymentAffinityCheck)), + len(callback_list), + ) + callback_list.insert(index, callback) + + +class OptionalChecks: + """One router's optional pre-call checks (``optional_callbacks``) and the stores they read.""" + + def __init__( + self, + owner: "GlobalCallbackHolder", + host: "PythonHost", + cache: DualCache, + hosted: "HostRegistry", + ctx: object, + arguments: Mapping[str, object], + ) -> None: + self._owner: Final = owner + self._host: Final = host + self._cache: Final = cache + self._hosted: Final = hosted + self._ctx: Final = ctx + self.optional_callbacks: list[object] | None = None # mutable-ok: the router's live list, as Python keeps it + self.router_budget_logger: RouterBudgetLimiting | None = None + self.provider_budget_config: object = arguments.get("provider_budget_config") + ttl: Final = arguments.get("deployment_affinity_ttl_seconds") + self.deployment_affinity_ttl_seconds: object = 3600 if ttl is None else ttl + self.session_affinity_repin_on_failover: object = bool(arguments.get("session_affinity_repin_on_failover")) + affinity_config: Final = arguments.get("model_group_affinity_config") + self.model_group_affinity_config: dict[str, list[str]] | None = ( # mutable-ok: the argument as passed + affinity_config if isinstance(affinity_config, dict) else None # pyright: ignore[reportAttributeAccessIssue] # the argument as passed + ) + warn_on_unknown_model_group_affinity_flags(self.model_group_affinity_config) + self.prompt_caching_pin_cache: DualCache = DualCache( + redis_cache=cache.redis_cache, + in_memory_cache=InMemoryCache(max_size_in_memory=PROMPT_CACHE_PIN_CACHE_MAX_SIZE), + ) + self._request_scoped_selectors: tuple[CustomLogger, ...] = () + self._request_scoped_policies: tuple[CustomLogger, ...] = () + # ``_slack_alerting_loggers``: the Slack alerting logger ``alerting_config`` builds, if any. + self.slack_alerting_loggers: tuple[CustomLogger, ...] = () + # Whether the outcome callbacks would still be on the host lists (``_release_global_callbacks``). + self._outcome_callbacks_listed: bool = True + # The logging objects ``_core_bind_to_request_logging`` would have bound them to. + self._bound_requests: Final[WeakSet[Logging]] = WeakSet() + + # -- construction --------------------------------------------------------------------------- + + def build(self, checks: object, model_list: object) -> None: + """The constructor's checks: a budget appends ``router_budget_limiting`` to the list given (in + place, as Python does), the list is added, a per-group affinity config adds the affinity + check, and ``apply_default_settings`` adds the empty default list.""" + listed: list[object] | None = checks if isinstance(checks, list) else None # pyright: ignore[reportUnknownVariableType] # mutable-ok: appended in place, as Python does # rebind-ok: the argument as passed + if RouterBudgetLimiting.should_init_router_budget_limiter( # pyright: ignore[reportUnknownMemberType] # untyped model_list + model_list=model_list if isinstance(model_list, list) else None, # pyright: ignore[reportUnknownArgumentType] # the argument as passed + provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed + ): + if listed is not None: + listed.append("router_budget_limiting") + else: + listed = ["router_budget_limiting"] # rebind-ok: the list Python builds + if listed is not None: + self.add(listed) # pyright: ignore[reportUnknownArgumentType] # names as given + if self.model_group_affinity_config: + self.ensure_deployment_affinity_callback() + self.add([]) + + def _is_priority_group(self, model: str) -> bool: + """``_is_priority_routing_group``.""" + hosted: Final = self._hosted + resolved: Final = hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + group: Final = hosted.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + + async def repin_session_after_success( + self, litellm_params: Mapping[str, object], response: object, cooldowns: "CooldownCache" + ) -> None: + """``_repin_session_after_success``: with ``session_affinity_repin_on_failover`` on, a completed + success moves the session's pin to the deployment that served it, among the group's + deployments not cooling down.""" + if not self.session_affinity_repin_on_failover: + return + try: + if not completed_affinity_response(response): + return + callback: Final = next( + (logger for logger in self.optional_callbacks or () if isinstance(logger, DeploymentAffinityCheck)), + None, + ) + if callback is None: + return + model_group: Final = callback.get_repin_model_group(litellm_params) + if model_group is None or self._is_priority_group(model_group): + return + registered_ids: Final = self._hosted._core_get_candidate_model_ids_for_route( # pyright: ignore[reportPrivateUsage] # the host registry's candidate lookup + model=model_group, team_id=get_request_team_id(litellm_params) + ) + cooldown_ids: Final = frozenset( + model_id + for model_id, _ in await cooldowns.async_get_active_cooldowns( + model_ids=list(registered_ids), parent_otel_span=None + ) + ) + await callback.async_repin_session_after_success(litellm_params, model_group, registered_ids - cooldown_ids) + except Exception as error: # noqa: BLE001 # optional pin updates must not interrupt success accounting + verbose_router_logger.debug("Session re-pinning failed: %s", error) + + def _affinity_check(self, user_key: bool, responses_api: bool, session_id: bool) -> DeploymentAffinityCheck: + return DeploymentAffinityCheck( + cache=self._cache, + ttl_seconds=self.deployment_affinity_ttl_seconds, # pyright: ignore[reportArgumentType] # the argument as passed + enable_user_key_affinity=user_key, + enable_responses_api_affinity=responses_api, + enable_session_id_affinity=session_id, + model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_group, + session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, # pyright: ignore[reportArgumentType] # same + ) + + def _own(self, callback: CustomLogger) -> None: + if self.optional_callbacks is None: + self.optional_callbacks = [] + self.optional_callbacks.append(callback) + self._host.callbacks.add_litellm_callback(callback) + + def ensure_deployment_affinity_callback(self) -> None: + """``_ensure_deployment_affinity_callback``: the affinity check with every global flag off.""" + if any(isinstance(cb, DeploymentAffinityCheck) for cb in self.optional_callbacks or ()): + return + self._own(self._affinity_check(False, False, False)) + + def _add_affinity(self, checks: Sequence[object]) -> None: + user_key: Final = "deployment_affinity" in checks + responses_api: Final = "responses_api_deployment_check" in checks + session_id: Final = "session_affinity" in checks + if not (user_key or responses_api or session_id): + return + if self.optional_callbacks is None: + self.optional_callbacks = [] + existing: Final = next( + (cb for cb in self.optional_callbacks if isinstance(cb, DeploymentAffinityCheck)), + None, + ) + if existing is None: + self._own(self._affinity_check(user_key, responses_api, session_id)) + return + existing.enable_user_key_affinity = existing.enable_user_key_affinity or user_key + existing.enable_responses_api_affinity = existing.enable_responses_api_affinity or responses_api + existing.enable_session_id_affinity = existing.enable_session_id_affinity or session_id + existing.ttl_seconds = self.deployment_affinity_ttl_seconds # pyright: ignore[reportAttributeAccessIssue] # the argument as passed + if self.model_group_affinity_config: + existing.model_group_affinity_config = self.model_group_affinity_config + + def _add_encrypted_content_affinity(self, enable_global_affinity: bool) -> None: + """``_add_encrypted_content_affinity_check``.""" + if not ( + enable_global_affinity + or EncryptedContentAffinityCheck.has_model_group_affinity_enabled(self.model_group_affinity_config) + ): + return + if self.optional_callbacks is None: + self.optional_callbacks = [] + existing: Final = next( + (cb for cb in self.optional_callbacks if isinstance(cb, EncryptedContentAffinityCheck)), None + ) + if existing is not None: + existing.ctx = self._ctx # pyright: ignore[reportAttributeAccessIssue] # the core's RoutingContext serves registry() + existing.enable_global_affinity = existing.enable_global_affinity or enable_global_affinity + existing.model_group_affinity_config = self.model_group_affinity_config or {} + check: EncryptedContentAffinityCheck = existing # rebind-ok: the check to move + else: + check = EncryptedContentAffinityCheck( # rebind-ok: the check to move + ctx=self._ctx, # pyright: ignore[reportArgumentType] # see above + enable_global_affinity=enable_global_affinity, + model_group_affinity_config=self.model_group_affinity_config, + ) + self._own(check) + _move_before_deployment_affinity(self.optional_callbacks, check) + _move_before_deployment_affinity(_callbacks(self._host), check) + + def add(self, checks: Sequence[object] | None) -> None: + """``add_optional_pre_call_checks``.""" + if checks is None: + return + self._add_affinity(checks) + self._add_encrypted_content_affinity("encrypted_content_affinity" in checks) + for name in checks: + if name in _AFFINITY_CHECKS: + continue + callback: CustomLogger | None = None + if name == "prompt_caching": + callback = PromptCachingDeploymentCheck( + cache=self.prompt_caching_pin_cache, is_priority_group=self._is_priority_group + ) + elif name == "router_budget_limiting": + if self.budget_limiter() is not None: + continue + callback = RouterBudgetLimiting( + dual_cache=self._cache, + provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed + model_list=self._hosted._core_model_list, # pyright: ignore[reportPrivateUsage, reportArgumentType] # the live model_list + ) + self.router_budget_logger = callback + elif name == "enforce_model_rate_limits": + callback = ModelRateLimitingCheck(dual_cache=self._cache) + if callback is None: + continue + if self.optional_callbacks is not None and any( + isinstance(existing, type(callback)) for existing in self.optional_callbacks + ): + continue + self._own(callback) + + def set(self, checks: object) -> None: + """``set_optional_pre_call_checks``: drop the toggleable checks the list leaves out, then add it.""" + if checks is None: + return + listed: Final[tuple[object, ...]] = tuple(checks) if isinstance(checks, list | tuple) else () # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # names as given + requested: Final = frozenset(listed) + for name, callback_cls in _RUNTIME_TOGGLEABLE.items(): + if name not in requested: + self._remove_of_type(callback_cls) + self.add(listed) + + def _remove_of_type(self, callback_cls: type[CustomLogger]) -> None: + """``_remove_optional_callbacks_of_type``: kept on the host while any other live router of + the process has a check of that type.""" + if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): + return + self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] + others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner)) + if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others): + return + callbacks: Final = _callbacks(self._host) + for cb in tuple(callbacks): + if type(cb) is callback_cls: + self._host.callbacks.remove_callback_from_list_by_object(callbacks, cb, require_self=False) # pyright: ignore[reportUnknownMemberType] # the manager's untyped helper + + # -- budgets -------------------------------------------------------------------------------- + + def budget_limiter(self) -> RouterBudgetLimiting | None: + """``_core_get_router_deployment_budget_limiter``: exact type, so a proxy subclass is not taken.""" + if self.router_budget_logger is not None: + return self.router_budget_logger + for cb in self.optional_callbacks or (): + if isinstance(cb, RouterBudgetLimiting) and type(cb) is RouterBudgetLimiting: + self.router_budget_logger = cb + return cb + return None + + def sync_deployment_budget(self, deployment: "Deployment") -> None: + """``_sync_deployment_budget_config``, after a deployment is added.""" + model_id: Final = deployment.model_info.id + if model_id is None: + return + limiter: RouterBudgetLimiting | None = self.budget_limiter() # rebind-ok: built when first needed + params: Final = deployment.litellm_params + if params.max_budget is None or params.budget_duration is None: + if limiter is not None: + limiter.unregister_deployment_budget(model_id=model_id) + return + if limiter is None: + self.add(["router_budget_limiting"]) + limiter = self.budget_limiter() # rebind-ok: the one just built + if limiter is not None: + limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) # pyright: ignore[reportUnknownMemberType] # pydantic's untyped dump + + def unregister_deployment_budget(self, model_id: str) -> None: + limiter: Final = self.budget_limiter() + if limiter is not None: + limiter.unregister_deployment_budget(model_id=model_id) + + # -- the routing callbacks ------------------------------------------------------------------ + + def loggers(self) -> tuple[CustomLogger, ...]: + """``_optional_loggers``.""" + return tuple(cb for cb in self.optional_callbacks or () if isinstance(cb, CustomLogger)) + + def _policies_not_set_globally(self) -> tuple[CustomLogger, ...]: + keys: Final = self._host.callbacks.get_custom_logger_key + global_keys: Final = frozenset(keys(cb) for cb in _callbacks(self._host) if isinstance(cb, CustomLogger)) + return tuple(policy for policy in self._request_scoped_policies if keys(policy) not in global_keys) + + def bind_to_request_logging(self, logging_obj: "Logging") -> None: + """``_core_bind_to_request_logging``'s routing half: the request-scoped selectors replace the + host's same-keyed callbacks for the request, and the policies the host does not run are + added to it. The outcome callbacks are not bound: the Rust core records every attempt's + outcome itself, so the request is noted for ``outcome_callbacks`` instead.""" + self._bound_requests.add(logging_obj) + for selector in self._request_scoped_selectors: + logging_obj.replace_global_callback(selector) + for policy in self._policies_not_set_globally(): + logging_obj.add_dynamic_callback(policy) + + def outcome_callbacks( + self, request: Mapping[str, object], *, sync: bool, make_call: bool = True + ) -> Mapping[str, bool]: + """Router-core's ``OutcomeCallbacks`` for ``request``, empty (its default) while the outcome + callbacks are listed. Past that only the request's bound logging object runs them, and + ``_core_bind_to_request_logging`` binds neither ``sync_deployment_callback_on_success`` nor any + success callback a sync call runs. ``make_call``: the attempt runs through ``make_call``.""" + if self._outcome_callbacks_listed: + return {} + bound: Final = request.get("litellm_logging_obj") in self._bound_requests + return {"success": bound and not sync, "failure": bound, "make_call": make_call} + + def routing_callbacks(self) -> Iterable[object]: + """``_routing_callbacks``: the host's callbacks, or after ``observe(request_scoped=True)`` the + host's callbacks with the router's own selectors and policies in place of theirs.""" + if not (self._request_scoped_selectors or self._request_scoped_policies): + return tuple(_callbacks(self._host)) + keys: Final = self._host.callbacks.get_custom_logger_key + replaced: Final = frozenset(map(keys, self._request_scoped_selectors)) + return ( + *self._host.callbacks.without_custom_logger_keys(_callbacks(self._host), replaced), # pyright: ignore[reportArgumentType] # the process list holds any callback + *self._request_scoped_selectors, + *self._policies_not_set_globally(), + ) + + async def filter_deployments( + self, + model: str, + deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: object, + request: Mapping[str, object], + ) -> list[object]: # mutable-ok: as the filters return it + """``async_callback_filter_deployments``: each routing callback narrows the deployments in + turn; a filter's error is raised for the Rust core to decide on.""" + remaining: list[object] = list(deployments) # mutable-ok: filter contract # rebind-ok: narrowed + base: Final[object] = CustomLogger.async_filter_deployments # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + request_kwargs: Final = dict(request) + for callback in tuple(self.routing_callbacks()): + # The base filter returns its input unchanged: only overriding filters are run. + if not isinstance(callback, CustomLogger) or type(callback).async_filter_deployments is base: # pyright: ignore[reportUnknownMemberType] # see above + continue + remaining = await callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportAssignmentType, reportUnknownVariableType] # see above + model=model, + healthy_deployments=remaining, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + request_kwargs=request_kwargs, + parent_otel_span=None, + ) + return remaining + + # -- release -------------------------------------------------------------------------------- + + def serve_through_request_logging(self, selectors: Sequence[CustomLogger], dropped: Sequence[object]) -> None: + """``_core_serve_through_request_logging``: keep running ``selectors`` (and request-scoped + adaptive hooks) and the optional checks, but take everything off the host lists.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + adaptive: Final = tuple( + AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) # pyright: ignore[reportArgumentType] # an adaptive router + for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values()) # pyright: ignore[reportPrivateUsage] # the host registry's live registry + ) + self._request_scoped_selectors = (*selectors, *adaptive) + self._request_scoped_policies = (*self.loggers(), *self.slack_alerting_loggers) + self.release(dropped) + + def own_adaptive_hooks(self) -> tuple[CustomLogger, ...]: + """``_adaptive_router_hooks``: the host's adaptive post-call hooks of this router's routers.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + own: Final = frozenset( + id(tagged.strategy) + for tagged in chain.from_iterable(self._hosted._core_adaptive_routers.values()) # pyright: ignore[reportPrivateUsage] # see above + ) + return tuple( + hook + for hook in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own + ) + + def retire_state(self, selectors: Sequence[object]) -> None: + """``retire_routing_state``: stop the background work of the selectors and the budget limiter.""" + for strategy in (*selectors, *self.loggers()): + if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting): + strategy.retire() + + def release(self, dropped: Sequence[object]) -> None: + """``_release_global_callbacks``: drop ``dropped``, the optional checks and the adaptive hooks + from the host lists, then hand each vacated check to a surviving router's same-keyed one.""" + self._outcome_callbacks_listed = False + with routers_holding_global_callbacks_lock: + routers_holding_global_callbacks.discard(self._owner) + adaptive: Final = self.own_adaptive_hooks() + slack: Final = self.slack_alerting_loggers + vacated: Final = self._host.callbacks.drop_callbacks( + (*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack) + ) + with routers_holding_global_callbacks_lock: + survivors: Final = tuple(routers_holding_global_callbacks) + candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) # pyright: ignore[reportPrivateUsage] # the holder protocol + for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated): + register_router_selector(self._host, candidate) + adaptive_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, adaptive)) + survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None) # pyright: ignore[reportPrivateUsage] # see above + if survivor is not None and vacated & adaptive_keys: + survivor._sync_adaptive_router_hooks() # pyright: ignore[reportPrivateUsage] # see above + + +_ENCRYPTED_PIN: Final = "_encrypted_content_affinity_pinned" + + +def _request_with_failed_ids(request: Mapping[str, object], step: Mapping[str, object]) -> Mapping[str, object]: + """The request as the filters read it in the Python router: the Rust walk holds what the Python + one keeps in the attempt's kwargs, ``_excluded_deployment_ids`` from a weighted failover hop, + ``_retry_skipped_deployment_ids`` from the retry loop (``deployment_ids_failed_in_request``) and + ``_target_order`` from an order fallback.""" + excluded: Final = tuple(str(id) for id in _ids_of(step.get("excluded_ids"))) + skipped: Final = tuple(str(id) for id in _ids_of(step.get("skipped_ids"))) + target_order: Final = step.get("target_order") + if not excluded and not skipped and target_order is None: + return request + return { + **request, + **({"_excluded_deployment_ids": list(excluded)} if excluded else {}), + **({"_retry_skipped_deployment_ids": skipped} if skipped else {}), + **({"_target_order": target_order} if target_order is not None else {}), + } + + +def _ids_of(value: object) -> Sequence[object]: + return value if isinstance(value, list | tuple) else () # pyright: ignore[reportUnknownVariableType] # the step's id list + + +def filtered_ids(remaining: Iterable[object], candidates: Sequence[str]) -> list[str]: # mutable-ok: a JSON list + """The ids of the deployments the filters left, in their order, among the candidates.""" + ids: Final[list[str]] = [] # mutable-ok: built here + for deployment in remaining: + found = (as_str_mapping((as_str_mapping(deployment) or {}).get("model_info")) or {}).get("id") + if isinstance(found, str) and found in candidates and found not in ids: + ids.append(found) + return ids + + +class RustOptionalChecks: + """``RustCore`` members for its optional pre-call checks and routing callbacks.""" + + # RustCore sets these as Final attributes in its constructor. + @property + def _opt(self) -> OptionalChecks: + checks: Final[OptionalChecks] = self.__dict__["_checks"] # pyright: ignore[reportAny] # RustCore's own attribute + return checks + + @property + def _sel(self) -> "StrategySelectors": + selectors: Final[StrategySelectors] = self.__dict__["_selectors"] # pyright: ignore[reportAny] # same + return selectors + + @property + def _reg(self) -> "HostRegistry": + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # same + return hosted + + @property + def optional_callbacks(self) -> "list[object] | None": # mutable-ok: the router's live list + return self._opt.optional_callbacks + + @property + def router_budget_logger(self) -> RouterBudgetLimiting | None: + return self._opt.router_budget_logger + + @property + def deployment_affinity_ttl_seconds(self) -> object: + return self._opt.deployment_affinity_ttl_seconds + + @property + def session_affinity_repin_on_failover(self) -> object: + return self._opt.session_affinity_repin_on_failover + + @property + def model_group_affinity_config(self) -> "dict[str, list[str]] | None": # mutable-ok: the argument as passed + return self._opt.model_group_affinity_config + + def add_optional_pre_call_checks(self, optional_pre_call_checks: "Sequence[object] | None") -> None: + self._opt.add(optional_pre_call_checks) + + def apply_default_settings(self) -> None: + """Add the (empty) default checks; an enabled continuation registers the prefill check.""" + from litellm.router_rust.invoke.streaming import register_continuation_prefill_check + + self._opt.add([]) + if getattr(self, "enable_mid_stream_fallback_continuation", False): + register_continuation_prefill_check(self._opt._host) # pyright: ignore[reportPrivateUsage] # the checks' host + + # The members another router's retire reads (`global_callback_holders`). + + def _global_callback_candidates(self) -> tuple[CustomLogger, ...]: + return (*self._sel.callback_selectors(), *self._opt.loggers(), *self._opt.slack_alerting_loggers) + + @property + def _core_adaptive_routers(self) -> Mapping[str, object]: + return self._reg._core_adaptive_routers # pyright: ignore[reportPrivateUsage] # the host registry's live registry + + def _sync_adaptive_router_hooks(self) -> None: + strategies: Final = self._reg.strategies + if strategies is not None: + strategies._sync_adaptive_router_hooks() # pyright: ignore[reportPrivateUsage] # the strategies' own hook sync + + def _join_callback_holders(self) -> None: + with routers_holding_global_callbacks_lock: + routers_holding_global_callbacks.add(self) + live_check_owners.add(self) + + def outcome_callbacks( + self, request: Mapping[str, object], *, sync: bool, make_call: bool = True + ) -> Mapping[str, bool]: + return self._opt.outcome_callbacks(request, sync=sync, make_call=make_call) + + def _serve_through_request_logging(self) -> None: + selectors: Final = self._sel + self._opt.serve_through_request_logging( + selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors()) + ) + + def _retire_callbacks(self) -> None: + """``_core_discard``'s callback half: retire the routing state, release the callbacks. The + selectors stay: a retired router still picks with them.""" + live_check_owners.discard(self) + selectors: Final = (*self._sel.callback_selectors(), *self._sel.override_selectors()) + self._opt.retire_state(selectors) + self._opt.release(selectors) + + async def _answer_callback_filters( + self, step: Mapping[str, object], request: Mapping[str, object], messages: object + ) -> tuple[object, BaseException | None]: + """A ``callback_filters`` step: the remaining ids, or a filter's error as an attempt + failure with the error itself, which the caller keeps to raise.""" + from litellm.router_rust.core import attempt_failure + + candidates: Final = [str(id) for id in step.get("candidates") or ()] # pyright: ignore[reportGeneralTypeIssues, reportUnknownVariableType, reportUnknownArgumentType] # the step's id list + records: Final = self._records_of(candidates) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # RustCore's deployment dicts + narrowed: Final = _request_with_failed_ids(request, step) + try: + remaining: Final = await self._opt.filter_deployments( + str(step.get("model_group")), + records, # pyright: ignore[reportUnknownArgumentType] # see above + messages, + narrowed, + ) + except Exception as error: # noqa: BLE001 # handed to the Rust core as the try's failure + return attempt_failure(error), error + # A filter's pin lands on the request kwargs the pick goes on to read. + if narrowed is not request and narrowed.get(_ENCRYPTED_PIN) and isinstance(request, dict): + request[_ENCRYPTED_PIN] = True # rebind-ok: the request dict the pick goes on to read + return filtered_ids(remaining, candidates), None # pyright: ignore[reportUnknownArgumentType] # see above + + +__all__ = [ + "OptionalChecks", + "RustOptionalChecks", + "filtered_ids", + "register_router_selector", +] diff --git a/litellm/router_rust/steps/plugins.py b/litellm/router_rust/steps/plugins.py new file mode 100644 index 00000000000..4af73313838 --- /dev/null +++ b/litellm/router_rust/steps/plugins.py @@ -0,0 +1,87 @@ +""" +Host side of the Rust core's routing-plugin pipeline (``Router(plugins=[...])``). + +The Rust core decides where the pipeline applies (only the async pick runs it; the sync pick +refuses a router with plugins) and which deployments the plugins' candidate models keep +(``router-core/src/routing_plugins.rs``). The plugins themselves are live Python objects, so this +module runs them on a ``routing_plugins`` select step, the way the Python router core's +``_run_routing_plugins`` does, and reports the candidate models left on the request. +""" + +from collections.abc import Mapping, Sequence +from typing import Final, Protocol + +from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias +from litellm.types.router import RoutingContext, RoutingPlugin + +SYNC_PATH_MESSAGE: Final = ( + "Router(plugins=[...]) is configured, but this call resolved to the synchronous " + "deployment-selection path, which never runs the routing-plugin pipeline. This " + "happens for sync Router methods (e.g. Router.completion()) and for async calls " + "with a routing_strategy that has no async-native selector (e.g. legacy " + "'usage-based-routing', v1). Silently skipping " + "configured plugins would let a policy plugin (e.g. a deny-all rule) be bypassed. " + "Use an async Router method with a supported routing_strategy (simple-shuffle, " + "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), " + "or remove `plugins` from the Router config." +) +_CANDIDATES_KEY: Final = "_routing_plugin_candidate_models" + + +class PluginRegistry(Protocol): + def _core_get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> Sequence[Mapping[str, object]] | None: ... + + +def _resolved_litellm_models( + registry: PluginRegistry, model: str +) -> list[str]: # mutable-ok: the plugins' candidate list + """``_core_resolved_litellm_models``: the provider models a call to ``model`` is served by.""" + return [ + litellm_model + for deployment in registry._core_get_model_list(model_name=model) or () # pyright: ignore[reportPrivateUsage] # the host registry's listing + if isinstance(litellm_model := (as_str_mapping(deployment.get("litellm_params")) or {}).get("model"), str) + and litellm_model + ] + + +def candidate_models(request: Mapping[str, object]) -> Mapping[str, object]: + """The answer to a ``routing_plugins`` step: the candidate models the plugins left on the + request, ``None`` when none narrowed it.""" + metadata: Final = as_str_mapping(request.get(get_metadata_variable_name_from_kwargs(dict(request)))) + return {"models": metadata.get(_CANDIDATES_KEY) if metadata is not None else None} + + +async def run_plugins( + plugins: Sequence[RoutingPlugin], + registry: PluginRegistry, + model_group_alias: Mapping[str, object], + model: str, + request: dict[str, object], # mutable-ok: the plugins stamp their outcome on the caller's request + messages: object, +) -> None: + """``_run_routing_plugins``: run ``plugins`` in order over ``model``'s candidate models, then + stamp their signals (and a narrowed candidate list) on the request's metadata.""" + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + registered: Final = resolve_model_group_alias(model_group_alias, model) or model + models: Final = _resolved_litellm_models(registry, registered) + metadata_key: Final = get_metadata_variable_name_from_kwargs(request) + metadata: Final = request.setdefault(metadata_key, {}) + raw_messages: Final[object] = messages or list[object]() + context = RoutingContext( # rebind-ok: each plugin returns the context the next one reads + raw_messages=raw_messages, # pyright: ignore[reportArgumentType] # the request's messages + structured_messages=resolve_structured_messages(messages=messages, request_kwargs=request) or [], # pyright: ignore[reportArgumentType] # the request's messages + candidate_models=list(models), + metadata=metadata, # pyright: ignore[reportArgumentType] # the request's metadata dict + ) + for plugin in plugins: + context = await plugin.run(context) + metadata["routing_plugin_signals"] = context.signals # pyright: ignore[reportIndexIssue] # the request's metadata dict + if len(context.candidate_models) < len(models): + metadata[_CANDIDATES_KEY] = context.candidate_models # pyright: ignore[reportIndexIssue] # see above + + +__all__ = ["SYNC_PATH_MESSAGE", "PluginRegistry", "candidate_models", "run_plugins"] diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py new file mode 100644 index 00000000000..156c7b5e20f --- /dev/null +++ b/litellm/router_rust/steps/precall.py @@ -0,0 +1,494 @@ +""" +Host-side facts for the Rust core's ``enable_pre_call_checks`` filter. + +The Rust core decides which deployments the pre-call checks drop and which error a group with +nothing left raises (``router-core/src/precall.rs``). What needs live Python objects is answered +here, on a ``pre_call`` select step: the prompt's token count (messages, Responses input and +instructions, tools and an Anthropic ``system`` block), each deployment's context window from +its router model info, and which requested params each deployment takes. Ported from the Python +router core's ``_pre_call_checks`` helpers; it never reaches a Python router core. + +``pre_call_checks`` is the one decision made here: ``RoutingContext.pre_call_checks`` hands over +deployment records rather than a step, and those need not be in the Rust registry. +""" + +from collections.abc import Callable, Mapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol + +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider +from litellm.litellm_core_utils.token_counter import ( + offload_token_count, + served_openai_models, + single_encoding_openai_model, + split_embedding_input_tokens, +) +from litellm.router_python_host import python_host +from litellm.router_strategy.complexity_router.context_compaction import compaction_pending +from litellm.types.router import LiteLLM_Params + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.types.utils import LlmProviders + +# The chat params a deployment must take when the request sends them (`_PRE_CALL_CHECK_FILTERED_PARAMS`). +_FILTERED_PARAMS: Final = frozenset({"response_format"}) + + +class PreCallRegistry(Protocol): + """The host registry reads the facts need.""" + + def _core_get_model_info(self, id: str) -> Mapping[str, object] | None: ... # the registry's name + + def _core_deployments_named(self, model_name: str) -> Sequence[Mapping[str, object]] | None: ... + + def _core_get_router_model_info( + self, deployment: Mapping[str, object], received_model_name: str + ) -> Mapping[str, object]: ... + + +def count_tokens( + host: "PythonHost", + messages: object, + input: object, # the request field's name + request_kwargs: Mapping[str, object] | None, + tokenizer_model: str, + deployments: Sequence[Mapping[str, object]] = (), +) -> int: + """``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus + the tools and Anthropic ``system`` block the message list never carries. Embedding media an + item of ``deployments`` reads as media is left out of the input: it never counts as text.""" + from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message + + extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({}) + raw_instructions: Final = extras.get("instructions") + instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None + raw_tools: Final = extras.get("tools") + tools: Final = raw_tools if _is_list(raw_tools) and raw_tools else None + system_message: Final = anthropic_system_to_openai_message(extras.get("system")) + counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host's overloads name litellm's untyped returns + if messages is not None: + counted_messages: Final = (system_message, *_items(messages)) if system_message is not None else messages + return counter(model=tokenizer_model, messages=counted_messages, tools=tools) + if input is not None: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) + input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped + input=text_input, # pyright: ignore[reportArgumentType] # str | list, as the request carries it + responses_api_request={"instructions": instructions} if instructions is not None else {}, + only_since_compaction=True, + ) + return counter(model=tokenizer_model, messages=input_messages, tools=tools) + raise ValueError("Either messages or input must be provided to count tokens") + + +def _max_input_tokens(registry: PreCallRegistry, model: str, deployment: Mapping[str, object]) -> int | None: + """``_deployment_max_input_tokens``: the declared context window, or None when there is none or + the deployment cannot be resolved.""" + try: + model_info: Final = registry._core_get_router_model_info( # pyright: ignore[reportPrivateUsage] # the host registry's lookup + deployment=deployment, received_model_name=model + ) + except Exception as e: # noqa: BLE001 # best-effort: an unmappable deployment must not hide the others + verbose_router_logger.debug("litellm.router.py::_deployment_max_input_tokens: skipping deployment. Got - %s", e) + return None + max_input_tokens: Final = model_info.get("max_input_tokens") + return max_input_tokens if isinstance(max_input_tokens, int) else None + + +def _items(value: object) -> tuple[object, ...]: + """The items of a list, tuple or set; nothing for any other value.""" + return tuple(value) if isinstance(value, list | tuple | set | frozenset) else () # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # the items are read as objects + + +def _is_list(value: object) -> bool: + return isinstance(value, list) + + +def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + return frozenset(entry for entry in _items(declared) if isinstance(entry, str)) + + +def _capability_model_and_provider(deployment_params: Mapping[str, object], group: str) -> tuple[str, str]: + model_label: Final = str(deployment_params.get("model") or group) + custom_llm_provider: Final = deployment_params.get("custom_llm_provider") + authenticating_provider: Final = declared_authenticating_provider( + model_label, + custom_llm_provider, # pyright: ignore[reportArgumentType] # passed as configured, as the Python core does + ) + if authenticating_provider is not None: + return model_label.removeprefix(f"{authenticating_provider}/"), authenticating_provider + model, provider, _, _ = python_host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # see count_tokens + model=model_label, + custom_llm_provider=custom_llm_provider, + litellm_params=LiteLLM_Params.model_validate(dict(deployment_params)), + ) + return model, provider + + +def _supports_param(deployment: Mapping[str, object], group: str, param: str) -> bool: + """``_deployment_supports_param``.""" + deployment_params: Final = as_str_mapping(deployment.get("litellm_params")) + if not deployment_params: + return True + deployment_model_info: Final = as_str_mapping(deployment.get("model_info")) + base_model: Final = ( + deployment_model_info.get("base_model") if deployment_model_info else None + ) or deployment_params.get("base_model") + try: + model, custom_llm_provider = _capability_model_and_provider(deployment_params, group) + supported: Final = python_host.invoke("get_supported_openai_params")( # pyright: ignore[reportUnknownMemberType] # see count_tokens + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_supports_param: keeping %s for model=%s. Got - %s", param, group, e + ) + return True + return supported is None or param in supported + + +def _lacks_params(deployment: Mapping[str, object], group: str, params: frozenset[str]) -> bool: + return any(not _supports_param(deployment, group, param) for param in params) + + +def _rejects_params( + deployment: Mapping[str, object], group: str, params: frozenset[str], request_kwargs: Mapping[str, object] +) -> bool: + """``_deployment_rejects_params``: lacks a param neither the deployment nor the request drops or allows.""" + call_settings: Final[Mapping[str, object]] = { + **(as_str_mapping(deployment.get("litellm_params")) or {}), + **request_kwargs, + } + if call_settings.get("drop_params") is True: + return False + dropped: Final = call_settings.get("additional_drop_params") + dropped_names: Final = frozenset( + entry for entry in (_items(dropped) if _is_list(dropped) else ()) if isinstance(entry, str) + ) + return _lacks_params(deployment, group, params - _declared_param_allowlist(call_settings) - dropped_names) + + +def _requested_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: + """``_pre_call_check_requested_chat_params``.""" + from litellm import utils as litellm_utils + + return _filtered_params( + litellm_utils.get_non_default_params(passed_params=dict(request_kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an untyped helper + ) + + +def _filtered_params(requested: Mapping[object, object]) -> frozenset[str]: + return frozenset( + param + for param, value in requested.items() + if isinstance(param, str) and param in _FILTERED_PARAMS and not isinstance(value, str) + ) + + +class PreCallFacts: + """The facts of one ``pre_call`` step.""" + + def __init__( + self, + host: "PythonHost", + registry: PreCallRegistry, + step: Mapping[str, object], + records: Callable[[str], Mapping[str, object]], + request: Mapping[str, object] | None, + messages: object, + input: object, # the request field's name + ) -> None: + self._host: Final = host + self._registry: Final = registry + self._model: Final = str(step.get("model_group")) + self._name: Final = str(step.get("model")) + self._candidates: Final = [str(id) for id in _items(step.get("candidates"))] + self._pool: Final = [str(id) for id in _items(step.get("pool"))] + self._records: Final = records + self._request: Final = request + self._messages: Final = messages + self._input: Final = input + self._has_input: Final = messages is not None or input is not None + self._windows: Final = {id: _max_input_tokens(registry, self._model, records(id)) for id in self._candidates} + + def _record(self, id: str) -> Mapping[str, object]: # a deployment id + return self._records(id) + + def _needs_count(self) -> bool: + return self._has_input and any(limit is not None for limit in self._windows.values()) + + def _tokenizer_model(self) -> str: + named: Final = self._registry._core_deployments_named(self._model) or () # pyright: ignore[reportPrivateUsage] # the host registry's index + return ( + single_encoding_openai_model( + served_openai_models(named) or served_openai_models(self._record(id) for id in self._candidates) + ) + or "" + ) + + def _candidate_records(self) -> Sequence[Mapping[str, object]]: + return tuple(self._record(id) for id in self._candidates) + + def _count(self) -> int | None: + try: + return count_tokens( + self._host, + self._messages, + self._input, + self._request, + self._tokenizer_model(), + self._candidate_records(), + ) + except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request + verbose_router_logger.error( + "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e + ) + return None + + async def _count_async(self) -> int | None: + try: + return await offload_token_count(count_tokens)( + self._host, + self._messages, + self._input, + self._request, + self._tokenizer_model(), + self._candidate_records(), + ) + except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request + verbose_router_logger.error( + "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e + ) + return None + + def _answer(self, input_tokens: int | None) -> Mapping[str, object]: + if not self._has_input: + return {"has_input": False} + request: Final[Mapping[str, object]] = self._request if self._request is not None else {} + params: Final = ( + _requested_params(request) + if self._request is not None and self._host.dynamic().drop_params is False + else frozenset[str]() + ) + deployments: Final[dict[str, dict[str, object]]] = {} # mutable-ok: built here, then returned + for id in dict.fromkeys([*self._candidates, *self._pool]): + record = self._record(id) + deployments[id] = { + "max_input_tokens": self._windows.get(id), + "lacks_params": bool(params) and id in self._candidates and _lacks_params(record, self._model, params), + "rejects_params": bool(params) and _rejects_params(record, self._model, params, request), + } + allowed_region: Final = request.get("allowed_model_region") + return { + "has_input": True, + "countable": not compaction_pending(self._request), + "input_tokens": input_tokens, + "allowed_model_region": allowed_region, + "requested_params": sorted(params), + "deployments": deployments, + } + + def answer(self) -> Mapping[str, object]: + """The facts on the sync path: the prompt is counted inline.""" + return self._answer(self._count() if self._needs_count() else None) + + async def answer_async(self) -> Mapping[str, object]: + """The facts on the async path: the prompt is counted off the event loop.""" + return self._answer(await self._count_async() if self._needs_count() else None) + + +def pre_call_checks( + host: "PythonHost", + registry: PreCallRegistry, + model: str, + deployments: Sequence[Mapping[str, object]], + messages: object, + input: object, # the request field's name + request_kwargs: Mapping[str, object] | None, + input_token_count: int | None = None, + skip_inline_token_count: bool = False, + request_deployments: Sequence[Mapping[str, object]] | None = None, +) -> Sequence[Mapping[str, object]]: + """``_pre_call_checks`` over the given deployments, as ``router-core/src/precall.rs`` decides it.""" + import litellm.exceptions as litellm_exceptions + + pool: Final = ( + tuple(request_deployments) + if request_deployments is not None + else (tuple(registry._core_deployments_named(model) or ()) or tuple(deployments)) # pyright: ignore[reportPrivateUsage] # the host registry's index + ) + candidates: Final = {f"candidate-{index}": record for index, record in enumerate(deployments)} + group: Final = {f"pool-{index}": record for index, record in enumerate(pool)} + records: Final = {**candidates, **group} + step: Final = {"model_group": model, "model": model, "candidates": list(candidates), "pool": list(group)} + facts: Final = PreCallFacts(host, registry, step, records.__getitem__, request_kwargs, messages, input) + counted: Final = ( + input_token_count + if input_token_count is not None or skip_inline_token_count or not facts._needs_count() # pyright: ignore[reportPrivateUsage] # this module's own class + else facts._count() # pyright: ignore[reportPrivateUsage] # see above + ) + answer: Final = facts._answer(counted) # pyright: ignore[reportPrivateUsage] # see above + rows: Final = as_str_mapping(answer.get("deployments")) or {} + allowed_region: Final = answer.get("allowed_model_region") + + def row(id: str) -> Mapping[str, object]: # a candidate or pool key + return as_str_mapping(rows.get(id)) or {} + + def over_window(id: str) -> bool: + limit: Final = row(id).get("max_input_tokens") + return isinstance(limit, int) and answer.get("countable") is True and counted is not None and counted > limit + + def out_of_region(id: str) -> bool: + params: Final = as_str_mapping(records[id].get("litellm_params")) or {} + return allowed_region is not None and params.get("region_name") != allowed_region + + too_small: Final = tuple(id for id in candidates if over_window(id)) + invalid: Final = frozenset(too_small) | frozenset(id for id in candidates if out_of_region(id)) + param_candidates: Final = tuple(id for id in candidates if id not in invalid) + lacking: Final = tuple(row(id).get("lacks_params") is True for id in param_candidates) + rejecting: Final = tuple(row(id).get("rejects_params") is True for id in param_candidates) + failing: Final = rejecting if lacking and all(lacking) else lacking + excluded: Final = invalid | frozenset(id for id, fails in zip(param_candidates, failing, strict=True) if fails) + if len(excluded) == len(candidates): + requested: Final = [str(param) for param in _items(answer.get("requested_params"))] + if not too_small and group and all(row(id).get("rejects_params") is True for id in group): + raise litellm_exceptions.UnsupportedParamsError( + message=( + f"no deployment of model={model} supports {requested}. " + "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, " + f"or send allowed_openai_params={requested} to pass them through" + ), + model=model, + ) + if too_small: + detail: Final = "".join( + f"Model={_error_model(records[id])}, Max Input Tokens={row(id).get('max_input_tokens')}, Got={counted}" + for id in too_small + ) + raise litellm_exceptions.ContextWindowExceededError( + message="litellm._pre_call_checks: Context Window exceeded for given call. No models have context " + f"window large enough for this call.\n{detail}", + model=model, + llm_provider="", + ) + return tuple(record for id, record in candidates.items() if id not in excluded) + + +def _error_model(deployment: Mapping[str, object]) -> object: + """The name a context-window error gives a deployment: its base model, else its model.""" + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + return model_info.get("base_model") or params.get("base_model") or params.get("model") + + +def request_carries_image(messages: object, request: Mapping[str, object]) -> bool: + """``_request_carries_image``: the messages (or the request's own messages or input) hold an image.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + return request_contains_image_content(messages or request.get("messages") or request.get("input")) + + +def declares_no_vision(deployment: Mapping[str, object], group: str) -> bool: + """``_deployment_declares_no_vision``: ``model_info.supports_vision`` is ``False``, or, when it + is unset, the cost map says the model has vision explicitly disabled.""" + from litellm import utils as litellm_utils + + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + declared: Final = model_info.get("supports_vision") + if declared is not None: + return declared is False + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + if not params: + return False + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model, custom_llm_provider = _capability_model_and_provider(params, group) + return litellm_utils.is_vision_explicitly_disabled( + base_model if isinstance(base_model, str) else model, custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + +__all__ = [ + "PreCallFacts", + "PreCallRegistry", + "count_tokens", + "declares_no_vision", + "pre_call_checks", + "request_carries_image", +] + + +def _route_capability_resolvers() -> Mapping[str, Callable[[str, "LlmProviders"], object | None]]: + """``_ROUTE_CAPABILITY_RESOLVERS``: the calls only some providers serve, by handler name.""" + from litellm.utils import ProviderConfigManager + + return { + "adecisions": ProviderConfigManager.get_provider_decisions_config, # pyright: ignore[reportUnknownMemberType] # litellm's config lookup + "decisions": ProviderConfigManager.get_provider_decisions_config, # pyright: ignore[reportUnknownMemberType] # see above + } + + +# ``_DECISIONS_MEDIA_CALL_TYPES``: the calls whose audio or videos some deployments refuse. +DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) + + +def serves_route_capability(handler: str | None) -> bool: + """Whether only some providers serve the call ``handler`` names.""" + return handler is not None and handler in _route_capability_resolvers() + + +def cannot_serve_route(deployment: Mapping[str, object], group: str, handler: str) -> bool: + """Not ``_deployment_serves_route``: the deployment's provider has no config for the call.""" + from litellm.types.utils import LlmProviders + + resolver: Final = _route_capability_resolvers().get(handler) + params: Final = as_str_mapping(deployment.get("litellm_params")) + if resolver is None or params is None: + return False + try: + model, provider_name = _capability_model_and_provider(params, group) + provider: Final = next((member for member in LlmProviders if member.value == provider_name), None) + return provider is None or resolver(model, provider) is None + except Exception as e: # noqa: BLE001 # best-effort capability filter, as the Python core keeps it + verbose_router_logger.debug( + "litellm.router.py::_deployment_serves_route: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + +def refuses_decisions_media( + deployment: Mapping[str, object], group: str, sent_kwargs: Mapping[str, object], *, audio: bool, videos: bool +) -> bool: + """``_decisions_deployment_refuses_media``.""" + from litellm.decisions.call import deployment_refuses_media + + model_info: Final = as_str_mapping(deployment.get("model_info")) or {} + params: Final = as_str_mapping(deployment.get("litellm_params")) or {} + try: + model, custom_llm_provider = _capability_model_and_provider(params, group) + return deployment_refuses_media( + model, custom_llm_provider, model_info, {**params, **sent_kwargs}, audio=audio, videos=videos + ) + except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself + verbose_router_logger.debug( + "litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s", + group, + e, + ) + return False diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py new file mode 100644 index 00000000000..a83fbe29996 --- /dev/null +++ b/litellm/router_rust/steps/prerouting.py @@ -0,0 +1,1107 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAny=false, reportExplicitAny=false +# (the ported core code reads the untyped strategy routers, LiteLLM_Params dict fields and request dicts) +""" +Host-side pre-routing for ``RustCore``. + +Before an async pick resolves its name, the Rust core asks the host to pre-route it (a +``pre_route`` step, ``router-core/src/prerouting.rs``): the complexity, auto, adaptive and quality +routers registered from ``auto_router/...`` marker deployments, and the Claude Code session +binding, may send the request to another model group. They are live Python objects, so they stay +host-side for now (bridged, to port): this module registers them from the host registry's +deployments the way the Python router's ``init_*_router_deployment``, +``_finalize_adaptive_router_if_configured`` and ``_unregister_pre_routing_strategy_for_deployment`` +do, and answers the step the way ``_core_async_pre_routing_hook`` and ``_pin_tier_params_onto_request`` +do. It never reaches a Python router core. +""" + +import json +import logging +import re +from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast # noqa: TID251 # the strategy routers are untyped + +from litellm._internal_context import with_service_target +from litellm._logging import verbose_router_logger +from litellm.caching.caching import DualCache, InMemoryCache +from litellm.caching.redis_cache import log_redis_failure +from litellm.constants import ( + CLIENT_OUTPUT_CEILING_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, + OUTPUT_TOKEN_CEILING_PARAMS, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, +) +from litellm.integrations.otel.routing import routing_decision_attributes +from litellm.integrations.otel.runtime import phase_attributes +from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, without +from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference +from litellm.router_api import deployment_id +from litellm.router_host import router_host +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, + STRATEGY_ROUTER_PARAM_FIELDS, + GatedAutoRouterCapability, + capability_limit_violation, + claimed_capabilities, + classify_strategy_router_model, + count_capability_routers, +) +from litellm.router_strategy.complexity_router.context_compaction import ( + arm_compaction, + is_native_compaction_call, + reject_recursive_compactor, +) +from litellm.router_strategy.tag_based_routing import get_routing_tags_from_request_kwargs, is_valid_deployment_tag +from litellm.router_utils.fallback_event_handlers import record_pre_routing_selection +from litellm.types.router import ConsumedRequestTagsStamp, Deployment, LiteLLM_Params, TaggedPreRoutingStrategy +from litellm.types.utils import ( + PROMPT_QUOTING_ROUTING_DECISION_FIELDS, + CustomPricingLiteLLMParams, + StandardLoggingRoutingDecision, + all_litellm_params, +) +from litellm.utils import provider_rejectable_params + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_strategy.auto_router.auto_router import PreRoutingHookResponse + +# The live registry dicts: model_name -> the strategies registered under it, one per tag set. +StrategyRegistry: TypeAlias = "dict[str, list[TaggedPreRoutingStrategy[Any]]]" + +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( + {"model", "api_base", "api_key", "api_version", "responses_api_version"} +) +_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") +_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 +CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" +# `_async_get_available_deployment` runs the pre-routing hook only for the strategies with an async +# pick; any other router strategy takes the sync path, which never pre-routes. +_ASYNC_PICK_STRATEGIES: Final = frozenset( + {"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"} +) + + +def picks_async(router_strategy: str | None) -> bool: + """Whether the router's strategy has an async selector: the others pick on the sync path even + for async callers (``_async_get_available_deployment``).""" + return router_strategy in _ASYNC_PICK_STRATEGIES + + +TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset( + { + "additional_drop_params", + "drop_params", + "messages", + "model", + "extra_headers", + "max_tokens", + "max_completion_tokens", + } +) + + +def _json_default_stable_id(value: object) -> str: + return f"{type(value).__module__}.{type(value).__qualname__}" + + +def _deployment_tags(deployment: Deployment) -> tuple[str, ...]: + return tuple(deployment.litellm_params.tags or ()) + + +def _has_registered_strategy(registry: StrategyRegistry, model_name: str, tags: tuple[str, ...]) -> bool: + return any(existing.tags == tags for existing in registry.get(model_name, [])) + + +def _routing_definition_fingerprint(deployment: Deployment) -> str | None: + """Identify configured routing fields, excluding external files and live strategy state.""" + try: + definition: Final = json.dumps( + deployment.litellm_params.model_dump( + include=set(STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}), exclude_none=True, warnings=False + ), + sort_keys=True, + default=_json_default_stable_id, + ) + except (TypeError, ValueError): + return None + return deployment_id(deployment.model_name, {"routing_definition": definition}) + + +def _unregister(registry: StrategyRegistry, model_name: str, tags: tuple[str, ...]) -> bool: + existing: Final = registry.get(model_name, []) + remaining: Final = [entry for entry in existing if entry.tags != tags] + if len(remaining) == len(existing): + return False + if remaining: + registry[model_name] = remaining # rebind-ok: the live registry, edited in place as the Python router does + else: + registry.pop(model_name, None) + return True + + +def is_strategy_marker(deployment: Mapping[str, object]) -> bool: + params: Final = deployment.get("litellm_params") + if not isinstance(params, Mapping): + return False + model: Final = params.get("model") + return isinstance(model, str) and classify_strategy_router_model(model) is not None + + +def _participates_in_adaptive_routing(params: LiteLLM_Params) -> bool: + kind: Final = classify_strategy_router_model(params.model) + if kind == "adaptive": + return True + if kind != "complexity": + return False + config: Final = params.complexity_router_config + return bool(config.get("adaptive")) if config else False + + +class PreRoutingStrategies: + """The pre-routing strategies of one router, kept on its host registry (``registry().plugins`` + reads ``_core_auto_routers`` and its siblings there).""" + + def __init__( + self, hosted: "HostRegistry", host: "PythonHost", ctx: object, capability_limit: object, cache: DualCache + ) -> None: + self._hosted: Final = hosted + self._host: Final = host + self._ctx: Final = ctx + self._capability_limit: object = capability_limit + self._cache: Final = cache + self._affinity: object = None + # The router's own `_ensure_deployment_affinity_callback` (its optional checks), when set. + self.ensure_affinity: Callable[[], None] | None = None + + # -- the registries ------------------------------------------------------------------------ + + @property + def auto(self) -> StrategyRegistry: + return self._hosted._core_auto_routers # pyright: ignore[reportPrivateUsage] # the registries live on the host registry + + @property + def complexity(self) -> StrategyRegistry: + return self._hosted._core_complexity_routers # pyright: ignore[reportPrivateUsage] # see above + + @property + def adaptive(self) -> StrategyRegistry: + return self._hosted._core_adaptive_routers # pyright: ignore[reportPrivateUsage] # see above + + @property + def quality(self) -> StrategyRegistry: + return self._hosted._core_quality_routers # pyright: ignore[reportPrivateUsage] # see above + + def all(self) -> tuple[StrategyRegistry, ...]: + """In the order ``_select_pre_routing_strategy`` searches them.""" + return (self.auto, self.complexity, self.adaptive, self.quality) + + def any_registered(self) -> bool: + return any(self.all()) + + # -- registry hooks (HostRegistry calls these) ----------------------------------------------- + + def reset(self) -> None: + """``set_model_list`` drops the auto, complexity and quality routers (never the adaptive ones).""" + self._hosted._core_quality_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_complexity_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above + + def register(self, deployment: Deployment) -> None: + """The end of ``_add_deployment``: an auto, complexity or quality marker registers its router + (adaptive ones wait for ``finalize``).""" + match classify_strategy_router_model(deployment.litellm_params.model): + case "semantic": + self._init_auto_router(deployment) + case "complexity": + self._init_complexity_router(deployment) + case "quality": + self._init_quality_router(deployment) + case _: + pass + + def release(self, deployment: Deployment) -> None: + """``_unregister_pre_routing_strategy_for_deployment``.""" + if not deployment.litellm_params.model.startswith("auto_router/"): + return + model_name: Final = deployment.model_name + tags: Final = _deployment_tags(deployment) + for registry in (self.auto, self.complexity, self.quality): + _ = _unregister(registry, model_name, tags) + if _unregister(self.adaptive, model_name, tags): + self._sync_adaptive_router_hooks() + + def release_entry(self, entry: Mapping[str, object]) -> None: + """``release`` of a deleted model_list entry; a failure only logs, as the delete has happened.""" + try: + self.release(Deployment(**entry)) # pyright: ignore[reportArgumentType] # a model_list entry + except Exception: # noqa: BLE001 # the delete already happened; a failed release only logs + verbose_router_logger.exception( + "delete_deployment: could not release pre-routing strategies for model_id=%s; " + "the deployment is out of the model_list and its indices are repaired", + as_dict(entry.get("model_info")).get("id"), + ) + + def upserted(self, deployment: Deployment, previous: Deployment | None) -> None: + """After an upsert's re-add: rebuild the adaptive routers when it touched one.""" + if _participates_in_adaptive_routing(deployment.litellm_params) or ( + previous is not None and _participates_in_adaptive_routing(previous.litellm_params) + ): + self.finalize() + + def without_capability_limit(self, action: Callable[[], object]) -> None: + """Run ``action`` without judging the capability ceiling: a rollback re-admits a serving router.""" + resolver: Final = self._capability_limit + self._capability_limit = None + try: + _ = action() + finally: + self._capability_limit = resolver + + def finalize(self) -> None: + """``_finalize_adaptive_router_if_configured``: an adaptive router for every adaptive marker, + and the adaptive companion of every complexity router configured with ``adaptive``.""" + for entry in self._hosted._core_model_list or []: # pyright: ignore[reportPrivateUsage] # the host registry's live list + params = entry.get("litellm_params") + lp_model = params.get("model") if isinstance(params, dict) else None + if not (isinstance(lp_model, str) and lp_model.startswith("auto_router/adaptive_router")): + continue + model_name = entry.get("model_name") + if not model_name or not params: + continue + deployment = Deployment( + model_name=str(model_name), + litellm_params=LiteLLM_Params.model_validate(params), + model_info=entry.get("model_info"), # pyright: ignore[reportArgumentType] # the entry's model_info + ) + if _has_registered_strategy(self.adaptive, str(model_name), _deployment_tags(deployment)): + continue + self._init_adaptive_router(deployment) + for model_name, tagged_complexity_routers in self.complexity.items(): + for tagged in tagged_complexity_routers: + complexity_router = tagged.strategy + if not complexity_router.config.adaptive: + continue + if _has_registered_strategy(self.adaptive, model_name, tagged.tags): + continue + adaptive_router = complexity_router._ensure_adaptive_router() + if adaptive_router is not None: + self.adaptive[model_name] = [ + *self.adaptive.get(model_name, []), + TaggedPreRoutingStrategy( + tags=tagged.tags, + strategy=adaptive_router, + definition_fingerprint=tagged.definition_fingerprint, + deployment=tagged.deployment, + ), + ] + self._sync_adaptive_router_hooks() + + def retire(self) -> None: + """Release the adaptive routers' post-call hooks this router registered.""" + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + + own: Final = frozenset(id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive.values())) + hooks: Final[list[object]] = [ # mutable-ok: built here + hook + for hook in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own + ] + if self._affinity is not None: + hooks.append(self._affinity) + self._affinity = None + if hooks: + self._host.callbacks.drop_callbacks(hooks) + + # -- building -------------------------------------------------------------------------------- + + def _register( + self, registry: StrategyRegistry, deployment: Deployment, strategy: object, strategy_label: str + ) -> None: + tags: Final = _deployment_tags(deployment) + if _has_registered_strategy(registry, deployment.model_name, tags): + raise ValueError( + f"{strategy_label} deployment {deployment.model_name} with tags {list(tags)} already exists. " + "Please use a different model name or set different tags." + ) + registry[deployment.model_name] = [ # rebind-ok: the live registry, edited in place as the Python router does + *registry.get(deployment.model_name, []), + TaggedPreRoutingStrategy( + tags=tags, + strategy=strategy, + definition_fingerprint=_routing_definition_fingerprint(deployment), + deployment=deployment, + ), + ] + + def _capability_violation(self, capability: GatedAutoRouterCapability) -> str | None: + resolver: Final = self._capability_limit + # `self.auto_router_capability_limit() if ... is not None`: a value that cannot be called raises. + limit: Final = resolver() if resolver is not None else None # pyright: ignore[reportCallIssue] # called as given, as Python calls it + others: Final = count_capability_routers( + (entry for entry in self._hosted._core_model_list if isinstance(entry, Mapping)), # pyright: ignore[reportPrivateUsage, reportUnnecessaryIsInstance] # the live list + capability=capability, + ) + return capability_limit_violation(capability=capability, held=others + 1, limit=limit) # pyright: ignore[reportArgumentType] # the resolver's limit + + def _init_auto_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.auto_router.auto_router import AutoRouter + + params: Final = deployment.litellm_params + if params.auto_router_config_path is None and params.auto_router_config is None: + raise ValueError( + "auto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_params" + ) + if params.auto_router_default_model is None: + raise ValueError( + "auto_router_default_model is required for auto-router deployments. Please set it in the litellm_params" + ) + if params.auto_router_embedding_model is None: + raise ValueError( + "auto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params" + ) + max_input_chars: Final = params.auto_router_max_input_chars + router: Final = AutoRouter( + model_name=deployment.model_name, + auto_router_config_path=params.auto_router_config_path, + auto_router_config=params.auto_router_config, + default_model=params.auto_router_default_model, + embedding_model=params.auto_router_embedding_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + max_input_chars=max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, + ) + self._register(self.auto, deployment, router, "Auto-router") + + def _init_complexity_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig + + config: Final = deployment.litellm_params.complexity_router_config + for capability in claimed_capabilities(config): + if (violation := self._capability_violation(capability)) is not None: + raise ValueError(violation) + default_model: Final = ( + ComplexityRouterConfig.model_validate(config).resolve_default_model( + deployment.litellm_params.complexity_router_default_model + ) + if config + else deployment.litellm_params.complexity_router_default_model + ) + if default_model is None: + raise ValueError( + "complexity_router_default_model is required for complexity-router deployments, " + "or configure tiers in complexity_router_config. Please set it in the litellm_params" + ) + router: Final = ComplexityRouter( + model_name=deployment.model_name, + default_model=default_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + complexity_router_config=config, + ) + self._register(self.complexity, deployment, router, "Complexity-router") + if router._uses_deployment_pin: # pyright: ignore[reportPrivateUsage] # the Python router reads the same flag + self._ensure_deployment_affinity_callback() + + def _ensure_deployment_affinity_callback(self) -> None: + """``_ensure_deployment_affinity_callback``: the affinity check (every global flag off) a + complexity router's session pin needs, registered once.""" + from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck + + if self.ensure_affinity is not None: + self.ensure_affinity() + return + if self._affinity is not None: + return + hosted: Final = self._hosted + + def is_priority_group(model: str) -> bool: + resolved: Final = hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + group: Final = hosted.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + + affinity: Final = DeploymentAffinityCheck( + cache=self._cache, + ttl_seconds=3600, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=False, + model_group_affinity_config=None, + is_priority_group=is_priority_group, + session_affinity_repin_on_failover=False, + ) + self._affinity = affinity + self._host.callbacks.add_litellm_callback(affinity) + + def _init_quality_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.quality_router.quality_router import QualityRouter + + config: Final = deployment.litellm_params.quality_router_config + configured: Final = deployment.litellm_params.quality_router_default_model + default_model: Final = configured if configured is not None or not config else config.get("default_model") + if default_model is None: + raise ValueError( + "quality_router_default_model is required for quality-router deployments, " + "or set default_model in quality_router_config. Please configure it in the litellm_params" + ) + router: Final = QualityRouter( + model_name=deployment.model_name, + default_model=default_model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + quality_router_config=config, + ) + self._register(self.quality, deployment, router, "Quality-router") + + def _init_adaptive_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + from litellm.types.router import AdaptiveRouterConfig, AdaptiveRouterPreferences + + raw_config: Final = deployment.litellm_params.adaptive_router_config + if raw_config is None: + raise ValueError("adaptive_router_config is required for adaptive-router deployments.") + config: Final = AdaptiveRouterConfig(**raw_config) + model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} # mutable-ok: built here + model_to_cost: Final[dict[str, float]] = {} # mutable-ok: built here + for name in config.available_models: + deployments = self._hosted._core_deployments_named(name) # pyright: ignore[reportPrivateUsage] # the host registry's name index + if not deployments: + continue + first = deployments[0] + model_info = first.get("model_info") + info: Mapping[str, object] = model_info if isinstance(model_info, dict) else {} + prefs_raw = info.get("adaptive_router_preferences") + if isinstance(prefs_raw, dict): + model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) + cost = _adaptive_cost(as_dict(first.get("litellm_params")), info) + if isinstance(cost, int | float | str): + model_to_cost[name] = float(cost) + router: Final = AdaptiveRouter( + router_name=deployment.model_name, + config=config, + model_to_prefs=model_to_prefs, + model_to_cost=model_to_cost, + ) + self._register(self.adaptive, deployment, router, "Adaptive-router") + self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=router)) + verbose_router_logger.info( + "AdaptiveRouter[%s] initialized with %d models", deployment.model_name, len(config.available_models) + ) + + def _sync_adaptive_router_hooks(self) -> None: + """One post-call hook per registered adaptive router, plus the header hook while any exists.""" + from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, + ) + + callbacks: Final = self._host.callbacks + for callback in callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): + callbacks.remove_callback_from_all_lists(callback) + for tagged in chain.from_iterable(self.adaptive.values()): + callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)) + if self.adaptive: + callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + + +# -- the request side -------------------------------------------------------------------------- + + +def _stamp_or_clear_metadata_key( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs, stamped in place + key: str, + value: object | None, +) -> None: + """Write a proxy-internal metadata key for this routing attempt, or clear it from both buckets.""" + if value is None: + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + bucket.pop(key, None) + return + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket[key] = value + + +def _redact_prompt_text_if_needed( + request_kwargs: Mapping[str, object], routing_decision: StandardLoggingRoutingDecision +) -> StandardLoggingRoutingDecision: + from litellm.litellm_core_utils.redact_messages import should_redact_message_logging + + if not should_redact_message_logging( + { + "litellm_params": request_kwargs, + "standard_callback_dynamic_params": request_kwargs.get("standard_callback_dynamic_params"), + } + ): + return routing_decision + kept: Final = without(routing_decision, *PROMPT_QUOTING_ROUTING_DECISION_FIELDS) + return cast(StandardLoggingRoutingDecision, kept) # cast-ok: dropping optional keys preserves the type + + +def _record_routing_decision( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs + routing_decision: StandardLoggingRoutingDecision | None, +) -> None: + """Make the request's metadata describe this routing attempt, and only this one.""" + from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters + from litellm.types.router import BaselineRouteStamp + + phase_attributes(routing_decision_attributes(routing_decision)) + baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None + baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None + router_name: Final = routing_decision.get("router_model_name") if routing_decision else None + caller_parameters: Final = capture_baseline_parameters(request_kwargs) if router_name and baseline_model else None + _stamp_or_clear_metadata_key( + request_kwargs, + "_autorouter_baseline_route", + BaselineRouteStamp(router_name, baseline_model, baseline_id, caller_parameters) + if router_name and baseline_model and baseline_id + else None, + ) + _stamp_or_clear_metadata_key( + request_kwargs, + "routing_decision", + None if routing_decision is None else _redact_prompt_text_if_needed(request_kwargs, routing_decision), + ) + + +def _as_output_cap(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + cap: Final = int(float(value)) + except (ValueError, OverflowError): + return None + return cap if cap >= 0 else None + + +def _stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str, int] | None: + stamped: Final = next( + ( + bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY) + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket + ), + None, + ) + if not isinstance(stamped, dict): + return None + return MappingProxyType( + { + str(carrier): cap + for carrier, value in stamped.items() + if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := _as_output_cap(value)) is not None + } + ) + + +def _restore_client_output_ceiling( + request_kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs, restored in place +) -> None: + callers_ceiling: Final = _stamped_client_ceiling(request_kwargs) + if callers_ceiling is None: + return + for carrier in OUTPUT_TOKEN_CEILING_PARAMS: + request_kwargs.pop(carrier, None) + request_kwargs.update(callers_ceiling) + + +def _stamp_client_output_ceiling(request_kwargs: dict[str, object]) -> None: # mutable-ok: the request kwargs + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket.setdefault( + CLIENT_OUTPUT_CEILING_METADATA_KEY, + {carrier: request_kwargs[carrier] for carrier in OUTPUT_TOKEN_CEILING_PARAMS if carrier in request_kwargs}, + ) + + +def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None: # mutable-ok: same + nested: Final = request_kwargs.get(carrier) + if not isinstance(nested, Mapping): + return + sanitized: Final = without(nested, "effort") + if sanitized: + request_kwargs[carrier] = sanitized # rebind-ok: copy-on-write of the request's nested carrier + else: + request_kwargs.pop(carrier, None) + + +def _tier_ceiling_under_the_surface_name( + tier_litellm_params: Mapping[str, object], responses_call: bool +) -> Mapping[str, object]: + surface_key: Final = "max_output_tokens" if responses_call else "max_tokens" + carried: Final = tuple( + key + for key in (surface_key, "max_tokens", "max_completion_tokens", "max_output_tokens") + if key in tier_litellm_params + ) + if not carried: + return tier_litellm_params + return MappingProxyType( + { + **without(tier_litellm_params, *OUTPUT_TOKEN_CEILING_PARAMS), + surface_key: tier_litellm_params[carried[0]], + } + ) + + +def _drop_client_carriers_a_tier_pin_supersedes( + request_kwargs: dict[str, object], # mutable-ok: the request kwargs + tier_litellm_params: Mapping[str, object], +) -> None: + if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params): + _stamp_client_output_ceiling(request_kwargs) + for carrier in OUTPUT_TOKEN_CEILING_PARAMS: + request_kwargs.pop(carrier, None) + if "reasoning_effort" not in tier_litellm_params: + return + request_kwargs.pop("thinking", None) + _pop_effort_from_nested_carrier(request_kwargs, "output_config") + _pop_effort_from_nested_carrier(request_kwargs, "reasoning") + + +def _request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None: + proxy_server_request: Final = request_kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, Mapping): + return None + headers: Final = proxy_server_request.get("headers") + if not isinstance(headers, Mapping): + return None + return next( + ( + value + for key, value in headers.items() + if isinstance(key, str) and key.lower() == header_name and isinstance(value, str) + ), + None, + ) + + +def _session_router_cache_key(request_kwargs: Mapping[str, object]) -> str | None: + session_id: Final = _request_header(request_kwargs, "x-claude-code-session-id") + if session_id is None or _CLAUDE_CODE_SESSION_ID_RE.fullmatch(session_id) is None: + return None + metadata_name: Final = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata" + metadata: Final = request_kwargs.get(metadata_name) + if not isinstance(metadata, Mapping): + return None + caller_scope: Final = metadata.get("user_api_key_hash") + if not isinstance(caller_scope, str) or not caller_scope: + return None + return f"claude_code_session_router:v1:{caller_scope}:{session_id}" + + +class PreRouter: + """Answers the Rust core's ``pre_route`` steps (``_core_async_pre_routing_hook``).""" + + def __init__( + self, + strategies: PreRoutingStrategies, + hosted: "HostRegistry", + host: "PythonHost", + ctx: object, + routing_cache: DualCache, + ) -> None: + self._strategies: Final = strategies + self._hosted: Final = hosted + self._host: Final = host + self._ctx: Final = ctx + self._session_cache: Final = DualCache(redis_cache=routing_cache.redis_cache, in_memory_cache=InMemoryCache()) + + # -- the step -------------------------------------------------------------------------------- + + async def pre_route( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the request kwargs the hook stamps + *, + messages: object, + input: object, + fallback_depth: int, + router_strategy: str | None, + enable_tag_filtering: bool, + match_any: bool, + public_router: object, + run_plugins: Callable[[str], Awaitable[None]] | None = None, + pass_through: bool = False, + ) -> str | None: + """The start of ``_async_get_available_deployment``: run the hook, record its pick and pin the + tier's litellm_params. The model group to resolve, or ``None`` to resolve ``model`` itself. + ``pass_through`` is ``_async_get_available_deployment_for_pass_through``, which runs the hook + whatever the router's strategy.""" + if not pass_through and not picks_async(router_strategy): + return None + responses_call: Final = input is not None and messages is None + response: Final = await self._hook( + model, + request_kwargs, + messages=messages, + input=input, + fallback_depth=fallback_depth, + enable_tag_filtering=enable_tag_filtering, + match_any=match_any, + public_router=public_router, + run_plugins=run_plugins, + ) + routed: Final = response.model if response is not None else model + if response is not None: + record_pre_routing_selection(request_kwargs, routed) + self._pin_tier_params( + routed, response.litellm_params if response is not None else None, request_kwargs, responses_call + ) + return routed if response is not None else None + + # -- the hook -------------------------------------------------------------------------------- + + async def _hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + *, + messages: object, + input: object, + fallback_depth: int, + enable_tag_filtering: bool, + match_any: bool, + public_router: object, + run_plugins: Callable[[str], Awaitable[None]] | None, + ) -> "PreRoutingHookResponse | None": + _restore_client_output_ceiling(request_kwargs) + _record_routing_decision(request_kwargs, None) + requested: Final = self._hosted._core_get_model_from_alias(model=model) or model # pyright: ignore[reportPrivateUsage] # the host registry's alias lookup + registered: Final = await self._resolve_session_router( + model, requested, request_kwargs, fallback_depth, enable_tag_filtering, match_any + ) + # The routing plugins run on the session-resolved name, before a strategy is selected, so a + # strategy reads their signals; router-core narrows the pick to what they left. + if run_plugins is not None: + await run_plugins(registered) + selected: Final = self._select_strategy(registered, request_kwargs, enable_tag_filtering, match_any) + if selected is None: + await arm_compaction(request_kwargs, None) + _stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, None) + _stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, None) + return None + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + + marker: Final = self._selected_marker(registered, selected.tags, request_kwargs) + definition: Final = selected.deployment + config_id: Final = definition.model_info.id if definition else None + updated_at: Final = definition.model_info.updated_at if definition else None + router_kind: Final = classify_strategy_router_model(definition.litellm_params.model) if definition else None + provenance: Final[StandardLoggingRoutingDecision] = { + "router_model_name": definition.model_name if definition else registered, + **({"router_type": router_kind} if router_kind is not None else {}), + **({"router_config_id": config_id} if config_id is not None else {}), + **({"router_config_updated_at": updated_at.isoformat()} if updated_at is not None else {}), + **( + {"router_config_fingerprint": selected.definition_fingerprint} + if selected.definition_fingerprint is not None + else {} + ), + } # pyright: ignore[reportAssignmentType] # the optional keys, as the Python router builds them + phase_attributes(routing_decision_attributes(provenance)) + reject_recursive_compactor(registered) + strategy: Final = selected.strategy + complexity: Final = strategy if isinstance(strategy, ComplexityRouter) else None + await arm_compaction( + request_kwargs, + complexity.config.context_compaction if complexity is not None else None, + _tier_members(complexity.config.tiers) if complexity is not None else (), # pyright: ignore[reportArgumentType] # the tiers' model names + parent_model=model, + ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext + allow_escalation=complexity is not None and complexity.config.enable_context_window_escalation, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + ) + await router_host().authorize_strategy_inference( + deployment=marker, + request_kwargs=request_kwargs, + llm_router=public_router, # pyright: ignore[reportArgumentType] # the litellm.Router this core serves + ) + compression_policy: Final = router_host().compression_policy( + llm_router=public_router, # pyright: ignore[reportArgumentType] # same + model_alias=registered, + request_kwargs=request_kwargs, + request_tags=get_routing_tags_from_request_kwargs(request_kwargs), + ) + independent_compression: Final = compression_policy is not None and not ( + compression_policy.is_same + and compression_policy.model is not None + and router_host().model_hop_compression_armed() + ) + routing_messages: Final = ( + await router_host().messages_for_routing( + policy=compression_policy, + messages=messages, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + if independent_compression + else None + ) + routed: Final = await strategy.async_pre_routing_hook( + model=registered, + request_kwargs=request_kwargs, + messages=routing_messages if routing_messages is not None else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=input, # pyright: ignore[reportArgumentType] # the request's input, as given + specific_deployment=False, + ) + hooked: Final[PreRoutingHookResponse | None] = routed + traced: Final = ( + hooked.model_copy(update={"routing_decision": {**provenance, **hooked.routing_decision}}) + if hooked is not None and hooked.routing_decision is not None + else hooked + ) + response: Final = ( + traced.model_copy(update={"messages": messages}) + if traced is not None and routing_messages is not None and traced.messages == routing_messages + else traced + ) + _record_routing_decision(request_kwargs, response.routing_decision if response else None) + _stamp_or_clear_metadata_key( + request_kwargs, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + response.session_affinity_ttl_seconds if response else None, + ) + request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) + consumed: Final = ( + ConsumedRequestTagsStamp(model_group=response.model, tags=selected.tags) + if response is not None + and selected.tags + and request_tags + and is_valid_deployment_tag(selected.tags, request_tags, match_any) # pyright: ignore[reportArgumentType] # the tags as the Python router passes them + else None + ) + _stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, consumed) + marker_params: Final = ( + self._forwardable_marker_params(registered, selected.tags, request_kwargs) if response is not None else () + ) + tier_param_keys: Final = tuple(response.litellm_params or ()) if response is not None else () + overridden: Final = frozenset(tier_param_keys) | ( + OUTPUT_TOKEN_CEILING_PARAMS if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_param_keys) else frozenset() + ) + newly_forwarded: Final = tuple( + (key, value) for key, value in marker_params if key not in request_kwargs and key not in overridden + ) + if response is not None: + _stamp_client_output_ceiling(request_kwargs) + request_kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, None) + request_kwargs.update(newly_forwarded) + if newly_forwarded: + request_kwargs.update({_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: tuple(key for key, _ in newly_forwarded)}) + return response + + # -- strategy selection ---------------------------------------------------------------------- + + def _deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[Mapping[str, object]]: + return self._hosted._core_deployments_for_request(model, request_kwargs) # pyright: ignore[reportPrivateUsage] # the host registry's lookup + + def _select_strategy( + self, model: str, request_kwargs: Mapping[str, object], enable_tag_filtering: bool, match_any: bool + ) -> "TaggedPreRoutingStrategy[Any] | None": + """``_select_pre_routing_strategy``.""" + registries: Final = self._strategies.all() + if not any(registries): + return None + deployments: Final = self._deployments_for_request(model, request_kwargs) + registered_names: Final = tuple(dict.fromkeys(str(d.get("model_name")) for d in deployments)) or (model,) + candidates: Final = ( + tuple( # comprehension-ok: registry by registry, name by name, as the Python router searches + tagged for registry in registries for name in registered_names for tagged in registry.get(name, []) + ) + ) + if not candidates: + return None + request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) + if request_tags: + for tagged in candidates: + if tagged.tags and is_valid_deployment_tag(list(tagged.tags), request_tags, match_any): + return tagged + for tagged in candidates: + if "default" in tagged.tags: + return tagged + request_scoped: Final = request_kwargs.get("enable_tag_filtering") is True + if ( + (enable_tag_filtering or request_scoped) + and all(tagged.tags for tagged in candidates) + and any(not is_strategy_marker(d) for d in deployments) + ): + return None + return candidates[0] + + def _selected_marker( + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object] | None: + markers: Final = tuple( + deployment + for deployment in self._deployments_for_request(model, request_kwargs) + if str(as_dict(deployment.get("litellm_params")).get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + and "model" in as_dict(deployment.get("litellm_params")) + ) + tag_matched: Final = tuple( + deployment + for deployment in markers + if tuple(_tags_of(as_dict(deployment.get("litellm_params")))) == strategy_tags + ) + return tag_matched[0] if tag_matched else (markers[0] if markers else None) + + def _forwardable_marker_params( + self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object] + ) -> tuple[tuple[str, object], ...]: + marker: Final = self._selected_marker(model, strategy_tags, request_kwargs) + if marker is None: + return () + return tuple( + (key, value) + for key, value in as_dict(marker.get("litellm_params")).items() + if key not in _ALIAS_PARAMS_NEVER_FORWARDED + and key not in CustomPricingLiteLLMParams.model_fields + and value is not None + ) + + # -- the Claude Code session binding --------------------------------------------------------- + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _delete_binding(self, cache_key: str) -> None: + try: + await self._session_cache.async_delete_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request + verbose_router_logger.warning( + "Failed to delete Claude Code session router binding; the binding may remain until its TTL expires: %s", + e, + ) + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _get_binding(self, cache_key: str) -> object: + try: + if self._session_cache.redis_cache is None: + return await self._session_cache.async_get_cache(key=cache_key) + return await self._session_cache.redis_cache.async_get_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # an optional binding must not make routing depend on Redis + log_redis_failure( + verbose_router_logger, + logging.WARNING, + "Failed to read Claude Code session router binding; using the requested model", + e, + ) + return None + + @with_service_target(CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET) + async def _resolve_session_router( + self, + model: str, + registered: str, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + fallback_depth: int, + enable_tag_filtering: bool, + match_any: bool, + ) -> str: + """``_resolve_claude_code_session_router``.""" + if is_native_compaction_call() or not self._strategies.any_registered(): + return registered + cache_key: Final = _session_router_cache_key(request_kwargs) + if cache_key is None: + return registered + depth: Final = fallback_depth or request_kwargs.get("fallback_depth") + if depth not in (None, 0): + return registered + agent_id: Final = _request_header(request_kwargs, "x-claude-code-agent-id") + if agent_id is not None: + bound_model: Final = await self._get_binding(cache_key) + if not isinstance(bound_model, str): + return registered + bound_registered: Final = self._hosted._core_get_model_from_alias(model=bound_model) or bound_model # pyright: ignore[reportPrivateUsage] # see _hook + if self._select_strategy(bound_registered, request_kwargs, enable_tag_filtering, match_any) is None: + await self._delete_binding(cache_key) + return registered + await self._session_cache.async_set_cache( + key=cache_key, value=bound_model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS + ) + _stamp_or_clear_metadata_key(request_kwargs, "model_group", bound_model) + return bound_registered + if self._select_strategy(registered, request_kwargs, enable_tag_filtering, match_any) is None: + return registered + await self._session_cache.async_set_cache( + key=cache_key, value=model, ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS + ) + return registered + + # -- tier litellm_params --------------------------------------------------------------------- + + def _tier_params_the_target_accepts( + self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + from litellm.router_rust.invoke.shaping import deployment_accepts_param + + deployments: Final = self._hosted._core_get_model_list(model_name=model) or () # pyright: ignore[reportPrivateUsage] # the host registry's lookup + if not deployments: + return tier_params + allowlisted: Final = _allowlist(tier_params) | _allowlist(request_kwargs) + candidates: Final = provider_rejectable_params(tier_params) - TIER_PARAMS_NEVER_DROPPED - allowlisted + unsupported: Final = frozenset( + param + for param in candidates + if not any(deployment_accepts_param(deployment, model, param, self._host) for deployment in deployments) + ) + if not unsupported: + return tier_params + verbose_router_logger.warning( + "litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them", + ", ".join(sorted(unsupported)), + model, + ) + return MappingProxyType(without(tier_params, *unsupported)) + + def _pin_tier_params( + self, + model: str, + tier_litellm_params: Mapping[str, object] | None, + request_kwargs: dict[str, object], # mutable-ok: see pre_route + responses_call: bool, + ) -> None: + """``_pin_tier_params_onto_request``.""" + if not tier_litellm_params: + return + reject_api_key_secret_reference(tier_litellm_params) + accepted: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs) + surface: Final = _tier_ceiling_under_the_surface_name(accepted, responses_call=responses_call) + _drop_client_carriers_a_tier_pin_supersedes(request_kwargs, surface) + request_kwargs.update(surface) + + +def as_dict(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + return cast(Mapping[str, object], value) # cast-ok: a str-keyed config or request dict + + +def _adaptive_cost(params: Mapping[str, object], info: Mapping[str, object]) -> object: + """A model's input price for the adaptive router: litellm_params win over model_info.""" + cost: Final = params.get("input_cost_per_token") + return cost if cost is not None else info.get("input_cost_per_token") + + +def _tier_members(tiers: Mapping[str, object]) -> tuple[object, ...]: + """Every model the tier pools name, once, in order.""" + pools: Final = ( + (pool,) if isinstance(pool, str) or not isinstance(pool, Iterable) else pool for pool in tiers.values() + ) + return tuple(dict.fromkeys(chain.from_iterable(pools))) + + +def _tags_of(params: Mapping[str, object]) -> Sequence[object]: + tags: Final = params.get("tags") + return tags if isinstance(tags, list | tuple) else () + + +def _allowlist(params: Mapping[str, object]) -> frozenset[str]: + declared: Final = params.get("allowed_openai_params") + if not isinstance(declared, list | tuple | set | frozenset): + return frozenset() + return frozenset(entry for entry in declared if isinstance(entry, str)) + + +__all__ = ["PreRouter", "PreRoutingStrategies", "is_strategy_marker"] diff --git a/litellm/router_rust/steps/selection.py b/litellm/router_rust/steps/selection.py new file mode 100644 index 00000000000..183350bc05f --- /dev/null +++ b/litellm/router_rust/steps/selection.py @@ -0,0 +1,445 @@ +""" +Host-side strategy selectors for ``RustCore``. + +The Rust core decides which strategy picks and whose state it reads (``strategy::routing_context``: +a request override, then the routing group, then the router's own strategy) and asks the host to +run the stateful strategies it does not implement natively through a ``select`` step. The +strategies themselves are the standalone ``litellm/router_strategy`` loggers, which double as +``CustomLogger`` callbacks that record traffic in the host's routing cache. This module builds, +registers and retires them the way the Python router's ``routing_strategy_init``, +``_init_routing_groups`` and ``_get_override_strategy_selector`` do. It never reaches a Python +router core. +""" + +import threading +from collections.abc import Iterable, Mapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy +from litellm.router_strategy.least_busy import ( + CALL_IDENTITY, + SELECTION_RESERVATION_MAX_KEYS, + LeastBusyLoggingHandler, + call_id_of, +) +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler +from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler +from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 +from litellm.types.router import RoutingGroup, RoutingStrategy + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.caching.in_memory_cache import InMemoryCache + from litellm.router_python_host import PythonHost + +Selector: TypeAlias = ( + LeastBusyLoggingHandler + | LowestCostLoggingHandler + | LowestLatencyLoggingHandler + | LowestTPMLoggingHandler + | LowestTPMLoggingHandler_v2 +) + +# The legacy router attribute that holds the default group's selector, per strategy. +SELECTOR_ATTRIBUTES: Final = MappingProxyType( + { + "least-busy": "leastbusy_logger", + "usage-based-routing": "lowesttpm_logger", + "usage-based-routing-v2": "lowesttpm_logger_v2", + "latency-based-routing": "lowestlatency_logger", + "cost-based-routing": "lowestcost_logger", + } +) + + +def normalize_strategy(strategy: object) -> str | None: + """A strategy as the string Python compares against (``_normalize_strategy``).""" + if isinstance(strategy, RoutingStrategy): + return strategy.value + return strategy if isinstance(strategy, str) else None + + +def build_selector( + strategy: str | None, + args: Mapping[str, object], + cache: "DualCache", + reservations: "InMemoryCache | None" = None, +) -> Selector | None: + """``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every + strategy without a selector. Every least-busy selector of a router keeps its pick + reservations in the router's one ``reservations`` store.""" + routing_args: Final = dict(args) + match strategy: + case "least-busy": + return LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + case "usage-based-routing": + return LowestTPMLoggingHandler(router_cache=cache, routing_args=routing_args) + case "usage-based-routing-v2": + return LowestTPMLoggingHandler_v2(router_cache=cache, routing_args=routing_args) + case "latency-based-routing": + return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args) + case "cost-based-routing": + return LowestCostLoggingHandler(router_cache=cache, routing_args={}) + case _: + return None + + +def _callbacks(host: "PythonHost") -> list[object]: # mutable-ok: the process list itself + """``litellm.callbacks``, which the host serves like its other callback lists.""" + return host.callback_list("callbacks") # pyright: ignore[reportArgumentType] # not in the CallbackList literal + + +class StrategySelectors: + """The selectors of one router: the default group's, each routing group's, and the lazily + built per-request override ones.""" + + def __init__(self, host: "PythonHost", cache: "DualCache") -> None: + from litellm.caching.in_memory_cache import InMemoryCache + + self._host: Final = host + self._cache: Final = cache + # `_core_least_busy_reservations`: the picks least-busy reserved, shared by its selectors. + self.reservations: Final = InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS) + self._strategy: str | None = None + # The router's `routing_strategy`, which a direct `routing_strategy_init` leaves as it was. + self._configured: str | None = None + self._default: Selector | None = None + self._groups: dict[str, Selector] = {} # mutable-ok: replaced as routing groups change + self._group_strategies: frozenset[str] = frozenset() + self._overrides: dict[str, Selector | None] = {} # mutable-ok: filled lazily per strategy + self._lock: Final = threading.Lock() + + # -- building ------------------------------------------------------------------------------ + + def _register(self, selector: CustomLogger) -> None: + """``_register_router_selector``: least-busy also counts calls as they start.""" + host: Final = self._host + if isinstance(selector, LeastBusyLoggingHandler): + if isinstance(host.callback_list("input_callback"), list): # pyright: ignore[reportUnnecessaryIsInstance] # a process list a user may have reassigned + host.callbacks.add_litellm_input_callback(selector) + else: + host.set_callback_list("input_callback", [selector]) + if isinstance(_callbacks(host), list): # pyright: ignore[reportUnnecessaryIsInstance] # see above + host.callbacks.add_litellm_callback(selector) + + def _unregister(self, selectors: Sequence[object]) -> None: + live: Final = [selector for selector in selectors if selector is not None] + for selector in live: + if isinstance(selector, BaseRoutingStrategy): + selector.retire() + if live: + self._host.callbacks.drop_callbacks(live) + + def init_default(self, strategy: object, args: Mapping[str, object], *, configured: bool = True) -> None: + """``routing_strategy_init``: drop the old default and override selectors, build the new one. + ``configured``: ``strategy`` is the router's ``routing_strategy`` from now on.""" + self._unregister([self._default, *self._overrides.values()]) + self._overrides = {} + self._default = None + self._strategy = normalize_strategy(strategy) + if configured: + self._configured = self._strategy + self._default = build_selector(self._strategy, args, self._cache, self.reservations) + if self._default is not None: + self._register(self._default) + + def rebuild_default(self, args: Mapping[str, object]) -> None: + """``_apply_updated_routing_strategy_args``: relink the default selector to new args, keeping + the old one when they are invalid.""" + from pydantic import ValidationError + + from litellm._logging import verbose_router_logger + + if self._default is None or self._strategy != self._configured: + return + try: + rebuilt: Final = build_selector(self._strategy, args, self._cache, self.reservations) + except (TypeError, ValidationError): + verbose_router_logger.exception( + "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy + ) + return + self._unregister([self._default]) + self._default = rebuilt + if rebuilt is not None: + self._register(rebuilt) + + def init_groups(self, groups: Sequence[tuple[str, str, Mapping[str, object]]]) -> None: + """``_replace_routing_groups``: one selector per group, from (name, strategy, args).""" + self._unregister(list(self._groups.values())) + built: Final = { + name: selector + for name, strategy, args in groups + if ( + selector := build_selector( + None if strategy == "priority" else strategy, args, self._cache, self.reservations + ) + ) + is not None + } + for selector in built.values(): + self._register(selector) + self._groups = built + self._group_strategies = frozenset(strategy for _, strategy, _ in groups) + + def retire(self) -> None: + self._unregister([self._default, *self._groups.values(), *self._overrides.values()]) + self.clear() + + def clear(self) -> None: + """Forget every selector, after the caller released them from the host lists.""" + self._default = None + self._groups = {} + self._overrides = {} + + def callback_selectors(self) -> tuple[Selector, ...]: + """``_callback_selectors``: the default group's selector and each routing group's.""" + return tuple(selector for selector in (self._default, *self._groups.values()) if selector is not None) + + def override_selectors(self) -> tuple[Selector, ...]: + """The per-request override selectors built so far.""" + return tuple(selector for selector in self._overrides.values() if selector is not None) + + @property + def default(self) -> Selector | None: + """The default selector of the router's ``routing_strategy``, if it has one.""" + return self._default if self._strategy == self._configured else None + + def attribute(self, name: str) -> Selector | None: + """A legacy ``*_logger`` attribute: the default selector when it is that strategy's.""" + return self._default if SELECTOR_ATTRIBUTES.get(self._strategy or "") == name else None + + def latency_args(self) -> Mapping[str, object] | None: + """The default latency selector's full routing args, as ``get_settings`` reports them.""" + if not isinstance(self._default, LowestLatencyLoggingHandler): + return None + reported: Final[object] = self._default.routing_args.json() # pyright: ignore[reportUnknownMemberType] # RoutingArgs.json is untyped + return as_str_mapping(reported) + + # -- picking ------------------------------------------------------------------------------- + + def selector_for( + self, strategy: str, scope: str, group: str | None, request: Mapping[str, object] | None + ) -> Selector | None: + """The selector a Rust ``select`` step names (``_get_routing_context``).""" + if scope == "group": + return self._groups.get(group or "") + if scope == "override": + return self._override(strategy, request) + return self._default if strategy == self._strategy else None + + def _override(self, strategy: str, request: Mapping[str, object] | None) -> Selector | None: + """``_get_override_strategy_selector`` and ``_bind_override_selector_to_request``.""" + if strategy == self._configured: + return self.default + with self._lock: + if strategy not in self._overrides: + self._overrides[strategy] = build_selector(strategy, {}, self._cache, self.reservations) + selector: Final = self._overrides[strategy] + if selector is not None and request is not None and strategy not in self._registered_strategies(): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = request.get("litellm_logging_obj") + if isinstance(logging_obj, Logging): + logging_obj.add_dynamic_callback(selector) + return selector + + def _registered_strategies(self) -> frozenset[str]: + return frozenset({*((self._configured,) if self._configured else ()), *self._group_strategies}) + + def is_override(self, strategy: str, selector: object) -> bool: + """Whether ``selector`` is a per-request override one, whose pre-call check the router runs + itself because it is not in the callback lists.""" + return selector is not None and self._overrides.get(strategy) is selector + + +async def pick_async( + strategy: str, + selector: Selector, + model: str, + deployments: list[dict[str, object]], # mutable-ok: the selectors take a list of deployment dicts + messages: object, + input: object, + request_kwargs: dict[str, object] | None, # mutable-ok: the latency selector reads it as given + reserve: bool = False, +) -> object: + """``_select_deployment_async``; ``reserve``: least-busy holds the pick for the request's call.""" + match selector: + case LeastBusyLoggingHandler() if strategy == "least-busy": + return await selector.async_get_available_deployments( + model_group=model, + healthy_deployments=deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), + ) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + case LowestTPMLoggingHandler() if strategy == "usage-based-routing": + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in ( + "usage-based-routing-v2", + "cost-based-routing", + ): + return await selector.async_get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestLatencyLoggingHandler() if strategy == "latency-based-routing": + return await selector.async_get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + case _: + return None + + +def pick_sync( + strategy: str, + selector: Selector, + model: str, + deployments: list[dict[str, object]], # mutable-ok: see pick_async + messages: object, + input: object, + request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async + reserve: bool = False, +) -> object: + """``_select_deployment_sync``: cost-based routing has no sync pick.""" + match selector: + case LeastBusyLoggingHandler() if strategy == "least-busy": + return selector.get_available_deployments( + model_group=model, + healthy_deployments=deployments, + reserve=reserve, + call_id=call_id_of(CALL_IDENTITY.validate_python(request_kwargs or {})), + ) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in ( + "usage-based-routing", + "usage-based-routing-v2", + ): + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + ) + case LowestLatencyLoggingHandler() if strategy == "latency-based-routing": + return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts + model_group=model, + healthy_deployments=deployments, + messages=messages, # pyright: ignore[reportArgumentType] # same + input=input, # pyright: ignore[reportArgumentType] # same + request_kwargs=request_kwargs, + ) + case _: + return None + + +async def routing_pre_call_checks(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None: + """``async_routing_strategy_pre_call_checks``: every ``CustomLogger`` among the router's routing + callbacks (``_routing_callbacks``) checks the picked deployment before the call + (usage-based-routing-v2 counts its rpm here). A check's error fails the attempt; the core + records that failure like any other.""" + for callback in tuple(callbacks): + if isinstance(callback, CustomLogger): + await callback.async_pre_call_check(dict(deployment), None) # pyright: ignore[reportUnknownMemberType] # CustomLogger takes an untyped deployment dict + + +def routing_pre_call_checks_sync(callbacks: Iterable[object], deployment: Mapping[str, object]) -> None: + """``routing_strategy_pre_call_checks``: the sync check of each ``CustomLogger`` callback.""" + for callback in tuple(callbacks): + if isinstance(callback, CustomLogger): + callback.pre_call_check(dict(deployment)) # pyright: ignore[reportUnknownMemberType] # see above + + +def picked_id(picked: object) -> str | None: + """The id of the deployment a selector returned.""" + info: Final = as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {} + found: Final = info.get("id") + return str(found) if found is not None else None + + +__all__ = [ + "SELECTOR_ATTRIBUTES", + "Selector", + "StrategySelectors", + "build_selector", + "group_documents", + "group_settings", + "member_map", + "normalize_strategy", + "parse_groups", + "pick_async", + "pick_sync", + "picked_id", + "routing_pre_call_checks", + "routing_pre_call_checks_sync", +] + + +def parse_groups( + groups_input: object, model_names: frozenset[str], aliases: frozenset[str] +) -> tuple[RoutingGroup, ...]: + """``_init_routing_groups``' validation: ``parse_routing_groups``, then a priority group may not + share its name with a model group or alias (a plain group only warns: its name is shadowed).""" + from litellm._logging import verbose_router_logger + from litellm.litellm_core_utils.routing_groups import ( + parse_routing_groups, # pyright: ignore[reportUnknownVariableType] # takes untyped dicts + ) + + if not groups_input or not isinstance(groups_input, list | tuple): + return () + groups: Final = parse_routing_groups( # pyright: ignore[reportUnknownVariableType] # returns RoutingGroup models + list(groups_input), # pyright: ignore[reportUnknownArgumentType] # validated by parse_routing_groups + known_model_names=model_names, + ) + for group in groups: + if group.group_name in model_names or group.group_name in aliases: + if group.routing_strategy == "priority": + raise ValueError("Priority routing group names must not shadow a model or alias") + verbose_router_logger.warning( + "routing_groups: group_name '%s' is shadowed by an existing model_name or model_group_alias; " + "the group's strategy still applies to its members, but the name is not callable until renamed.", + group.group_name, + ) + return groups + + +def group_documents(groups: Sequence[RoutingGroup]) -> list[Mapping[str, object]]: # mutable-ok: a JSON list + """The routing groups as the Rust core reads them.""" + return [ + { + "group_name": group.group_name, + "models": list(group.models), + "routing_strategy": group.routing_strategy, + **({"model_priorities": dict(group.model_priorities)} if group.model_priorities is not None else {}), + } + for group in groups + ] + + +def group_settings(groups: Sequence[RoutingGroup]) -> list[object]: # mutable-ok: the settings snapshot's list + """``get_settings()['routing_groups']``.""" + return [ + group.model_dump(exclude={"model_priorities"}) if group.model_priorities is None else group.model_dump() + for group in groups + ] + + +def member_map(groups: Sequence[RoutingGroup]) -> Mapping[str, str]: + """``_model_to_group``: each member model group of a non-priority routing group, to the group + (a model listed by two groups maps to the last).""" + kept: Final = [group for group in groups if group.routing_strategy != "priority"] + return dict(chain.from_iterable(dict.fromkeys(group.models, group.group_name).items() for group in kept)) diff --git a/litellm/router_rust/surface/README.md b/litellm/router_rust/surface/README.md new file mode 100644 index 00000000000..ad9c0c62ed3 --- /dev/null +++ b/litellm/router_rust/surface/README.md @@ -0,0 +1,14 @@ +# surface + +The Python `Router` API, kept over router-core's state so Python callers see today's behavior. router-core holds the state and the rules; these modules are views and adapters. + +- `registry`, `model_list`, `indexes`, `members`: the deployment list, indexes and lookups; writes become changesets +- `legacy_members`: the old public members outside the entry points +- `model_info`: model and group info, answering the `model_info` step's lookups +- `replay`: re-registers pricing after a price data reload +- `settings`: `update_settings` rules and the settings view +- `health`, `health_routing`, `retries`: outcome callbacks, health-check routing and retry answers +- `fallbacks`: fallback config validation +- `alerting`: Slack alerting + +A Rust host needs none of this. diff --git a/litellm/router_rust/surface/__init__.py b/litellm/router_rust/surface/__init__.py new file mode 100644 index 00000000000..74704783ecb --- /dev/null +++ b/litellm/router_rust/surface/__init__.py @@ -0,0 +1 @@ +"""The Python ``Router`` API kept over router-core's state, for Python callers.""" diff --git a/litellm/router_rust/surface/alerting.py b/litellm/router_rust/surface/alerting.py new file mode 100644 index 00000000000..760eecb35f4 --- /dev/null +++ b/litellm/router_rust/surface/alerting.py @@ -0,0 +1,33 @@ +"""The Rust core's Slack alerting: ``alerting_config`` built into a ``SlackAlerting`` logger on the +host, as the Python core's ``_initialize_alerting`` builds it.""" + +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.router_python_host import PythonHost + from litellm.types.router import AlertingConfig + + +def initialize_alerting(host: "PythonHost", alerting_config: "AlertingConfig | None") -> "SlackAlerting | None": + """``_initialize_alerting``: a ``SlackAlerting`` logger with the config's threshold and webhook, + added to the host's callbacks, and its slow-response check to the host's success callbacks. + No config builds nothing.""" + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + + if alerting_config is None: + return None + logger: Final = SlackAlerting( + alerting_threshold=alerting_config.alerting_threshold, + alerting=["slack"], + default_webhook_url=alerting_config.webhook_url, + ) + host.callbacks.add_litellm_callback(logger) + host.callbacks.add_litellm_success_callback(logger.response_taking_too_long_callback) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # SlackAlerting's untyped callback + verbose_router_logger.info("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") + return logger + + +__all__ = ["initialize_alerting"] diff --git a/litellm/router_rust/surface/fallbacks.py b/litellm/router_rust/surface/fallbacks.py new file mode 100644 index 00000000000..48fbc972b43 --- /dev/null +++ b/litellm/router_rust/surface/fallbacks.py @@ -0,0 +1,129 @@ +""" +Fallback configuration checks ``RustCore`` runs on the host: ``validate_fallbacks``, with the +Python router's error texts. router-core walks the fallbacks; this module only refuses a list it +could not walk the way the Python router would, before the core is built. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +from litellm.router_utils.fallback_event_handlers import check_non_standard_fallback_format + + +def refuse_malformed_fallbacks(fallback_param: Sequence[object] | None) -> None: + """``Router.validate_fallbacks``: every entry is a one-key dict whose targets are a list (or a + client-side dict target).""" + if fallback_param is None: + return + for fallback_dict in fallback_param: + if not isinstance(fallback_dict, dict): + raise ValueError(f"Item '{fallback_dict}' is not a dictionary.") # noqa: TRY004 # the Python router's error class + if len(fallback_dict) != 1: # pyright: ignore[reportUnknownArgumentType] # only the size is read + raise ValueError( + f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys." # pyright: ignore[reportUnknownArgumentType] # see above + ) + ((model_group, targets),) = fallback_dict.items() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # an entry as configured + entry: list[object] = [fallback_dict] # mutable-ok: the one-entry list the check reads + if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=entry): + continue + raise TypeError( + f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. " + f"Use {{'{model_group}': ['{targets}']}}" + ) + + +def rejected_fallback_target(targets: Sequence[object]) -> Exception | None: + """The error ``run_async_fallback`` raises before its walk when a chain, from any list and of + any level, has a dict target a request may not set (a server-owned federation field, an + ``oidc/`` api key): every target of that walk is refused.""" + from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference + from litellm.types.router import reject_server_owned_wif_params + + for target in targets: + if isinstance(target, dict): + try: + reject_server_owned_wif_params(target) # pyright: ignore[reportUnknownArgumentType] # a chain as configured + reject_api_key_secret_reference(target) # pyright: ignore[reportUnknownArgumentType] # see above + except Exception as error: # noqa: BLE001 # the walk's own refusal, reported as its failure + return error + return None + + +# The `mock_testing_*` flags the router reads, and the mock failure each asks router-core for. +MOCK_FLAGS: Final = MappingProxyType( + { + "mock_testing_fallbacks": "fallbacks", + "mock_testing_context_fallbacks": "context_window", + "mock_testing_content_policy_fallbacks": "content_policy", + "mock_testing_rate_limit_error": "rate_limit", + } +) + + +def mock_failure(request: Mapping[str, object]) -> str | None: + """The mock failure a request asks for (``MockRouterTestingParams.from_kwargs``, then + ``_handle_mock_testing_rate_limit_error``): the first fallback flag that is ``True`` (a string + is read as a bool), else the rate limit flag when it is ``True`` itself.""" + from litellm.secret_managers.main import str_to_bool + + for flag in ("mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks"): + value = request.get(flag) + if (str_to_bool(value) if isinstance(value, str) else value) is True: + return MOCK_FLAGS[flag] + return "rate_limit" if request.get("mock_testing_rate_limit_error") is True else None + + +def mock_error( + mock: str, model: str, num_retries: object, request: Mapping[str, object], configured: Mapping[str, object] +) -> Exception: + """The exception the Python router raises for a mock failure; ``configured`` holds the + router's fallback lists, which the request's own replace.""" + import litellm.exceptions as litellm_exceptions + + def listed(name: str) -> object: + return request.get(name, configured.get(name)) + + if mock == "rate_limit": + return litellm_exceptions.RateLimitError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a rate limit error.", + num_retries=num_retries if isinstance(num_retries, int) else None, + ) + if mock == "context_window": + return litellm_exceptions.ContextWindowExceededError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. \ + Context_Window_Fallbacks={listed('context_window_fallbacks')}", + ) + if mock == "content_policy": + return litellm_exceptions.ContentPolicyViolationError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. \ + Context_Policy_Fallbacks={listed('content_policy_fallbacks')}", + ) + return litellm_exceptions.InternalServerError( + model=model, + llm_provider="", + message=f"This is a mock exception for model={model}, to trigger a fallback. Fallbacks={listed('fallbacks')}", + ) + + +class RustFallbacks: + """Fallback members served on the host.""" + + def validate_fallbacks(self, fallback_param: Sequence[object] | None) -> None: + refuse_malformed_fallbacks(fallback_param) + + +__all__ = [ + "MOCK_FLAGS", + "RustFallbacks", + "mock_error", + "mock_failure", + "refuse_malformed_fallbacks", + "rejected_fallback_target", +] diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py new file mode 100644 index 00000000000..5547471855a --- /dev/null +++ b/litellm/router_rust/surface/health.py @@ -0,0 +1,558 @@ +""" +The deployment outcome callbacks ``RustCore`` serves as members: ``deployment_callback_on_failure``, +``deployment_callback_on_success`` and ``sync_deployment_callback_on_success``. router-core counts +the outcome and decides the cooldown; this module reads litellm's logging kwargs, drops the +failures the Python router does not hold against the deployment, and drives the Rust call. The +stream wrappers report a stream's outcome through the same members when the stream ends. +""" + +from collections.abc import Mapping +from datetime import datetime +from typing import TYPE_CHECKING, Final, Literal + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +if TYPE_CHECKING: + from litellm.router_rust.steps.callbacks import OptionalChecks + from litellm.router_rust.surface.registry import HostRegistry + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _FailureReportSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + _SuccessReportSteps, # pyright: ignore[reportPrivateUsage] # see above + ) + from litellm.types.router import Deployment + + +def _cast_status(exception_status: object) -> int | None: + if isinstance(exception_status, int) and not isinstance(exception_status, bool): + return exception_status + if isinstance(exception_status, str): + try: + return int(exception_status) + except ValueError: + return 500 + return None + + +def failure_status(exception_status: object) -> int | None: + """``cast_exception_status_to_int``: a number as given, a numeric string read as one, any + other string 500. A number no status can hold reads as no status.""" + status: Final = _cast_status(exception_status) + return status if status is not None and 0 <= status <= 65535 else None + + +def _logged_deployment_id(kwargs: Mapping[str, object]) -> object: + """``litellm_params.model_info.id`` of a logged call.""" + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + return (as_str_mapping(litellm_params.get("model_info")) or {}).get("id") + + +def first_cooldown(*sources: Mapping[str, object] | None) -> int | float | None: + """``_first_present(*sources, key="cooldown_time")``, the number as it was written.""" + for source in sources: + value = source.get("cooldown_time") if source is not None else None + if isinstance(value, int | float) and not isinstance(value, bool): + return value + return None + + +def _configured_cooldown(model_info: object, litellm_params: Mapping[str, object]) -> float | None: + """``first_cooldown`` of a logged call, as the float router-core takes.""" + seconds: Final = first_cooldown(as_str_mapping(model_info), litellm_params) + return None if seconds is None else float(seconds) + + +def python_cooldown_time(seconds: float, *sources: float | None) -> float: + """``seconds`` as ``deployment_callback_on_failure`` hands them on. Router-core's duration is + always a float; Python's is the number that won its precedence, an int or a float as it was + written. ``sources`` run in that precedence, and only the type is read from them.""" + return next((source for source in sources if source is not None and source == seconds), seconds) + + +def _fallback_cooldown(deployment: "Deployment | None") -> float | None: + """``first_cooldown`` of the deployment a failed hop tried, as ``_core_get_model_info`` lists it.""" + if deployment is None: + return None + listed: Final = deployment.model_dump() + return first_cooldown(as_str_mapping(listed.get("model_info")), as_str_mapping(listed.get("litellm_params"))) + + +def _deployment_cooldown(deployment: Mapping[str, object]) -> float | None: + """``_configured_cooldown`` of a deployment as listed.""" + return _configured_cooldown(deployment.get("model_info"), as_str_mapping(deployment.get("litellm_params")) or {}) + + +def logged_call(op: str, kwargs: Mapping[str, object], started: datetime | None = None) -> Mapping[str, object]: + """The logging kwargs ``deployment_callback_on_failure`` would read for a call made with + ``kwargs``: the call's logging object's (a proxy request carries one), else what the request says. + Read before the failure logger stamps the call's ``end_time``, so it is stamped here.""" + ended: Final = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline's api_call_start_time + details: Final = as_str_mapping(getattr(kwargs.get("litellm_logging_obj"), "model_call_details", None)) + call_type: Final = op.rsplit(".", 1)[-1] + if details is not None: + return {"call_type": call_type, **details, "end_time": ended} + fields: Final = ("metadata", "litellm_metadata", "client_side_timeout", "timeout") + return { + "call_type": call_type, + "litellm_params": {name: kwargs.get(name) for name in fields}, + "start_time": started, + "end_time": ended, + } + + +def _authenticates_per_caller(litellm_params: object) -> bool: + """``github_copilot_per_user_credential_name`` in ``_should_run_cooldown_logic``. Per-user params + with no credential raise there, which ends Python's failure callback short of any cooldown.""" + from litellm.exceptions import BadRequestError + from litellm.llms.github_copilot.per_user_auth import github_copilot_per_user_credential_name + + try: + return github_copilot_per_user_credential_name(litellm_params) is not None + except BadRequestError: + return True + + +def failure_already_logged(kwargs: Mapping[str, object], event: Literal["async_failure", "sync_failure"]) -> bool: + """``should_run_logging``: a logging object logs one failure a request. The attempts of a request + that shares one (``litellm_logging_obj``) after the first logged failure run no failure callback; + without one, every litellm call logs on its own.""" + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + return isinstance(logging_obj, Logging) and logging_obj.model_call_details.get(f"has_logged_{event}") is True + + +def _success_target(kwargs: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None: + """The deployment id and metadata a success callback counts under, or None.""" + params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + metadata: Final = as_str_mapping(params.get("metadata")) + if metadata is None: + return None + model_info: Final = as_str_mapping(params.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + if metadata.get("model_group") is None or deployment_id is None: + return None + return str(deployment_id), metadata + + +class RustHealthCallbacks: + """Outcome callbacks read into the Rust core's health state.""" + + _handle: "RouterCoreHandle" + _cooldown_given: object + + def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]: + raise NotImplementedError + + async def _drive_cache_steps( + self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]" + ) -> Mapping[str, object]: + raise NotImplementedError + + def _sync_allowed_fails(self) -> None: + raise NotImplementedError + + def _core_get_allowed_fails_from_policy(self, exception: Exception) -> int | None: + """The router ``allowed_fails_policy``'s allowance for this exception, decided by router-core.""" + from litellm.router_rust.core import attempt_failure + + return self._handle.allowed_fails_for(dict(attempt_failure(exception))) + + def _deployment_of(self, deployment_id: object) -> "Deployment | None": + # RustCore sets `_hosted` as an attribute in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + return hosted._core_get_deployment(model_id=deployment_id) if isinstance(deployment_id, str) else None # pyright: ignore[reportPrivateUsage] # the host registry's id lookup + + def deployment_tracks_usage(self, deployment_id: object, model_group: object) -> bool: + """``_cached_deployment_tracks_usage``. A deployment whose model is not mapped raises there, + which ends Python's usage write, so it reads as not tracked.""" + # RustCore sets `_hosted` as an attribute in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + if not isinstance(deployment_id, str) or not isinstance(model_group, str): + return False + try: + return hosted.cached_deployment_tracks_usage(deployment_id, model_group) + except Exception: # noqa: BLE001 # as `deployment_callback_on_success` swallows it + return False + + def cooldown_event(self, deployment_id: str, exception_status: object, cooldown_time: float | None) -> None: + """The end of ``set_cooldown_deployments``, once the cooldown is written: the cooldown event + Prometheus reads, as a task of the running loop. Outside one this raises ``RuntimeError``.""" + import asyncio + + from litellm.router_api import RoutingContext + from litellm.router_utils import cooldown_handlers + + _ = asyncio.get_running_loop() + _ = asyncio.create_task( # fire and forget, as set_cooldown_deployments schedules it + cooldown_handlers.router_cooldown_event_callback( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + deployment_id=deployment_id, + exception_status=exception_status, # pyright: ignore[reportArgumentType] # the status as raised + cooldown_time=cooldown_time, + ) + ) + + def cooldown_event_on_own_loop( + self, deployment_id: str, exception_status: object, cooldown_time: float | None + ) -> None: + """``cooldown_event`` for a sync walk ``function_with_fallbacks`` would run on its private + loop: ``set_cooldown_deployments`` finds that loop however the caller runs, and the event's + task keeps what it raises to itself.""" + import asyncio + + from litellm.litellm_core_utils import asyncify + + async def fire() -> None: + self.cooldown_event(deployment_id, exception_status, cooldown_time) + await asyncio.sleep(0) + + asyncify.run_async_function(fire) # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + + def python_cooldown(self, seconds: float, configured: float | None = None, error: object = None) -> float: + """The ``time_to_cooldown`` Python hands ``set_cooldown_deployments``, for router-core's + ``seconds``. A failure callback's precedence is the ``configured`` number, then the ``retry-after`` + header of ``error``, then the router's own; a path passing neither hands on the router's own.""" + from litellm.router_rust.core import retry_after_seconds + + retry_after: Final = retry_after_seconds(error) if isinstance(error, BaseException) else None + default: Final = self._cooldown_given + return python_cooldown_time( + seconds, + configured, + None if retry_after is None else int(retry_after), + default if isinstance(default, int | float) else None, + ) + + def cooled_down( + self, + done: Mapping[str, object], + deployment_id: object, + exception_status: object, + *, + own_loop: bool = False, + configured: float | None = None, + error: object = None, + ) -> bool: + """Whether a failure report cooled the deployment down, with its cooldown event, which + carries ``python_cooldown`` of ``configured`` and ``error``.""" + cooled: Final = bool(done.get("cooled")) + if cooled and isinstance(deployment_id, str): + seconds: Final = done.get("cooldown_time") + event: Final = self.cooldown_event_on_own_loop if own_loop else self.cooldown_event + event( + deployment_id, + exception_status, + self.python_cooldown(seconds, configured, error) if isinstance(seconds, float) else None, + ) + return cooled + + def per_user_deployment(self, deployment_id: object) -> bool: + """The failed deployment authenticates per caller (``_should_run_cooldown_logic``), which is + all ``set_cooldown_deployments`` reads when it is called without the request's kwargs.""" + deployment: Final = self._deployment_of(deployment_id) + return deployment is not None and _authenticates_per_caller(deployment.litellm_params) + + def failure_exits( + self, exception: object, logged: Mapping[str, object], deployment_id: object + ) -> Mapping[str, bool]: + """What the Python router does not hold against the deployment, for every failure it logs. + ``outside_deployment``: ``deployment_callback_on_failure``'s early exits, counting nothing. + ``caller_scoped``: ``_should_run_cooldown_logic``'s per-caller exits, counted but not cooled; + its exits on the exception's class are router-core's (``FailureKind``).""" + from litellm.router_utils.cooldown_handlers import ( + _is_per_user_provider_request, # pyright: ignore[reportPrivateUsage] # the cooldown check's own rule + is_background_response_poll_not_found, + is_caller_scoped_auth_failure, + is_caller_timeout_408, + is_failure_outside_selected_deployment, + is_video_lookup_not_found, + ) + + status: Final[object] = getattr(exception, "status_code", "") + litellm_params: Final = as_str_mapping(logged.get("litellm_params")) or {} + outside: Final = ( + is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None) + or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params)) + or is_video_lookup_not_found(exception, logged) + or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised + or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above + ) + return { + "outside_deployment": outside, + "caller_scoped": _is_per_user_provider_request(logged) or self.per_user_deployment(deployment_id), + } + + def trigger_fallback_cooldown( + self, kwargs: Mapping[str, object], exception: object, generic: bool, *, own_loop: bool = False + ) -> None: + """The failed fallback hop's ``_trigger_cooldown_for_failed_deployment``, which runs only when + the request's logging object blocked the hop's failure callbacks. ``generic``: the call's kwargs + carry ``original_generic_function``. ``own_loop``: see ``cooldown_event_on_own_loop``.""" + from litellm._logging import verbose_router_logger + from litellm.router_utils.fallback_event_handlers import ( + _failure_callbacks_blocked_logging_object, # pyright: ignore[reportPrivateUsage] # the fallback path's own rule + ) + + blocked: Final = _failure_callbacks_blocked_logging_object(kwargs) + if blocked is None or not isinstance(exception, Exception): + return + try: + call: Final = self._fallback_failure_call(kwargs, exception, blocked.model_call_details, generic) + if call is not None: + _ = self.cooled_down( + self._drive_sync_steps(call), + getattr(exception, "failed_deployment_id", None), + getattr(exception, "status_code", ""), + own_loop=own_loop, + configured=_fallback_cooldown( + self._deployment_of(getattr(exception, "failed_deployment_id", None)) + ), + error=exception, + ) + except Exception as e: # noqa: BLE001 # best effort, as the Python helper: it never breaks the fallback response + verbose_router_logger.debug("Error triggering cooldown for fallback deployment: %s", e) + + def _fallback_failure_call( + self, + kwargs: Mapping[str, object], + exception: Exception, + model_call_details: Mapping[str, object], + generic: bool, + ) -> "RouterCall[_FailureReportSteps] | None": + """The Rust call recording a failed hop, or None past the exits of + ``_trigger_cooldown_for_failed_deployment``: another list than ``failure_exits``, checked per + failed hop. The selection tag is not on it; a stream stall and a generic call's 404 are.""" + from litellm.router_rust.core import attempt_failure + from litellm.router_utils.cooldown_handlers import ( + _is_per_user_provider_request, # pyright: ignore[reportPrivateUsage] # the cooldown check's own rule + cast_exception_status_to_int, + is_advisor_orchestration_failure, + is_caller_scoped_auth_failure, + is_caller_timeout_408, + is_stream_stall, + ) + from litellm.router_utils.fallback_event_handlers import ( + _REQUEST_SCOPED_STATUS_CODES, # pyright: ignore[reportPrivateUsage] # the fallback path's own rule + ) + + self._sync_allowed_fails() + exception_status: Final[object] = getattr(exception, "status_code", "") + if ( + is_advisor_orchestration_failure(exception) + or is_stream_stall(exception) + or (generic and cast_exception_status_to_int(exception_status) in _REQUEST_SCOPED_STATUS_CODES) # pyright: ignore[reportArgumentType] # the status as raised + or is_caller_timeout_408(model_call_details, exception_status, ended=datetime.now()) # pyright: ignore[reportArgumentType] # noqa: DTZ005 # see logged_call + ): + return None + deployment_id: Final[object] = getattr(exception, "failed_deployment_id", None) + if not isinstance(deployment_id, str): + return None + deployment: Final = self._deployment_of(deployment_id) + if is_caller_scoped_auth_failure(deployment, exception_status): # pyright: ignore[reportArgumentType] # see above + return None + return self._handle.report_failure( + deployment_id, + { + **attempt_failure(exception), + "status": failure_status(exception_status), + "caller_scoped": _is_per_user_provider_request(kwargs) or self.per_user_deployment(deployment_id), + }, + deployment_cooldown=True, + count_only=exception_status == "", + cooldown_time=None if deployment is None else _deployment_cooldown(deployment.model_dump()), + ) + + def deployment_callback_on_failure( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> bool: + """Count the failure and cool the deployment down when the rules say so.""" + call: Final = self._failure_call(kwargs) + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + return call is not None and self.cooled_down( + self._drive_sync_steps(call), + _logged_deployment_id(kwargs), + getattr(kwargs.get("exception"), "status_code", ""), + configured=first_cooldown(as_str_mapping(litellm_params.get("model_info")), litellm_params), + error=kwargs.get("exception"), + ) + + async def report_stream_failure( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> None: + """A stream's failure after its attempt returned it, as an async call's failure callbacks + record it: counted, cooled down by the rules, and one more request this minute + (``async_deployment_callback_on_failure``). No cooldown event: the stream logs its failure + through ``dispatch_failure_handlers``, whose sync callbacks run in an executor thread, where + ``set_cooldown_deployments`` has no loop to schedule one on.""" + call: Final = self._failure_call(kwargs, counts_request=True, callbacks=callbacks) + if call is not None: + _ = await self._drive_cache_steps(call) + + def report_stream_failure_sync( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> None: + """``report_stream_failure`` for a sync stream: a sync call's failure callback counts no request. + No cooldown event: the stream logs its failure in a ``threading.Thread`` of its own, which + has no running loop whatever the consumer's thread has.""" + call: Final = self._failure_call(kwargs, callbacks=callbacks) + if call is not None: + _ = self._drive_sync_steps(call) + + def _failure_call( + self, + kwargs: Mapping[str, object], + counts_request: bool = False, + callbacks: Mapping[str, bool] | None = None, + ) -> "RouterCall[_FailureReportSteps] | None": + """The Rust call recording a logged failure, or None when it records nothing: one the Python + router does not hold against the deployment is still a request of the minute + (``async_deployment_callback_on_failure`` makes none of the failure callback's exits).""" + from litellm.litellm_core_utils import core_helpers + from litellm.router_rust.core import attempt_failure + + self._sync_allowed_fails() + exception: Final = kwargs.get("exception") + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + model_info: Final = litellm_params.get("model_info", {}) + info: Final = as_str_mapping(model_info) + deployment_id: Final = info.get("id") if info is not None else None + if deployment_id is None: + return None + exits: Final = self.failure_exits(exception, kwargs, deployment_id) + if exits["outside_deployment"] and not counts_request: + return None + exception_status: Final[object] = getattr(exception, "status_code", "") + described: Final[Mapping[str, object]] = ( + attempt_failure(exception) if isinstance(exception, BaseException) else {"ok": False, "kind": "APIError"} + ) + failure: Final = {**described, **exits, "status": failure_status(exception_status)} + metadata: Final[object] = core_helpers.get_litellm_metadata_from_kwargs(dict(kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # an untyped helper's dict + group: Final = (as_str_mapping(metadata) or {}).get("model_group") # pyright: ignore[reportUnknownArgumentType] # read as a mapping + usage_model: Final = (as_str_mapping(metadata) or {}).get("deployment") # pyright: ignore[reportUnknownArgumentType] # see above + return self._handle.report_failure( + str(deployment_id), + failure, + deployment_cooldown=True, + count_only=exception_status == "", + requested_group=group if isinstance(group, str) else None, + cooldown_time=_configured_cooldown(model_info, litellm_params), + counts_request=counts_request, + usage_model=usage_model if isinstance(usage_model, str) else None, + callbacks=callbacks, + ) + + def sync_deployment_callback_on_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> str | None: + """Count a success this minute; returns the counter's key.""" + return self.report_stream_success_sync(kwargs) + + def report_stream_success_sync( + self, kwargs: Mapping[str, object], callbacks: Mapping[str, bool] | None = None + ) -> str | None: + """``sync_deployment_callback_on_success``, for a sync stream that ended.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + + if is_batch_retrieve_call_type(kwargs.get("call_type")): + return None + target: Final = _success_target(kwargs) + if target is None: + return None + key: Final = self._drive_sync_steps(self._handle.count_success(target[0], callbacks)).get("key") + return key if isinstance(key, str) else None + + async def repin_after_success(self, kwargs: Mapping[str, object], completion_response: object) -> None: + """The session re-pin ``deployment_callback_on_success`` runs for a success logged against a + deployment the router serves (``_repin_session_after_success``). The Rust core runs it from the + attempt itself, or from the stream wrapper when a stream ends.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + + call_type: Final = kwargs.get("call_type") + if call_type == "_arealtime" or is_batch_retrieve_call_type(call_type): + return + target: Final = _success_target(kwargs) + # RustCore sets these as attributes in its constructor. + hosted: Final[HostRegistry] = self.__dict__["_hosted"] # pyright: ignore[reportAny] # RustCore's own attribute + if target is None or hosted._core_get_deployment(model_id=target[0]) is None: # pyright: ignore[reportPrivateUsage] # the host registry's id lookup + return + checks: Final[OptionalChecks] = self.__dict__["_checks"] # pyright: ignore[reportAny] # see above + cooldowns: Final[CooldownCache] = self.__dict__["_cooldowns"] # pyright: ignore[reportAny] # see above + await checks.repin_session_after_success( + as_str_mapping(kwargs.get("litellm_params")) or {}, completion_response, cooldowns + ) + + async def deployment_callback_on_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> None: + """Count a success and the call's tokens and request against a tracked deployment.""" + await self.report_stream_success(kwargs, completion_response) + + async def report_stream_success( + self, + kwargs: Mapping[str, object], + completion_response: object, + callbacks: Mapping[str, bool] | None = None, + ) -> None: + """``deployment_callback_on_success``, for a stream that ended. The re-pin is the host's half + of that callback, so it reads ``callbacks`` here.""" + from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens + + call_type: Final = kwargs.get("call_type") + if call_type == "_arealtime" or is_batch_retrieve_call_type(call_type): + return + logged: Final = as_str_mapping(kwargs.get("standard_logging_object")) + if logged is None: + return + target: Final = _success_target(kwargs) + if target is None: + return + if (callbacks or {}).get("success", True): + await self.repin_after_success(kwargs, completion_response) + tokens: Final[object] = logged.get("total_tokens", 0) + total: Final = float(tokens) if isinstance(tokens, int | float) else 0.0 + counted: Final = get_counted_usage_tokens(as_str_mapping(kwargs.get("litellm_params")) or {}) + usage_model: Final = target[1].get("deployment") + await self._drive_cache_steps( + self._handle.report_success( + target[0], + total if counted is None else max(0.0, total - counted), + 1.0 if counted is None else 0.0, + usage_model if isinstance(usage_model, str) else None, + self.deployment_tracks_usage(target[0], target[1].get("model_group")), + callbacks, + ) + ) + + async def count_realtime_session_success(self, deployment_id: str) -> None: + """``increment_deployment_successes_for_current_minute`` for a realtime session that ended + cleanly: a success alone, as ``deployment_callback_on_success`` skips ``_arealtime``. A + counter that cannot be written does not fail the served session.""" + from litellm._logging import verbose_router_logger + + try: + await self._drive_cache_steps(self._handle.count_success(deployment_id)) + except Exception as e: # noqa: BLE001 # as the Rust call loop treats a success it could not record + verbose_router_logger.debug("count_realtime_session_success: not recorded. Got - %s", e) + + +__all__ = [ + "RustHealthCallbacks", + "failure_status", + "logged_call", +] diff --git a/litellm/router_rust/surface/health_routing.py b/litellm/router_rust/surface/health_routing.py new file mode 100644 index 00000000000..46542bf9af2 --- /dev/null +++ b/litellm/router_rust/surface/health_routing.py @@ -0,0 +1,160 @@ +""" +Health-check routing, allowed-fails policies and the failover flags on the host side of the Rust +core: how the constructor resolves them, the static document router-core reads them from, and the +``health()`` members that read the registry's live model list or the background health checks' +results. router-core makes every routing decision (the health-check filter, the cooldown rules, +weighted failover); this module only reads Python values into its documents and back. + +The rules are the Python router core's, ported (never imported). +""" + +import dataclasses +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.types.router import AllowedFailsPolicy + +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + from litellm.router_config import RouterConfig + from litellm.types.router import Deployment, DeploymentTypedDict + +HEALTH_STATE_KEY: Final = DeploymentHealthCache.CACHE_KEY +# The failover flags the Python core keeps as plain router attributes, with their defaults. +FAILOVER_FLAGS: Final = ("enable_weighted_failover", "fallback_on_unsupported_params") + + +def resolved_allowed_fails_policy(value: object) -> AllowedFailsPolicy | None: + """The router's ``allowed_fails_policy`` as the constructor keeps it: a dict is read into an + ``AllowedFailsPolicy`` (a bad field raises the model's own error), a policy is kept, anything + else is no policy.""" + if isinstance(value, AllowedFailsPolicy): + return value + if isinstance(value, dict): + return AllowedFailsPolicy.model_validate(value) + return None + + +def _staleness(threshold: object) -> float: + """``health_check_staleness_threshold or DEFAULT_HEALTH_CHECK_INTERVAL * multiplier``.""" + from litellm.constants import DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER + + given: Final = threshold if isinstance(threshold, int | float) and not isinstance(threshold, bool) else None + return float(given or (DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER)) + + +@dataclasses.dataclass(frozen=True, slots=True) +class HealthRouting: + """The health-check routing settings, as the constructor resolves them.""" + + enable_health_check_routing: bool = False + health_check_staleness: float = 600.0 + health_check_ignore_transient_errors: bool = False + background_health_check_model_groups: frozenset[str] | None = None + + @classmethod + def of(cls, config: "RouterConfig") -> "HealthRouting": + groups: Final = config.get("background_health_check_model_groups") + return cls( + enable_health_check_routing=bool(config.get("enable_health_check_routing")), + health_check_staleness=_staleness(config.get("health_check_staleness_threshold")), + health_check_ignore_transient_errors=bool(config.get("health_check_ignore_transient_errors")), + background_health_check_model_groups=frozenset(groups) if groups is not None else None, # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # any iterable of names, as the Python core takes it + ) + + def static_document(self) -> Mapping[str, object]: + groups: Final = self.background_health_check_model_groups + return { + "enable_health_check_routing": True if self.enable_health_check_routing else None, + "health_check_staleness_threshold": self.health_check_staleness, + # Only a string can name a model group, so anything else in scope matches nothing. + "background_health_check_model_groups": sorted(name for name in groups if isinstance(name, str)) # pyright: ignore[reportUnnecessaryIsInstance] # `of` keeps any iterable the constructor was given + if groups is not None + else None, + } + + def health_checks(self, cache: "DualCache") -> DeploymentHealthCache: + return DeploymentHealthCache(cache=cache, staleness_threshold=self.health_check_staleness) + + +def failover_document(plain: Mapping[str, object]) -> Mapping[str, object]: + """The failover flags router-core reads, from the router's plain attributes (read for + truthiness, as the Python core's checks read them).""" + return { + "enable_weighted_failover": True if plain.get("enable_weighted_failover") else None, + "fallback_on_unsupported_params": False if not plain.get("fallback_on_unsupported_params", True) else None, + } + + +def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]: + return as_str_mapping(deployment.get("model_info")) or {} + + +def is_blocked(deployment: "Deployment") -> bool: + """``_core_is_deployment_blocked``.""" + model_info: Final[object | None] = getattr(deployment, "model_info", None) + if model_info is None: + return False + return getattr(model_info, "blocked", None) is True + + +def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + """``_core_are_all_deployments_blocked``.""" + return len(deployments) > 0 and all( + _model_info(deployment).get("blocked") is True # pyright: ignore[reportArgumentType] # a deployment dict + for deployment in deployments + ) + + +def fully_blocked_model_names( + model_list: Sequence[Mapping[str, object]], +) -> set[str]: # mutable-ok: the core's type, a fresh set + """``_core_get_fully_blocked_model_names``: groups whose every deployment is blocked.""" + blocked_by_name: Final[dict[str, bool]] = {} # mutable-ok: built here + for deployment in model_list: + name = deployment.get("model_name") or "" + if not isinstance(name, str) or not name: + continue + blocked = _model_info(deployment).get("blocked") is True + blocked_by_name[name] = blocked_by_name.get(name, True) and blocked + return {name for name, fully in blocked_by_name.items() if fully} + + +async def fully_unhealthy_model_names( + model_list: Sequence[Mapping[str, object]], health_checks: DeploymentHealthCache, policy: object +) -> set[str]: # mutable-ok: the core's type, a fresh set + """``_core_async_get_fully_unhealthy_model_names``: groups (and team public names) whose + every deployment failed its health check; nothing under an allowed_fails_policy.""" + if policy is not None: + return set() + unhealthy_ids: Final = await health_checks.async_get_unhealthy_deployment_ids() + if not unhealthy_ids: + return set() + unhealthy_by_name: Final[dict[str, bool]] = {} # mutable-ok: built here + for deployment in model_list: + model_info = _model_info(deployment) + names = [deployment.get("model_name") or ""] + team_public_model_name = model_info.get("team_public_model_name") + if team_public_model_name: + names.append(team_public_model_name) + unhealthy = model_info.get("id") in unhealthy_ids + for name in names: + if not isinstance(name, str) or not name: + continue + unhealthy_by_name[name] = unhealthy_by_name.get(name, True) and unhealthy + return {name for name, fully in unhealthy_by_name.items() if fully} + + +__all__ = [ + "FAILOVER_FLAGS", + "HEALTH_STATE_KEY", + "HealthRouting", + "all_blocked", + "failover_document", + "fully_blocked_model_names", + "fully_unhealthy_model_names", + "is_blocked", + "resolved_allowed_fails_policy", +] diff --git a/litellm/router_rust/surface/indexes.py b/litellm/router_rust/surface/indexes.py new file mode 100644 index 00000000000..f5aca3e1bad --- /dev/null +++ b/litellm/router_rust/surface/indexes.py @@ -0,0 +1,514 @@ +# pyright: reportIncompatibleMethodOverride=false +# (each view overrides its builtin's mutators with the same behaviour, as a changeset) +""" +``HostedIndexes``: the registry indexes the Rust core hands out, read back from router-core. + +router-core keeps the indexes the Python router keeps by history (``host_index.rs``): the id, +group and team position maps, the group names a ``set_model_list`` was given, the deployment +names that only ever grow, the team public names, the provider default ids and the wildcard +pattern routers. ``registry().index()``, ``names(kind=...)``, ``patterns()`` and the deprecated +attributes return the views here: a ``dict``, ``set`` or ``list`` (so ``isinstance`` and ``json`` +keep working), one object per router, whose contents are router-core's, read again when an entry +point hands a view out (or before a write to it) once the registry moved, not on every registry +write. A write to any of them, a nested one included (``index("model_name")["gpt"].append(3)``), +is a ``host_index`` changeset that replaces that index in router-core, and the view reads it back. +The lookups over these indexes run in router-core (``lookup.rs``), not over the views. +""" + +import copy +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import partial +from typing import ( + TYPE_CHECKING, + Final, + Literal, + SupportsIndex, + TypeAlias, + cast, # noqa: TID251 # any value a caller wrote into a view +) + +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter + +if TYPE_CHECKING: + from litellm.rust_bridge._native import RouterCoreHandle + +IndexField: TypeAlias = Literal[ + "id_map", + "model_name", + "team_model", + "model_names", + "deployment_names", + "team_public_model_names", + "provider_default_ids", + "patterns", + "team_patterns", +] +_Commit: TypeAlias = Callable[[object], None] +# Deployment copies a pattern router routes to, fresh for the caller. +Routed: TypeAlias = list[dict[str, object]] +_Pairs: TypeAlias = list[tuple[str, list[object]]] + + +def _stay() -> None: + """A view with no contents of its own to read again before a write.""" + + +def _as_set(value: object) -> set[object]: # mutable-ok: set's own in-place operand + return cast(set[object], value) # cast-ok: an in-place set operand, checked by set itself + + +def _as_mapping(value: object) -> Mapping[object, object]: + if not isinstance(value, Mapping): + return {} + return cast(Mapping[object, object], value) # cast-ok: any mapping a caller wrote + + +def _json(value: object, depth: int = 0) -> object: + """A view's contents as router-core takes them: plain JSON, a set as a list. A caller wrote them, + so a cycle is possible: anything nested past ``DEFAULT_MAX_RECURSE_DEPTH`` is refused.""" + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError(f"a router index is nested deeper than {DEFAULT_MAX_RECURSE_DEPTH} levels") + mapping: Final = _as_mapping(value) + if mapping or isinstance(value, Mapping): + return {str(key): _json(item, depth + 1) for key, item in mapping.items()} + if isinstance(value, list | tuple | set | frozenset): + items: Final = cast(Iterable[object], value) # cast-ok: any collection a caller wrote + return [_json(item, depth + 1) for item in items] + return value + + +class _Positions(list[object]): + """One index's position list (or a pattern's deployments): a write commits the whole list.""" + + def __init__(self, items: Iterable[object], commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__(items) + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Iterable[object]) -> None: + """Show router-core's contents, without a write.""" + super().__setitem__(slice(None), list(items)) + + def _write(self, change: Callable[[list[object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = list(self) + result: Final = change(changed) + self._commit(changed) + return result + + def append(self, value: object) -> None: + _ = self._write(lambda items: items.append(value)) + + def extend(self, values: Iterable[object]) -> None: + added: Final = tuple(values) + _ = self._write(lambda items: items.extend(added)) + + def insert(self, index: SupportsIndex, value: object) -> None: + _ = self._write(lambda items: items.insert(index, value)) + + def remove(self, value: object) -> None: + _ = self._write(lambda items: items.remove(value)) + + def pop(self, index: SupportsIndex = -1) -> object: + return self._write(lambda items: items.pop(index)) + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def sort(self, *args: object, **kwargs: object) -> None: # kwargs-ok: list.sort's own arguments + _ = self._write(lambda items: items.sort(*args, **kwargs)) # pyright: ignore[reportCallIssue, reportArgumentType] # list.sort's own arguments + + def reverse(self) -> None: + _ = self._write(lambda items: items.reverse()) # pyright: ignore[reportUnknownLambdaType] # list.reverse returns None + + def __setitem__(self, index: object, value: object) -> None: + _ = self._write(lambda items: items.__setitem__(index, value)) # pyright: ignore[reportCallIssue, reportArgumentType, reportUnknownLambdaType] # list's own item and slice assignment + + def __delitem__(self, index: object) -> None: + _ = self._write(lambda items: items.__delitem__(index)) # pyright: ignore[reportArgumentType] # list's own item and slice deletion + + def __iadd__(self, values: Iterable[object]) -> "_Positions": + self.extend(values) + return self + + def __imul__(self, count: SupportsIndex) -> "_Positions": + _ = self._write(lambda items: items.__imul__(count)) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (list, (list(self),)) + + +class _IndexDict(dict[object, object]): + """A key-to-positions index: a write commits the whole index.""" + + def __init__(self, commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__() + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Mapping[object, object]) -> None: + """Show router-core's contents, without a write.""" + super().clear() + super().update(items) + + def _write(self, change: Callable[[dict[object, object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = dict(self) + result: Final = change(changed) + self._commit(changed) + return result + + def __setitem__(self, key: object, value: object) -> None: + _ = self._write(lambda items: items.__setitem__(key, value)) + + def __delitem__(self, key: object) -> None: + _ = self._write(lambda items: items.__delitem__(key)) + + def pop(self, key: object, *default: object) -> object: + return self._write(lambda items: items.pop(key, *default)) + + def popitem(self) -> tuple[object, object]: + return cast(tuple[object, object], self._write(lambda items: items.popitem())) # cast-ok: dict.popitem's pair + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def update(self, *args: object, **kwargs: object) -> None: # kwargs-ok: dict.update's own arguments + _ = self._write(lambda items: items.update(*args, **kwargs)) # pyright: ignore[reportCallIssue, reportArgumentType] # dict.update's own arguments + + def setdefault(self, key: object, default: object = None) -> object: + if key in self: + return self[key] + self[key] = default + return self[key] + + def __ior__(self, other: object) -> "_IndexDict": + self.update(other) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (dict, (dict(self),)) + + +class _NameSet(set[object]): + """``model_names``: a write commits the whole set.""" + + def __init__(self, commit: _Commit, fresh: Callable[[], None] = _stay) -> None: + super().__init__() + self._commit: Final = commit + self._fresh: Final = fresh + + def _load(self, items: Iterable[object]) -> None: + """Show router-core's contents, without a write.""" + super().clear() + super().update(items) + + def _write(self, change: Callable[[set[object]], object]) -> object: # mutable-ok: the copy it changes + self._fresh() # a view held across a write elsewhere changes router-core's contents, not stale ones + changed: Final = set(self) + result: Final = change(changed) + self._commit(changed) + return result + + def add(self, value: object) -> None: + _ = self._write(lambda items: items.add(value)) + + def discard(self, value: object) -> None: + _ = self._write(lambda items: items.discard(value)) + + def remove(self, value: object) -> None: + _ = self._write(lambda items: items.remove(value)) + + def pop(self) -> object: + return self._write(lambda items: items.pop()) + + def clear(self) -> None: + _ = self._write(lambda items: items.clear()) + + def update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.update(*others)) + + def difference_update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.difference_update(*others)) + + def intersection_update(self, *others: Iterable[object]) -> None: + _ = self._write(lambda items: items.intersection_update(*others)) + + def symmetric_difference_update(self, other: Iterable[object]) -> None: + _ = self._write(lambda items: items.symmetric_difference_update(other)) + + def __ior__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__ior__(_as_set(other))) + return self + + def __iand__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__iand__(_as_set(other))) + return self + + def __isub__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__isub__(_as_set(other))) + return self + + def __ixor__(self, other: object) -> "_NameSet": + _ = self._write(lambda items: items.__ixor__(_as_set(other))) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (set, (set(self),)) + + +class HostedPatternRouter(PatternMatchRouter): + """A wildcard pattern router whose patterns are router-core's (``pattern_router``, or one team's + router): routing runs in router-core, and ``add_pattern`` / ``remove_deployment`` / a write to + ``patterns`` are changesets.""" + + def __init__(self, owner: "HostedIndexes", team: str | None) -> None: + super().__init__() # its empty `patterns` write is ignored: router-core holds them + self._owner: Final = owner + self._team: Final = team + self._view: Final = _IndexDict(self._commit, owner.refresh) + + @property + def patterns(self) -> dict[str, list[object]]: # pyright: ignore[reportIncompatibleVariableOverride] # the base class's attribute, read from router-core # mutable-ok: the live patterns, writes are changesets + self._owner.refresh() + return cast(dict[str, list[object]], self._view) # cast-ok: regex keys, deployment lists + + @patterns.setter + def patterns(self, value: Mapping[str, object]) -> None: # pyright: ignore[reportIncompatibleVariableOverride] # see the getter + if not hasattr(self, "_owner"): + return # the base constructor's empty dict: router-core holds this router's patterns + self._commit(value) + + def _show(self, pairs: Iterable[tuple[str, Sequence[object]]]) -> None: + shown: Final = { + regex: _Positions(deployments, partial(self._write_pattern, regex)) for regex, deployments in pairs + } + self._view._load(cast(Mapping[object, object], shown)) # pyright: ignore[reportPrivateUsage] # cast-ok: regex to deployments + + def _write_pattern(self, regex: str, deployments: object) -> None: + self._commit({**self.patterns, regex: deployments}) + + def _commit(self, value: object) -> None: + pairs: Final = [ + [str(regex), _json(deployments)] for regex, deployments in (as_str_mapping(value) or {}).items() + ] + self._owner.write_router(self._team, pairs) + + def route(self, request: str | None, filtered_model_names: Sequence[str] | None = None) -> Routed | None: # pyright: ignore[reportIncompatibleMethodOverride] # the base's list types, read only here + if not isinstance(request, str): + # The base class matches its regexes against it, which fails and routes nowhere. + return super().route(request, None if filtered_model_names is None else list(filtered_model_names)) # pyright: ignore[reportReturnType, reportUnknownMemberType, reportUnknownVariableType] # None, or the base's copies of the deployments + return self._owner.route(self._team, request, filtered_model_names) + + def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model goes to the base class + return super().get_models_by_pattern(model, custom_llm_provider) + provider: Final = custom_llm_provider or inferred_provider(model) + return self._owner.models_by_pattern(self._team, [model, f"{provider}/{model}"]) + + +class _TeamRouters(dict[str, HostedPatternRouter]): + """``team_pattern_routers``: team id to its router; removing a team's router is a changeset.""" + + def __init__(self, owner: "HostedIndexes") -> None: + super().__init__() + self._owner: Final = owner + + def _load(self, routers: Mapping[str, HostedPatternRouter]) -> None: + super().clear() + super().update(routers) + + def __delitem__(self, team: str) -> None: + self._owner.drop_team_router(team) + + def pop(self, team: str, *default: object) -> object: + if team not in self: + if default: + return default[0] + raise KeyError(team) + router: Final = self[team] + self._owner.drop_team_router(team) + return router + + def clear(self) -> None: + self._owner.write({"team_patterns": []}) + + def setdefault(self, team: str, default: HostedPatternRouter | None = None) -> HostedPatternRouter: + if team not in self: + self._owner.write_router(team, []) + return self[team] + + +class HostedIndexes: + """Router-core's host indexes, as the live objects the registry hands out.""" + + def __init__( + self, + handle: Callable[[], "RouterCoreHandle | None"], + apply: Callable[[Mapping[str, object]], None], + ) -> None: + self._handle: Final = handle + self._apply: Final = apply + self._read_at: tuple[object, int] | None = None + self._document: Mapping[str, object] = {} + # The views are read again when an entry point hands them out, and before a write to them. + self.id_map: Final = _IndexDict(lambda value: self.write({"id_map": value}), self.refresh) + self.model_name: Final = _IndexDict(lambda value: self.write({"model_name": value}), self.refresh) + self.team_model: Final = _IndexDict(lambda value: self._write_team_model(value), self.refresh) + self.model_names: Final = _NameSet(lambda value: self.write({"model_names": value}), self.refresh) + self.deployment_names: Final = _Positions( + (), lambda value: self.write({"deployment_names": value}), self.refresh + ) + self.provider_default_ids: Final = _Positions( + (), lambda value: self.write({"provider_default_ids": value}), self.refresh + ) + self.pattern_router: Final = HostedPatternRouter(self, None) + self.team_pattern_routers: Final = _TeamRouters(self) + self._team_routers: dict[str, HostedPatternRouter] = {} # mutable-ok: one router object per team + self._team_public: frozenset[str] = frozenset() + + # -- reading ---------------------------------------------------------------------------------- + + def refresh(self) -> None: + """Read router-core's indexes again when its registry moved since the last read.""" + handle: Final = self._handle() + if handle is None: + return + version: Final = handle.registry_version() + if self._read_at is not None and self._read_at[0] is handle and self._read_at[1] == version: + return + self._read_at = (handle, version) + self._document = as_str_mapping(handle.host_indexes()) or {} + self._show() + + def _field(self, name: str) -> object: + return self._document.get(name) + + def _show(self) -> None: + ids: Final = _as_mapping(self._field("id_map")) + self.id_map._load(ids) # pyright: ignore[reportPrivateUsage] # a view this object owns + named: Final[dict[object, object]] = {} # mutable-ok: built here + for name, positions in (as_str_mapping(self._field("model_name")) or {}).items(): + named[name] = self._positions(positions, partial(self._write_named, name)) + self.model_name._load(named) # pyright: ignore[reportPrivateUsage] # a view this object owns + teams: Final[dict[object, object]] = {} # mutable-ok: built here + for team, public, positions in _triples(self._field("team_model")): + teams[(team, public)] = self._positions(positions, partial(self._write_team_key, (team, public))) + self.team_model._load(teams) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.model_names._load(_items(self._field("model_names"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.deployment_names._load(_items(self._field("deployment_names"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self.provider_default_ids._load(_items(self._field("provider_default_ids"))) # pyright: ignore[reportPrivateUsage] # a view this object owns + self._team_public = frozenset(str(name) for name in _items(self._field("team_public_model_names"))) + self.pattern_router._show(_pattern_pairs(self._field("patterns"))) # pyright: ignore[reportPrivateUsage] # the router this object owns + routers: Final[dict[str, HostedPatternRouter]] = {} # mutable-ok: rebuilt each read + for team, pairs in _team_pairs(self._field("team_patterns")): + router = self._team_routers.get(team) or HostedPatternRouter(self, team) + router._show(pairs) # pyright: ignore[reportPrivateUsage] # the router this object owns + routers[team] = router + self._team_routers = routers + self.team_pattern_routers._load(routers) # pyright: ignore[reportPrivateUsage] # a view this object owns + + def _write_named(self, name: str, items: object) -> None: + self.write({"model_name": {**self.model_name, name: items}}) + + def _write_team_key(self, key: tuple[str, str], items: object) -> None: + self._write_team_model({**self.team_model, key: items}) + + def _positions(self, positions: object, commit: _Commit) -> object: + """A position list as a write-through list; anything else a caller wrote, as written.""" + items: Final = _items(positions) + return _Positions(items, commit) if isinstance(positions, list) else positions + + @property + def team_public_model_names(self) -> frozenset[str]: + self.refresh() + return self._team_public + + # -- writing ---------------------------------------------------------------------------------- + + def write(self, fields: Mapping[str, object]) -> None: + """Replace the named indexes in router-core, then read them back.""" + self._apply({"host_index": {name: _json(value) for name, value in fields.items()}}) + self.refresh() + + def assign(self, field: IndexField, value: object) -> None: + """A deprecated attribute assignment of one index (``router.model_names = {...}``).""" + if field == "team_model": + self._write_team_model(value) + else: + self.write({field: value}) + + def _write_team_model(self, value: object) -> None: + mapping: Final = _as_mapping(value) + triples: Final[list[list[object]]] = [] # mutable-ok: JSON triples + for key, positions in mapping.items(): + parts = cast(tuple[object, ...], key) if isinstance(key, tuple) else () # cast-ok: a key a caller wrote + if len(parts) == 2: + triples.append([str(parts[0]), str(parts[1]), _json(positions)]) + self.write({"team_model": triples}) + + def write_router(self, team: str | None, pairs: list[list[object]]) -> None: # mutable-ok: JSON pairs + """Replace one pattern router's patterns.""" + if team is None: + self.write({"patterns": pairs}) + return + teams: Final = [[t, _json(_pairs_of(router))] for t, router in self._team_routers.items() if t != team] + self.write({"team_patterns": [*teams, [team, pairs]]}) + + def drop_team_router(self, team: str) -> None: + if team not in self._team_routers: + raise KeyError(team) + self.write({"team_patterns": [[t, _json(_pairs_of(r))] for t, r in self._team_routers.items() if t != team]}) + + # -- routing ---------------------------------------------------------------------------------- + + def route(self, team: str | None, request: str, filtered: Sequence[str] | None) -> Routed | None: + handle: Final = self._handle() + routed: Final = ( + None + if handle is None + else handle.pattern_route(team, request, None if filtered is None else list(filtered)) + ) + return None if routed is None else [dict(deployment) for deployment in routed] + + def models_by_pattern(self, team: str | None, requests: list[str]) -> frozenset[str]: # mutable-ok: read only + handle: Final = self._handle() + found: Final = None if handle is None else handle.models_by_pattern(team, requests) + return frozenset(found or ()) + + +def _items(value: object) -> list[object]: # mutable-ok: a fresh list + return list(cast(list[object], value)) if isinstance(value, list) else [] # cast-ok: a JSON list + + +def _triples(value: object) -> Iterator[tuple[str, str, object]]: + for item in _items(value): + parts = _items(item) + if len(parts) == 3: + yield str(parts[0]), str(parts[1]), parts[2] + + +def _pattern_pairs(value: object) -> _Pairs: + pairs: Final[_Pairs] = [] + for item in _items(value): + parts = _items(item) + if len(parts) == 2: + pairs.append((str(parts[0]), _items(parts[1]))) + return pairs + + +def _team_pairs(value: object) -> Iterator[tuple[str, _Pairs]]: + for item in _items(value): + parts = _items(item) + if len(parts) == 2: + yield str(parts[0]), _pattern_pairs(parts[1]) + + +def _pairs_of(router: HostedPatternRouter) -> list[list[object]]: # mutable-ok: JSON pairs + view: Final[Mapping[object, object]] = router._view # pyright: ignore[reportPrivateUsage] # a router this module owns + return [[regex, copy.deepcopy(deployments)] for regex, deployments in view.items()] diff --git a/litellm/router_rust/surface/legacy_members.py b/litellm/router_rust/surface/legacy_members.py new file mode 100644 index 00000000000..6c30557eb4d --- /dev/null +++ b/litellm/router_rust/surface/legacy_members.py @@ -0,0 +1,547 @@ +""" +``RustLegacyMembers``: the old public ``Router`` members outside the entry points, on the Rust +core: the routing-state lifecycle, the routing callbacks' checks, one provider call's accounting +and the pre-routing strategy builders. Each one is the Python router core's, ported (never +imported), over the state the Rust core and its host keep. + +Not served, and refused by ``RustCore.__getattr__``: the Python fallback walk +(``async_function_with_fallbacks``, ``function_with_fallbacks``, ``async_function_with_retries``, +``async_function_with_fallbacks_common_utils``) and ``factory_function``. They run a caller's +``original_function``, which picks its own deployment; router-core makes each pick itself and +attempts one host op, named by the endpoint's plan. +""" + +import asyncio +import contextlib +import inspect +import traceback +import uuid +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.add_retry_fallback_headers import response_total_token_count +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.coroutine_checker import coroutine_checker +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.litellm_core_utils.token_counter import served_openai_models as _served_openai_models +from litellm.router_api import RoutingContext +from litellm.router_rust.invoke.websocket import admit_deployment_turn +from litellm.router_rust.steps.callbacks import RustOptionalChecks +from litellm.router_rust.steps.prerouting import TIER_PARAMS_NEVER_DROPPED as _TIER_PARAMS_NEVER_DROPPED +from litellm.router_rust.surface.members import RustRegistryMembers +from litellm.router_strategy.auto_router.auto_router_model_naming import ( + GatedAutoRouterCapability, + capability_limit_violation, + count_capability_routers, +) +from litellm.router_utils.cooldown_handlers import set_cooldown_deployments +from litellm.router_utils.endpoint_eligibility import endpoint_selection +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_io_token_refund_failure +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET +from litellm.types.router import RouterCacheEnum +from litellm.utils import get_utc_datetime + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + from litellm.caching.dual_cache import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.router_rust.invoke.endpoints import EndpointCore + from litellm.types.router import CustomRoutingStrategyBase, Deployment, RoutingGroup + +# `RoutingArgs.ttl`: a usage counter lives for its minute. +_USAGE_TTL: Final = 60 + +# What `initialize_assistants_endpoint` binds. +_ASSISTANTS_ENDPOINTS: Final = ( + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", +) + +# What `initialize_router_endpoints` binds, with the container file endpoints it reads from +# `litellm/containers/endpoints.json`. +_ROUTER_ENDPOINTS: Final = frozenset( + { + "_arealtime", + "_aresponses_websocket", + "aadapter_generate_content", + "aanthropic_messages", + "acancel_fine_tuning_job", + "acancel_interaction", + "acancel_responses", + "acompact_responses", + "acreate_agent", + "acreate_container", + "acreate_fine_tuning_job", + "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "acreate_skill", + "adecisions", + "adelete_agent", + "adelete_container", + "adelete_interaction", + "adelete_responses", + "adelete_skill", + "aextraction", + "afile_content", + "afile_delete", + "afile_list", + "agenerate_content", + "agenerate_content_stream", + "aget_agent", + "aget_interaction", + "aget_responses", + "aget_skill", + "aimage_edit", + "alist_agent_versions", + "alist_agents", + "alist_containers", + "alist_fine_tuning_jobs", + "alist_input_items", + "alist_skills", + "allm_passthrough_route", + "amoderation", + "anthropic_messages", + "aocr", + "arealtime_calls", + "arealtime_translation_calls", + "aresponses", + "aretrieve_container", + "aretrieve_fine_tuning_job", + "asearch", + "asystemone", + "avector_store_delete", + "avector_store_file_content", + "avector_store_file_create", + "avector_store_file_delete", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_update", + "avector_store_list", + "avector_store_retrieve", + "avector_store_search", + "avector_store_update", + "avideo_content", + "avideo_create_character", + "avideo_edit", + "avideo_extension", + "avideo_generation", + "avideo_get_character", + "avideo_list", + "avideo_remix", + "avideo_status", + "cancel_interaction", + "create_agent", + "create_container", + "create_interaction", + "decisions", + "delete_agent", + "delete_container", + "delete_interaction", + "extraction", + "generate_content", + "generate_content_stream", + "get_agent", + "get_interaction", + "list_agent_versions", + "list_agents", + "list_containers", + "ocr", + "responses", + "retrieve_container", + "search", + "systemone", + "vector_store_create", + "vector_store_delete", + "vector_store_file_content", + "vector_store_file_create", + "vector_store_file_delete", + "vector_store_file_list", + "vector_store_file_retrieve", + "vector_store_file_update", + "vector_store_list", + "vector_store_retrieve", + "vector_store_search", + "vector_store_update", + "video_content", + "video_create_character", + "video_edit", + "video_extension", + "video_generation", + "video_get_character", + "video_list", + "video_remix", + "video_status", + } +) + + +def _dispatch_failure(logging_obj: "Logging | None", error: Exception) -> None: + """The ``## LOG FAILURE EVENT`` of a routing callback that raised.""" + if logging_obj is not None: + asyncio.create_task( + logging_obj.dispatch_failure_handlers( + exception=error, + traceback_exception=traceback.format_exc(), + prefer_async_handlers=True, + ) + ) + + +class _FailedCallParams(TypedDict): + """The ``litellm_params`` of a failed call, as the failure callback reads them.""" + + metadata: ReadOnly[Mapping[str, object]] + model_info: ReadOnly[NotRequired[Mapping[str, object] | None]] + + +async def _deployments_filtered_by( + callback: CustomLogger, + model: str, + deployments: object, + messages: object, + request_kwargs: dict[str, object] | None, # mutable-ok: CustomLogger takes the request dict + parent_otel_span: "Span | None", +) -> object: + return await callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + model=model, + healthy_deployments=deployments, # pyright: ignore[reportArgumentType] # what the last callback returned + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + request_kwargs=request_kwargs, + parent_otel_span=parent_otel_span, + ) + + +def _metadata_model_id(model_info: object) -> object: + return model_info.get("id") if isinstance(model_info, dict) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the metadata's model_info dict + + +class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): + """Mixed into ``RustCore``, which supplies its host registry, selectors and checks.""" + + # RustCore sets these in its constructor. + _cache: "DualCache" + _cooldown_given: object + _custom_strategy: "CustomRoutingStrategyBase | None" + # The constructor's argument, or the attribute as assigned. + auto_router_capability_limit: "Callable[[], int | None] | None" + + TIER_PARAMS_NEVER_DROPPED: Final = _TIER_PARAMS_NEVER_DROPPED + + # -- registry lookups -------------------------------------------------------------------------- + + def get_routing_group(self, model_name: str) -> "RoutingGroup | None": + """The routing group callable as ``model_name`` (``get_routing_group``).""" + return self._registry_state.get_routing_group(model_name) + + def team_public_model_names_of(self, team_id: str) -> frozenset[str]: + """The public names ``team_id`` serves its own deployments under (``team_public_model_names_of``).""" + return self._registry_state.team_public_model_names_of(team_id) + + def served_openai_models(self, model_name: str) -> tuple[str | None, ...]: + """The OpenAI models ``model_name`` serves (``_served_openai_models``). The Python core keeps + an ``lru_cache`` over it, emptied with every model list change.""" + return _served_openai_models(self._registry_state._core_deployments_named(model_name) or ()) # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + + def auto_router_capability_violation(self, capability: GatedAutoRouterCapability) -> str | None: + """Why one more router claiming ``capability`` cannot join this router, or None when it can + (``auto_router_capability_violation``); the limit is resolved on every call.""" + resolver: Final = self.auto_router_capability_limit + limit: Final = resolver() if resolver is not None else None + others: Final = count_capability_routers( + (deployment for deployment in self._registry_state._core_model_list if isinstance(deployment, Mapping)), # pyright: ignore[reportPrivateUsage, reportUnnecessaryIsInstance] # the live list + capability=capability, + ) + return capability_limit_violation(capability=capability, held=others + 1, limit=limit) + + # -- routing state ----------------------------------------------------------------------------- + + def retire_routing_state(self) -> None: + """Stop the background work of the selectors and the budget limiter (``retire_routing_state``).""" + self._opt.retire_state((*self._sel.callback_selectors(), *self._sel.override_selectors())) + + def routing_strategy_init(self, routing_strategy: object, routing_strategy_args: Mapping[str, object]) -> None: + """``routing_strategy_init``: the default selector only. As on the Python core the router's + ``routing_strategy`` setting stays, so picks keep following it; ``apply`` changes it.""" + verbose_router_logger.info("Routing strategy: %s", routing_strategy) + validate_routing_strategy(routing_strategy) # pyright: ignore[reportArgumentType] # the strategy as given + # `_reset_custom_routing_strategy`. + self._custom_strategy = None + _ = self.__dict__.pop("get_available_deployment", None) + _ = self.__dict__.pop("_core_async_get_available_deployment", None) + self._sel.init_default(routing_strategy, routing_strategy_args, configured=False) + + def set_optional_pre_call_checks(self, optional_pre_call_checks: Iterable[object] | None) -> None: + """``set_optional_pre_call_checks``: drop the toggleable checks the list leaves out, then add it.""" + self._opt.set(None if optional_pre_call_checks is None else tuple(optional_pre_call_checks)) + + def initialize_assistants_endpoint(self) -> None: + """``initialize_assistants_endpoint`` binds the endpoints; this core answers them from + ``__getattr__``, so what is left to do is to drop an assigned replacement.""" + for name in _ASSISTANTS_ENDPOINTS: + _ = self.__dict__.pop(name, None) + + def initialize_router_endpoints(self) -> None: + """``initialize_router_endpoints``, as ``initialize_assistants_endpoint`` above.""" + from litellm.containers.endpoint_factory import ( + _generated_endpoints, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # the names the Python core registers + ) + + for name in (*_ROUTER_ENDPOINTS, *_generated_endpoints): + _ = self.__dict__.pop(name, None) + + # -- pre-routing strategies -------------------------------------------------------------------- + + def init_auto_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's auto-router (``init_auto_router_deployment``).""" + self._registry_state.strategies._init_auto_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # RustCore installs its strategies in its constructor + + def init_complexity_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's complexity-router (``init_complexity_router_deployment``).""" + self._registry_state.strategies._init_complexity_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + def init_quality_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's quality-router (``init_quality_router_deployment``).""" + self._registry_state.strategies._init_quality_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + def init_adaptive_router_deployment(self, deployment: "Deployment") -> None: + """Register the deployment's adaptive-router and its post-call hook (``init_adaptive_router_deployment``).""" + self._registry_state.strategies._init_adaptive_router(deployment) # pyright: ignore[reportOptionalMemberAccess, reportPrivateUsage] # see above + + # -- routing callbacks ------------------------------------------------------------------------- + + def routing_strategy_pre_call_checks( + self, + deployment: dict[str, object], # mutable-ok: CustomLogger takes the deployment dict + ) -> None: + """``routing_strategy_pre_call_checks``: the sync check of each ``CustomLogger`` callback.""" + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # CustomLogger takes an untyped deployment dict + + async def async_routing_strategy_pre_call_checks( + self, + deployment: dict[str, object], # mutable-ok: CustomLogger takes the deployment dict + parent_otel_span: "Span | None", + logging_obj: "Logging | None" = None, + ) -> None: + """``async_routing_strategy_pre_call_checks``: a check's rate limit cools the deployment down + under the Python rules (``set_cooldown_deployments``, cooldown event included).""" + from litellm import exceptions as litellm_exceptions + + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + try: + await callback.async_pre_call_check(deployment, parent_otel_span) # pyright: ignore[reportUnknownMemberType] # see above + except litellm_exceptions.RateLimitError as e: + _dispatch_failure(logging_obj, e) + _ = set_cooldown_deployments( + ctx=RoutingContext(self), # pyright: ignore[reportArgumentType] # the core's RoutingContext + exception_status=e.status_code, + original_exception=e, + deployment=deployment["model_info"]["id"], # pyright: ignore[reportIndexIssue, reportUnknownArgumentType] # read as the Python core reads it + time_to_cooldown=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it + ) + raise + except Exception as e: + _dispatch_failure(logging_obj, e) + raise + + async def async_callback_filter_deployments( + self, + model: str, + healthy_deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: object, + parent_otel_span: "Span | None", + request_kwargs: dict[str, object] | None = None, # mutable-ok: CustomLogger takes the request dict + logging_obj: "Logging | None" = None, + ) -> object: + """``async_callback_filter_deployments``: each routing callback narrows the deployments in turn.""" + remaining: object = healthy_deployments # rebind-ok: narrowed by each callback + for callback in self._opt.routing_callbacks(): + if isinstance(callback, CustomLogger): + try: + remaining = await _deployments_filtered_by( + callback, model, remaining, messages, request_kwargs, parent_otel_span + ) + except Exception as e: + _dispatch_failure(logging_obj, e) + raise + return remaining + + async def async_admit_deployment_turn( + self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + """Admit a later turn of an open websocket (``async_admit_deployment_turn``).""" + port: Final[EndpointCore] = self.__dict__["_port"] # pyright: ignore[reportAny] # RustCore's own attribute + return await admit_deployment_turn(port, model_group, opened_on, request_kwargs) + + # -- one provider call ------------------------------------------------------------------------- + + async def make_call( + self, + original_function: Callable[..., object], + *args: object, + **kwargs: object, # kwargs-ok: the endpoint's request + ) -> object: + """``make_call``: call the endpoint, count its usage and stamp the response's headers. A + failed call hands back the io tokens it reserved.""" + model_group: Final = kwargs.get("model") + generic_function: Final = kwargs.get("original_generic_function") + endpoint_function: Final = generic_function if callable(generic_function) else original_function + try: + with endpoint_selection( + getattr(endpoint_function, "__name__", None), + getattr(endpoint_function, "__module__", None), + model=model_group, + ): + response: object = original_function(*args, **kwargs) # rebind-ok: awaited below + if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): + awaitable: Final[Awaitable[object]] = response # pyright: ignore[reportAssignmentType] # the endpoint's awaitable + response = await awaitable # rebind-ok: awaited here + except Exception: + with contextlib.suppress(Exception): + await async_io_token_refund_failure( + self._cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) + ) + raise + await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) + return await self.set_response_headers( + response=response, model_group=model_group if isinstance(model_group, str) else None, request_kwargs=kwargs + ) + + async def increment_deployment_usage_for_response( + self, response: object, request_kwargs: Mapping[str, object] + ) -> None: + """Count a response's tokens and request against its deployment, once the call returned, and + note them on the request for the success callback (``increment_deployment_usage_for_response``).""" + if response is None: + return + try: + deployment_metadata: Final = find_deployment_metadata(request_kwargs) + model_group: Final = request_kwargs.get("model") + if deployment_metadata is None or not isinstance(model_group, str): + return + deployment_id: Final = _metadata_model_id(deployment_metadata["model_info"]) + if deployment_id is None: + return + total_tokens: Final = response_total_token_count(response) + deployment_name: Final = deployment_metadata.get("deployment") + deployment_metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] = total_tokens + try: + await self._increment_deployment_usage( + deployment_id=str(deployment_id), + deployment_name=deployment_name if isinstance(deployment_name, str) else None, + model_group=model_group, + total_tokens=total_tokens, + parent_otel_span=get_parent_otel_span_from_kwargs(dict(request_kwargs)), + ) + except Exception: + _ = deployment_metadata.pop(ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, None) + raise + except Exception as e: # noqa: BLE001 # as the Python core: usage accounting never fails a served call + verbose_router_logger.debug( + "litellm.router.Router::increment_deployment_usage_for_response(): Exception occured - %s", e + ) + + async def _increment_deployment_usage( + self, + *, + deployment_id: str, + deployment_name: str | None, + model_group: str, + total_tokens: float, + parent_otel_span: "Span | None", + ) -> None: + """``_increment_deployment_usage`` with its one request, in one pipeline; this worker's memory + then holds what Redis answered.""" + from litellm.types.caching import RedisPipelineIncrementOperation + + if not self._registry_state.cached_deployment_tracks_usage(deployment_id, model_group): + return + current_minute: Final = get_utc_datetime().strftime("%H-%M") + tpm_key: Final = RouterCacheEnum.TPM.value.format( + id=deployment_id, current_minute=current_minute, model=deployment_name + ) + rpm_key: Final = RouterCacheEnum.RPM.value.format( + id=deployment_id, current_minute=current_minute, model=deployment_name + ) + operations: Final[Sequence[RedisPipelineIncrementOperation]] = [ + RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=_USAGE_TTL) + for key, increment_value in ((tpm_key, total_tokens), (rpm_key, 1)) + ] + values: Final = await self._cache.async_increment_cache_pipeline( + increment_list=list(operations), parent_otel_span=parent_otel_span + ) + if values is not None and self._cache.redis_cache is not None: + for operation, value in zip(operations, values): + await self._cache.async_set_cache(operation["key"], int(value), local_only=True, ttl=_USAGE_TTL) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + + async def async_deployment_callback_on_failure( + self, kwargs: Mapping[str, object], completion_response: object, start_time: object, end_time: object + ) -> None: + """Count a failed call as one more request of the minute (``async_deployment_callback_on_failure``).""" + if is_batch_retrieve_call_type(kwargs.get("call_type")): + return + litellm_params: Final[_FailedCallParams] = kwargs["litellm_params"] # pyright: ignore[reportAssignmentType] # read as the Python core reads it, a missing key included + deployment_name: Final = litellm_params["metadata"].get("deployment", None) + model_group: Final = litellm_params["metadata"].get("model_group", None) + model_info: Final = litellm_params.get("model_info", {}) or {} + deployment_id: Final = model_info.get("id", None) + if model_group is None or deployment_id is None: + return + rpm_key: Final = RouterCacheEnum.RPM.value.format( + id=deployment_id, + current_minute=get_utc_datetime().strftime("%H-%M"), + model=deployment_name, + ) + with service_target(ROUTER_USAGE_TARGET): + await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped + key=rpm_key, + value=1, + parent_otel_span=get_parent_otel_span_from_kwargs(dict(kwargs)), + ttl=_USAGE_TTL, + ) + + # -- the plugin context ------------------------------------------------------------------------ + + def _update_kwargs_before_fallbacks( + self, + model: str, + kwargs: dict[str, object], # mutable-ok: the core updates the request kwargs in place + metadata_variable_name: str | None = "metadata", + ) -> None: + """``_update_kwargs_before_fallbacks``: a trace id and the requested group, in place.""" + _ = kwargs.setdefault("litellm_trace_id", str(uuid.uuid4())) + alias: Final = model if self._registry_state._core_get_model_from_alias(model=model) else None # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + kwargs.setdefault(metadata_variable_name, {}).update({"model_group": model, "model_group_alias": alias}) # pyright: ignore[reportArgumentType, reportAttributeAccessIssue, reportUnknownMemberType] # the bucket, as the Python core writes it + + def _params_sent_to_deployment( + self, request_kwargs: Mapping[str, object], deployment: "Deployment | None" + ) -> Mapping[str, object]: + """``_params_sent_to_deployment`` of a ``Deployment``, which ``RoutingContext.deployment_params`` hands in.""" + params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + configured: Final = as_str_mapping(RoutingContext(self).settings().default_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownArgumentType] # the core's RoutingContext + defaults: Final = {k: v for k, v in (configured or {}).items() if v is not None} + tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} + return MappingProxyType({**params, **defaults, **tool_choice, **request_kwargs}) + + +__all__ = ["RustLegacyMembers"] diff --git a/litellm/router_rust/surface/members.py b/litellm/router_rust/surface/members.py new file mode 100644 index 00000000000..0f7dc23d966 --- /dev/null +++ b/litellm/router_rust/surface/members.py @@ -0,0 +1,163 @@ +""" +``RustRegistryMembers``: the Rust core's public registry members outside ``registry()`` and +``apply``: ``add_deployment``, ``set_model_list``, the ``model_list`` attribute, the environment +gate, the deprecated id hash, ``print_deployment`` and the pre-call checks the plugin context reads. + +Each one works on the core's host registry (``litellm/router_rust/surface/registry.py``), which hands every +change to the Rust core. The rules are the Python router core's, ported (never imported). +""" + +import copy +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker +from litellm.router_api import deployment_id +from litellm.router_rust.steps.precall import count_tokens, pre_call_checks +from litellm.router_rust.surface.registry import HostRegistry, model_info_is_active_for_environment +from litellm.types.router import Deployment + +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + from litellm.router_rust.core import RustCore + + +class _CallCounter: + """``total_calls``, ``success_calls`` or ``fail_calls``: plain attributes on the Python core, + which a write replaces and a ``del`` removes; every call then reads what is there.""" + + def __init__(self, name: str) -> None: + self._name: Final = name + + def __get__(self, core: "RustCore | None", owner: type | None = None) -> object: + if core is None: + return self + try: + return getattr(core._endpoints.counters, self._name) # pyright: ignore[reportPrivateUsage] # the core's own counters + except AttributeError: + raise core._no_attribute(self._name) from None # pyright: ignore[reportPrivateUsage] # see above + + def __set__(self, core: "RustCore", value: object) -> None: + setattr(core._endpoints.counters, self._name, value) # pyright: ignore[reportPrivateUsage] # see __get__ + + def __delete__(self, core: "RustCore") -> None: + try: + delattr(core._endpoints.counters, self._name) # pyright: ignore[reportPrivateUsage] # see __get__ + except AttributeError: + raise core._no_attribute(self._name) from None # pyright: ignore[reportPrivateUsage] # see __get__ + + +class RustRegistryMembers: + """Mixed into ``RustCore``, which supplies its host registry and host.""" + + total_calls: Final = _CallCounter("total_calls") + success_calls: Final = _CallCounter("success_calls") + fail_calls: Final = _CallCounter("fail_calls") + + @property + def _registry_state(self) -> HostRegistry: + raise NotImplementedError + + @property + def _python_host(self) -> "PythonHost": + raise NotImplementedError + + def add_deployment(self, deployment: Deployment) -> Deployment | None: + """Add a deployment unless its id exists (``add_deployment``).""" + return self._registry_state.add_deployment(deployment) + + def set_model_list(self, model_list: Sequence[object]) -> None: + """Replace every deployment (``set_model_list``); aliases and wildcard routes stay.""" + self._registry_state.set_model_list(model_list) + + @property + def model_list(self) -> Sequence[object]: + """The live model list (the deprecated ``model_list`` attribute).""" + return self._registry_state._core_model_list # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + + def _assign_model_list(self, model_list: Sequence[object]) -> None: + """``router.model_list = [...]``: the list is swapped as given, without reindexing, and the + registry version moves once.""" + self._registry_state.replace_model_list(model_list) # pyright: ignore[reportArgumentType] # the list as assigned, as the core keeps it + + @staticmethod + def deployment_is_active_for_environment(deployment: Deployment) -> bool: + """Whether LITELLM_ENVIRONMENT admits the deployment (``deployment_is_active_for_environment``).""" + return model_info_is_active_for_environment(model_info=deployment.model_info) # pyright: ignore[reportArgumentType] # ModelInfo reads like a mapping + + @staticmethod + def generate_model_id(model_group: str, litellm_params: Mapping[str, object]) -> str: + """The deprecated id hash (``generate_model_id``).""" + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the router's deprecation notice + ) + + _note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)") + return deployment_id(model_group, litellm_params) + + async def set_response_headers( + self, response: object, model_group: str | None = None, request_kwargs: Mapping[str, object] | None = None + ) -> object: + """Stamp the group's remaining usage and the model id on a response (``set_response_headers``).""" + headed: Final[object] = await self._registry_state.set_response_headers( # pyright: ignore[reportAny] # the response as the core hands it back + response, model_group, None if request_kwargs is None else dict(request_kwargs) + ) + return headed + + def replay_model_cost_registrations(self) -> None: + """Re-register the served deployments' cost-map entries (``replay_model_cost_registrations``).""" + self._registry_state.replay_model_cost_registrations() + + def _count_pre_call_check_tokens( + self, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, # mutable-ok: the request field, as the Python core takes it + request_kwargs: Mapping[str, object] | None = None, + tokenizer_model: str = "", + deployments: Sequence[Mapping[str, object]] = (), + ) -> int: + """``_count_pre_call_check_tokens``, which context compaction counts with.""" + return count_tokens(self._python_host, messages, input, request_kwargs, tokenizer_model, deployments) + + def _pre_call_checks( + self, + model: str, + healthy_deployments: Sequence[Mapping[str, object]], + messages: object = None, + input: object = None, + request_kwargs: Mapping[str, object] | None = None, + input_token_count: int | None = None, + skip_inline_token_count: bool = False, + request_deployments: Sequence[Mapping[str, object]] | None = None, + ) -> Sequence[Mapping[str, object]]: + """``_pre_call_checks``, which context compaction admits a client-managed request with.""" + return pre_call_checks( + self._python_host, + self._registry_state, # pyright: ignore[reportArgumentType] # the host registry serves the reads + model, + healthy_deployments, + messages, + input, + request_kwargs, + input_token_count, + skip_inline_token_count, + request_deployments, + ) + + def print_deployment(self, deployment: Mapping[str, object]) -> Mapping[str, object]: + """A copy of the deployment, of the type it was given, with its api key masked + (``print_deployment``). A ``Deployment`` and its params read and write like dicts.""" + try: + printed: Final = copy.deepcopy(deployment) + litellm_params: Final = printed["litellm_params"] + if self._python_host.dynamic().redact_user_api_key_info: + masker: Final = SensitiveDataMasker(visible_prefix=2, visible_suffix=0) + printed["litellm_params"] = masker.mask_dict(litellm_params) # pyright: ignore[reportArgumentType, reportIndexIssue] # the deployment's params, written as the Python core writes them + elif "api_key" in litellm_params: # pyright: ignore[reportOperatorIssue] # see the docstring + litellm_params["api_key"] = litellm_params["api_key"][:2] + "*" * 10 # pyright: ignore[reportIndexIssue] # see the docstring + except Exception as e: + verbose_router_logger.debug("Error occurred while printing deployment - %s", e) + raise + else: + return printed diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py new file mode 100644 index 00000000000..f742225e2f4 --- /dev/null +++ b/litellm/router_rust/surface/model_info.py @@ -0,0 +1,871 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownParameterType=false, reportMissingParameterType=false, reportAny=false, reportExplicitAny=false, reportMissingTypeArgument=false, reportAttributeAccessIssue=false, reportTypedDictNotRequiredAccess=false +# (ported from the Python router core as it is written there, over untyped litellm helpers) +""" +``HostModelInfo``: model info, group info, usage reads and response headers for the Rust core's +host registry. + +router-core holds the rules: a deployment's model info (``get_router_model_info``, +``get_deployment_model_info``: ``router-core/src/model_info.rs``), a model group's info and the +configured-value reads (``_set_model_group_info``, ``get_model_listing_info``, the configured token +limits, mode, display name and service tiers: ``router-core/src/group_info.rs``), and the usage +counters' keys (``RouterCoreHandle.usage_key``). What lives in this process is answered here as a +query asks for it, at query time: the cost map (``get_model_info``, ``litellm.model_cost``), +provider resolution, the wildcard pattern routes, what the background refresh discovered, and a +group deployment's validated params, supported params, endpoints, fast mode and reasoning efforts. +``ModelGroupInfo`` is built here from the group's first deployment (its validation is the model's) +and takes what router-core's merge assigned. + +Still the Python router core's rules here: the model group alias a group-info query resolves +first (the alias map is the registry's live Python objects), the cache of discovered limits, the +configured ``max_input_tokens`` read (a value echoed from the loaded cost map is skipped), and the +response headers. +""" + +import builtins +import dataclasses +import math +from collections.abc import Callable, Mapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Union, cast # noqa: TID251 # the core's own signatures, ported unchanged + +from pydantic import BaseModel, TypeAdapter +from typing_extensions import overload + +import litellm +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.add_retry_fallback_headers import ( + apply_quality_router_decision_headers, + apply_remaining_usage_headers, + apply_response_model_id, + complexity_router_decision_headers, + ensure_response_additional_headers, + prepare_response_for_header_attachment, +) +from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints +from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.reasoning_effort_capability import ( + deployment_is_catalog_mapped, + intersect_supported_reasoning_efforts, + resolve_supported_reasoning_efforts, +) +from litellm.litellm_core_utils.routing_common_utils import team_may_use_deployment +from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.bedrock.common_utils import get_bedrock_base_model +from litellm.router_python_host import python_host +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET +from litellm.types.router import ( + Deployment, + DeploymentModelListingInfo, + DeploymentTypedDict, + DiscoveredDeploymentModelInfo, + LiteLLM_Params, + ModelGroupInfo, +) +from litellm.types.utils import COST_MAP_LOOKUP_KEY, ModelInfo, echoed_cost_map_fields +from litellm.types.utils import ModelInfo as ModelMapInfo +from litellm.utils import get_utc_datetime, is_generalized_model_info + +if TYPE_CHECKING: + from litellm.rust_bridge._native import ( + RouterCall, + RouterCoreHandle, + _ModelInfoSteps, # pyright: ignore[reportPrivateUsage] # the stub's step shapes + ) + +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + +# The key router-core's merge reads as an opaque value (``router-core/src/model_info.rs``). +_OPAQUE_KEY: Final = "__litellm_python_value__" + + +class _Wire: + """Values as router-core's model-info merge reads them: plain JSON, with what JSON cannot spell + (a datetime, a tuple, a non-finite float) passed as an opaque token the merge moves between + layers without reading, and restored to the same object on the way back.""" + + def __init__(self) -> None: + self._values: Final[list[object]] = [] # mutable-ok: the tokens of one query + + def to_wire(self, value: object) -> object: + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, int): + return value if -(2**63) <= value < 2**64 else self._token(value) + if isinstance(value, float): + return value if math.isfinite(value) else self._token(value) + if isinstance(value, list): + return [self.to_wire(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {str(key): self.to_wire(item) for key, item in value.items()} + return self._token(value) + + def from_wire(self, value: object) -> object: + if isinstance(value, dict): + if len(value) == 1 and _OPAQUE_KEY in value: + return self._values[int(value[_OPAQUE_KEY])] + return {str(key): self.from_wire(item) for key, item in value.items()} + if isinstance(value, list): + return [self.from_wire(item) for item in value] + return value + + def _token(self, value: object) -> Mapping[str, int]: + self._values.append(value) + return {_OPAQUE_KEY: len(self._values) - 1} + + +def _drive_model_info( + call: "RouterCall[_ModelInfoSteps]", answer: Callable[[Mapping[str, object]], object] +) -> Mapping[str, object]: + """Drive one model-info merge to its end, answering each lookup with ``answer``. A lookup that + raises raises out of the query, as it raised in the Python router.""" + outcome: object = None # rebind-ok: each lookup's answer + while True: + step = call.next(outcome) + if step["kind"] == "done": + return step + if step["kind"] == "failed": + if str(step.get("error_kind")) == "model_info": + exception: object = getattr(builtins, str(step.get("error_exception")), ValueError) + raise (exception if isinstance(exception, type) else ValueError)(step.get("error_detail")) + raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}") + outcome = answer(step) + + +def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, object]: + return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) + + +def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: + """``_operator_max_input_tokens``: the deployment's own limit, skipping one a saved + ``/model/info`` response copied from the cost map.""" + model_info: Final = _configured_model_info(deployment) + catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY) + catalog_entry: Final = ( + GetModelCostMap.loaded_model_cost_map().get(catalog_key) if isinstance(catalog_key, str) else None + ) + if catalog_entry is not None and "max_input_tokens" in echoed_cost_map_fields(model_info, catalog_entry): + return None + return coerce_token_limit(model_info.get("max_input_tokens")) + + +# The ModelGroupInfo fields router-core's group merge compares against, as the model validated them. +_GROUP_COMPARED_FIELDS: Final = frozenset( + { + "providers", + "max_input_tokens", + "max_output_tokens", + "input_cost_per_token", + "output_cost_per_token", + "supports_fast_mode", + "supported_endpoints", + "supported_reasoning_efforts", + } +) +# The ModelGroupInfo fields the Python router assigns tuples to. +_GROUP_TUPLE_FIELDS: Final = frozenset({"supported_endpoints", "supported_reasoning_efforts"}) + + +def _group_field(field: str, value: object) -> object: + """A merged group field as the Python router assigns it: a tuple field takes the list as a tuple.""" + if field in _GROUP_TUPLE_FIELDS and isinstance(value, list): + return tuple(cast(list[object], value)) # cast-ok: the merge's list + return value + + +@dataclasses.dataclass +class _GroupBuild: + """One group-info merge as the host answers it: the deployments it reads, each one's validated + params once its first lookup builds them, and the group built from the first deployment.""" + + model_list: "Sequence[dict[str, Any]]" # mutable-ok: the stored entries, read only + team_id: str | None + # Filled as the lookups arrive. + params: dict[int, LiteLLM_Params] = dataclasses.field(default_factory=dict) # mutable-ok: see above + info: ModelGroupInfo | None = None + + +class HostModelInfo: + """Mixed into ``HostRegistry``, which holds the model list, the indexes and the host.""" + + def _live_handle(self) -> "RouterCoreHandle | None": ... # the registry's: the handle ``RustCore`` set + + def _core_handle(self) -> "RouterCoreHandle": + handle: Final = self._live_handle() + if handle is None: + raise RuntimeError("the host registry reads model info through its Rust core") + return handle + + def _core_get_model_list_of_routed_group( + self, model_group: str, team_id: str | None = None + ) -> list: # mutable-ok: a fresh list, as the core returns it + if team_id is not None and self._team_deployments_named(team_id, model_group): + return self._core_get_all_deployments(model_name=model_group, team_id=team_id) + named: Final = [ + *self._core_get_all_deployments(model_name=model_group), + *self._core_get_model_list_from_routing_groups(model_name=model_group), + ] + return named or self._get_wildcard_deployments(model_name=model_group) + + def _get_wildcard_deployments( + self, model_name: str, team_id: str | None = None + ) -> list[dict[str, object]]: # mutable-ok: a fresh list + """The deployments a wildcard pattern routes ``model_name`` to, renamed to it (the routing + runs in router-core, through the registry's pattern router views).""" + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + matches: Final = [ + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), + *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), + ] + return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] # pyright: ignore[reportCallIssue] # a stored entry + + def _core_get_model_group_info( + self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None + ) -> ModelGroupInfo | None: + """ + For a given model group name, return the combined model info + + Returns: + - ModelGroupInfo if able to construct a model group + - None if error constructing model group info or hidden model group (unless include_hidden) + """ + ## Check if model group alias + if model_group in self._core_model_group_alias: + item: Final = self._core_model_group_alias[model_group] + if isinstance(item, str): + _router_model_group = item # rebind-ok: ported from the Python core unchanged + elif isinstance(item, dict): + if item["hidden"] is True and not include_hidden: + return None + else: + _router_model_group = item["model"] # rebind-ok: ported from the Python core unchanged + else: + return None + + return self._set_model_group_info( + model_group=_router_model_group, + user_facing_model_group_name=model_group, + team_id=team_id, + ) + + ## Check if actual model + return self._set_model_group_info( + model_group=model_group, user_facing_model_group_name=model_group, team_id=team_id + ) + + def _set_model_group_info( + self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None + ) -> ModelGroupInfo | None: + """ + For a given model group name, return the combined model info + + Returns: + - ModelGroupInfo if able to construct a model group + - None if error constructing model group info + + router-core merges the group (``router-core/src/group_info.rs``); this picks its + deployments, answers what needs this process (validated params, model info, provider, + capabilities), builds ``ModelGroupInfo`` from the first deployment and sets what the merge + assigned on it. + """ + serves_team_deployments: Final = team_id is not None and bool( + self._team_deployments_named(team_id, model_group) + ) + model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id) + description: Final[str | None] = next( + (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), + None, + ) + # The wildcard check is asked only for a deployment that is not the group's by name. + pattern_routes: Final = ( + not serves_team_deployments + and any("model_name" in m and m["model_name"] != model_group for m in model_list) + and self._core_pattern_router.route(model_group) is not None + ) + wire: Final = _Wire() + group: Final = _GroupBuild(model_list=model_list, team_id=team_id) + call: Final = self._core_handle().model_group_info( + { + "model_group": model_group, + "user_facing_model_group_name": user_facing_model_group_name, + "serves_team_deployments": serves_team_deployments, + "pattern_routes": pattern_routes, + "description": description, + "deployments": [wire.to_wire(model) for model in model_list], + } + ) + done: Final = _drive_model_info(call, partial(self._answer_group_lookup, group=group, wire=wire)) + answer: Final = as_str_mapping(done["value"]) + if answer is None or group.info is None: + return None + for field, value in (as_str_mapping(answer["assignments"]) or {}).items(): + setattr(group.info, field, _group_field(field, wire.from_wire(value))) + return group.info + + def _answer_group_lookup(self, step: Mapping[str, object], group: "_GroupBuild", wire: "_Wire") -> object: + """One lookup of a group-info merge: a deployment's facts, or the group's first entry.""" + match step["lookup"]: + case "group_params": + index: Final = int(str(step["index"])) + litellm_params: Final = LiteLLM_Params(**group.model_list[index]["litellm_params"]) + group.params[index] = litellm_params # rebind-ok: the merge's state, which each lookup advances + return { + "model": litellm_params.model, + "configurable_clientside_auth_params": wire.to_wire( + litellm_params.configurable_clientside_auth_params + ), + } + case "group_deployment": + return self._group_deployment(group, int(str(step["index"])), wire) + case "group_capabilities": + deployment: Final = group.model_list[int(str(step["index"]))] + merged_info: Final[object] = wire.from_wire(step["model_info"]) + model_info: Final = cast(Mapping[str, object], merged_info) # cast-ok: the merged model info + litellm_model: Final = str(step["litellm_model"]) + llm_provider: Final = str(step["llm_provider"]) + efforts: Final = resolve_supported_reasoning_efforts( + model_info, deployment_is_mapped=step["mapped"] is True + ) + return { + "supported_endpoints": wire.to_wire( + list( + deployment_supported_endpoints( + model_info.get("supported_endpoints"), deployment, llm_provider + ) + ) # pyright: ignore[reportArgumentType] # a stored entry, as the core passes it + if team_may_use_deployment(_configured_model_info(deployment).get("team_id"), group.team_id) # pyright: ignore[reportArgumentType] # the configured team id + else [] + ), + "supports_fast_mode": AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider), + "reasoning_efforts": None if efforts is None else list(efforts), + } + case "group_seed": + entry: Final[object] = wire.from_wire(step["info"]) + seed: Final = cast(dict[str, Any], entry) # cast-ok: the first deployment's entry + group.info = ModelGroupInfo(**seed) # rebind-ok: the merge's state, built from the first deployment + return group.info.model_dump(mode="json", include=set(_GROUP_COMPARED_FIELDS)) + case "reasoning_intersect": + current: Final = cast("Sequence[str] | None", step["current"]) # cast-ok: plain JSON efforts + resolved: Final = cast("Sequence[str] | None", step["resolved"]) # cast-ok: plain JSON efforts + intersected: Final = intersect_supported_reasoning_efforts(current, resolved) + return None if intersected is None else list(intersected) + case _: + return self._answer_model_info_lookup(step, wire) + + def _group_deployment(self, group: "_GroupBuild", index: int, wire: "_Wire") -> Mapping[str, object]: + """A group deployment's model info (None when it fails), whether the catalog maps it, and its + provider; with no model info, the provider's supported params.""" + model: Final = group.model_list[index] + litellm_params: Final = group.params[index] + model_info_dict: Final = model.get("model_info", {}) + model_info: object + try: + model_id: Final = model_info_dict.get("id", None) + model_info = ( # rebind-ok: one branch binds it + self._core_get_deployment_model_info(model_id=model_id, model_name=litellm_params.model) + if model_id is not None + else None + ) + except Exception: # noqa: BLE001 # ported as the core writes it + model_info = None # rebind-ok: see above + deployment_is_mapped: Final = deployment_is_catalog_mapped(model_info, model_info_dict) # pyright: ignore[reportArgumentType] # the merged model info, or None + litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) + answer: Final[dict[str, object]] = { # mutable-ok: built here + "model_info": wire.to_wire(model_info), + "mapped": deployment_is_mapped, + "litellm_model": litellm_model, + "llm_provider": llm_provider, + } + if model_info is None: + answer["supported_openai_params"] = wire.to_wire( + self._host.invoke("get_supported_openai_params")(model=litellm_model, custom_llm_provider=llm_provider) + ) + return answer + + def _cached_get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + """ + Cached version of get_model_group_info, uses @lru_cache wrapper + + This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request + """ + return self._core_get_model_group_info(model_group) + + def _core_cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + return self._cached_get_model_group_info(model_group) + + @overload + def _core_get_router_model_info( + self, + deployment: Union[dict, "Deployment"], # mutable-ok: ported from the Python core unchanged + received_model_name: str, + id: None = None, + ) -> ModelMapInfo: + pass + + @overload + def _core_get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> ModelMapInfo: + pass + + def _core_get_router_model_info( + self, + deployment: Union[dict, "Deployment"] | None, # mutable-ok: ported from the Python core unchanged + received_model_name: str, + id: str | None = None, + ) -> ModelMapInfo: + """ + For a given model id, return the model info (max tokens, input cost, output cost, etc.). + + Augment litellm info with additional params set in `model_info`. + + For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set. + For every other provider, a base_model naming the same model as `model:` can only widen its context window. + + router-core merges it (``router-core/src/model_info.rs``); this checks the deployment and + answers the cost-map lookups the merge asks for while it runs. + + Returns + - ModelInfo - If found -> typed dict with max tokens, input cost, etc. + + Raises: + - ValueError -> If model is not mapped yet + """ + if id is not None: + _deployment: Final = self._core_get_deployment(model_id=id) + if _deployment is not None: + deployment = _deployment # rebind-ok: ported from the Python core unchanged + + if deployment is None: + raise ValueError("Deployment not found") + + model_info_base_model: Final = (deployment.get("model_info") or {}).get("base_model", None) + params_base_model: Final = ( + (deployment.get("litellm_params") or {}).get("base_model", None) if model_info_base_model is None else None + ) + + ## GET PROVIDER - reuse LiteLLM_Params if already constructed + litellm_params_data: Final = deployment.get("litellm_params") + litellm_params: LiteLLM_Params + if isinstance(litellm_params_data, LiteLLM_Params): + litellm_params = litellm_params_data # rebind-ok: ported from the Python core unchanged + elif isinstance(litellm_params_data, dict) and "model" in litellm_params_data: + litellm_params = LiteLLM_Params.model_validate( # rebind-ok: ported from the Python core unchanged + litellm_params_data + ) + else: + raise ValueError( + f"Deployment missing valid litellm_params. " + f"Got: {type(litellm_params_data).__name__}, " + f"deployment_id: {(deployment.get('model_info') or {}).get('id', 'unknown')}" + ) + + wire: Final = _Wire() + user_model_info: dict[str, object] = {} # mutable-ok: filled when the merge asks for it # rebind-ok: see above + + def read_user_model_info() -> object: + nonlocal user_model_info + raw_user_model_info: Final = deployment.get("model_info") + user_model_info = dict( # rebind-ok: see above + _MODEL_INFO_ADAPTER.validate_python( + raw_user_model_info.model_dump(exclude_none=True) + if isinstance(raw_user_model_info, BaseModel) + else raw_user_model_info or {} + ) + ) + return wire.to_wire(user_model_info) + + def answer(step: Mapping[str, object]) -> object: + if step["lookup"] == "provider": + _model, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( + model=litellm_params.model, + litellm_params=litellm_params, + ) + return [_model, custom_llm_provider] + if step["lookup"] == "user_model_info": + return read_user_model_info() + return self._answer_model_info_lookup(step, wire) + + query: Final = { + "deployment_id": wire.to_wire((deployment.get("model_info") or {}).get("id")), + "model_info_base_model": wire.to_wire(model_info_base_model), + "params_base_model": wire.to_wire(params_base_model), + "params_model": litellm_params.model, + "extra_base_model": wire.to_wire((litellm_params.model_extra or {}).get("base_model")), + "received_model_name": received_model_name, + } + done: Final = _drive_model_info(self._core_handle().router_model_info(query), answer) + merged: Final = wire.from_wire(done["info"]) + if not isinstance(merged, dict): + return merged # pyright: ignore[reportReturnType] # the core returns get_model_info's answer as is + # The deployment's own values are its objects, as the Python router merges them. + for key in cast(list[str], done["user_keys"]): # cast-ok: router-core's key list + merged[key] = user_model_info[key] + return merged # pyright: ignore[reportReturnType] # the merged map, as the core builds it + + def _core_get_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None: + """ + For a given model id, return the model info + + 1. Check if model_id is in model info + 2. If not, check if litellm model name is in model info + 3. If not, return None + + router-core merges it (``router-core/src/model_info.rs``); this answers its lookups. + """ + wire: Final = _Wire() + call: Final = self._core_handle().deployment_model_info(model_id, wire.to_wire(model_name)) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + return wire.from_wire(done["info"]) # pyright: ignore[reportReturnType] # the merged map, as the core builds it + + def _answer_model_info_lookup(self, step: Mapping[str, object], wire: "_Wire") -> object: + """One cost-map lookup of a model-info merge, answered from this process.""" + match step["lookup"]: + case "model_info": + return self._model_info_row(wire.from_wire(step["model"]), wire, step) + case "cost_entry": + key: Final = str(step["key"]) + model_cost: Final = litellm.model_cost if step["process"] else self._host.dynamic().model_cost + return {"entry": wire.to_wire(model_cost.get(key))} if key in model_cost else None + case "pattern_models": + return self._pattern_models(str(step["name"]), wire) + case "bedrock_base": + return get_bedrock_base_model(str(step["model_id"])) + case "discovered": + return wire.to_wire(dict(self._core_get_discovered_model_info(wire.from_wire(step["id"])))) # pyright: ignore[reportArgumentType] # the deployment's id, as configured + case "log": + log: Final = verbose_router_logger.error if step["level"] == "error" else verbose_router_logger.debug + log("%s", step["message"]) + return None + case other: + raise ValueError(f"router-core asked an unknown model-info lookup {other!r}") + + def _model_info_row(self, model: object, wire: "_Wire", step: Mapping[str, object]) -> object: + """``get_model_info(model=...)`` as a lookup answers it: the row, ``{"failed": True}`` when a + tolerated lookup raises, and for an exact lookup whether a generalization rule made it.""" + # The base model's own window is read through the process host, as the Python router reads it. + host: Final = python_host if step["exact"] else self._host + try: + row: Final = host.invoke("get_model_info")(model=model) + except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model + if step["tolerate"]: + return {"failed": True} + raise + if step["exact"]: + return {"row": wire.to_wire(row), "generalized": is_generalized_model_info(row)} + return wire.to_wire(row) + + def _pattern_models(self, name: str, wire: "_Wire") -> object: + """The ``{id, model}`` of each deployment a wildcard pattern routes ``name`` to.""" + potential_models: Final = self._core_pattern_router.route(name) + if potential_models is None: + return None + routed: Final[list[object]] = [] # mutable-ok: built here + for potential_model in potential_models: + try: + routed.append( + { + "id": wire.to_wire((potential_model.get("model_info") or {}).get("id")), + "model": wire.to_wire((potential_model.get("litellm_params") or {}).get("model")), + } + ) + except Exception: # noqa: BLE001 # ported as the core writes it + verbose_router_logger.debug("model info lookup failed; ignored as the core ignores it") + return routed + + def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) + if ( + model_id is not None + and isinstance(cached, DiscoveredDeploymentModelInfo) + and cached.deployment is self._core_get_model_info(model_id) + ): + configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) + return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) + return MappingProxyType({}) + + def _core_get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + """ + Return what the concrete deployments behind model_name contribute to its + /v1/models entry: the cost-map keys for their underlying models, plus the widest + configured or discovered token limits. Resolved via O(1) index + lookup. + + Returns None for wildcard-expanded or unknown names, where the listed name is the + real model name and no deployment-specific information exists, and treats a + malformed configured limit as absent rather than failing the listing. + + The whole group is read rather than just its first deployment, so a group that + mixes models does not advertise a window that depends on config order; the widest + one is reported, which is what get_model_group_info already shows the Admin UI. + Keys are deduplicated, so the ordinary group of interchangeable deployments of one + model still costs the caller a single cost-map lookup. Unlike get_model_group_info, + this never triggers pattern matching or deep copies, so it is safe to call per + listed model on the /v1/models hot path. + + router-core combines the deployments (``router-core/src/group_info.rs``). + """ + deployments: Final = self._core_deployments_named(model_name) + if not deployments: + return None + wire: Final = _Wire() + call: Final = self._core_handle().model_listing_info([wire.to_wire(deployment) for deployment in deployments]) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + listing: Final = as_str_mapping(done["value"]) or {} + return DeploymentModelListingInfo( + cost_map_keys=tuple(cast(list[str], listing["cost_map_keys"])), # cast-ok: router-core's key list + max_input_tokens=cast("int | None", listing["max_input_tokens"]), # cast-ok: router-core's limit + max_output_tokens=cast("int | None", listing["max_output_tokens"]), # cast-ok: router-core's limit + ) + + def _core_get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": + """ + Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete + deployment of model_name, via O(1) index lookup. + + Returns (None, None) for wildcard-expanded or unknown names, and treats a + malformed configured value as absent rather than failing the caller. + + Deliberately reads one deployment rather than aggregating the group the way + get_model_listing_info does: its caller truncates an embedding input to this + value, so the widest window in a mixed group would be the wrong answer there. + """ + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return (None, None) + wire: Final = _Wire() + configured: Final = { + key: wire.to_wire(value) for key, value in deployment.model_info.model_dump(exclude_none=True).items() + } + call: Final = self._core_handle().configured_token_limits(wire.to_wire(deployment.model_info.id), configured) + done: Final = _drive_model_info(call, partial(self._answer_model_info_lookup, wire=wire)) + limits: Final = cast("list[int | None]", done["value"]) # cast-ok: router-core's two limits + return (limits[0], limits[1]) + + def _core_get_configured_mode(self, model_name: str) -> "str | None": + """Return the mode explicitly configured for a concrete deployment.""" + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + return self._core_handle().configured_text(_Wire().to_wire(deployment.model_info.get("mode"))) + + def _core_get_configured_display_name(self, model_name: str) -> "str | None": + """ + Return the display_name explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, and treats a + non-string or empty configured value as absent rather than failing the + listing. Like get_configured_token_limits, this never triggers pattern + matching or deep copies, so it is safe to call per listed model on the + /v1/models hot path. + """ + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + return self._core_handle().configured_text(_Wire().to_wire(deployment.model_info.get("display_name"))) + + def _core_get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + """ + Return the service_tiers value each deployment a request from team_id + to model_name can route to (see _routable_deployments) configures in + its model_info, unvalidated and in model_list order, None for a + deployment that sets none; the caller validates the shape it expects + and decides how the deployments combine. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + wire: Final = _Wire() + tiers: Final = self._core_handle().configured_service_tiers( + [ + wire.to_wire(dict(_configured_model_info(deployment))) + for deployment in self._routable_deployments(model_name, team_id) + ] + ) + return tuple(wire.from_wire(tier) for tier in tiers) + + def _core_get_configured_max_input_tokens( + self, model_name: str, team_id: str | None = None + ) -> tuple[int | None, ...]: + """ + Return the max_input_tokens each deployment a request from team_id to + model_name can route to (see _routable_deployments) sets in its own + model_info, in model_list order, None for a deployment that sets none + or a malformed one. Unlike get_model_listing_info, neither the cost map + nor a discovered limit is read, so only an operator's own value counts, + and a value a saved /model/info response copied from the cost map is + skipped too. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + return tuple( + _operator_max_input_tokens(deployment) for deployment in self._routable_deployments(model_name, team_id) + ) + + async def _core_get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: + """ + Returns current tpm/rpm usage for model group + + Parameters: + - model_group: str - the received model name from the user (can be a wildcard route). + + Returns: + - usage: Tuple[tpm, rpm] + """ + dt: Final = get_utc_datetime() + current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock + tpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + rpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + + model_list: Final = self._core_get_model_list(model_name=model_group) + if model_list is None: # no matching deployments + return None, None + + for model in model_list: + id: str | None = model.get("model_info", {}).get("id") + litellm_model: str | None = model["litellm_params"].get( + "model" + ) # USE THE MODEL SENT TO litellm.completion() - consistent with how global_router cache is written. + if id is None or litellm_model is None: + continue + tpm_keys.append(self._core_handle().usage_key(id, litellm_model, "tpm", current_minute)) + rpm_keys.append(self._core_handle().usage_key(id, litellm_model, "rpm", current_minute)) + combined_tpm_rpm_keys: Final = tpm_keys + rpm_keys + + with service_target(ROUTER_USAGE_TARGET): + combined_tpm_rpm_values: Final = await self._core_cache.async_batch_get_cache(keys=combined_tpm_rpm_keys) + if combined_tpm_rpm_values is None: + return None, None + + rpm_start: Final = len(tpm_keys) + tpm_usage_list: Final[list | None] = combined_tpm_rpm_values[:rpm_start] # mutable-ok: ported from the Python core unchanged # fmt: skip + rpm_usage_list: Final[list | None] = combined_tpm_rpm_values[rpm_start:] # mutable-ok: ported from the Python core unchanged # fmt: skip + + ## TPM + tpm_usage: int | None = None # rebind-ok: ported from the Python core unchanged + if tpm_usage_list is not None: + for t in tpm_usage_list: + if isinstance(t, int): + if tpm_usage is None: + tpm_usage = 0 + tpm_usage += t + ## RPM + rpm_usage: int | None = None # rebind-ok: ported from the Python core unchanged + if rpm_usage_list is not None: + for t in rpm_usage_list: + if isinstance(t, int): + if rpm_usage is None: + rpm_usage = 0 + rpm_usage += t + return tpm_usage, rpm_usage + + async def get_model_group_io_token_usage(self, model_group: str) -> tuple[int | None, int | None]: + """ + Returns current ITPM/OTPM usage for a model group (sum across deployments). + """ + dt: Final = get_utc_datetime() + current_minute: Final = dt.strftime("%H-%M") + itpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + otpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged + + model_list: Final = self._core_get_model_list(model_name=model_group) + if model_list is None: + return None, None + + for model in model_list: + model_id: str | None = model.get("model_info", {}).get("id") + litellm_model: str | None = model["litellm_params"].get("model") + if model_id is None or litellm_model is None: + continue + itpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "itpm", current_minute)) + otpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "otpm", current_minute)) + + with service_target(ROUTER_USAGE_TARGET): + combined_values: Final = await self._core_cache.async_batch_get_cache(keys=itpm_keys + otpm_keys) + if combined_values is None: + return None, None + + itpm_values: Final = combined_values[: len(itpm_keys)] + otpm_values: Final = combined_values[len(itpm_keys) :] + + total_itpm: int | None = None # rebind-ok: ported from the Python core unchanged + for value in itpm_values: + if isinstance(value, int): + total_itpm = (total_itpm or 0) + value + + total_otpm: int | None = None # rebind-ok: ported from the Python core unchanged + for value in otpm_values: + if isinstance(value, int): + total_otpm = (total_otpm or 0) + value + + return total_itpm, total_otpm + + async def _core_get_remaining_model_group_usage( + self, model_group: str + ) -> dict[str, int]: # mutable-ok: ported from the Python core unchanged + model_group_info: Final = self._cached_get_model_group_info(model_group) + + returned_dict: Final[dict[str, int]] = {} # mutable-ok: ported from the Python core unchanged + + # ITPM/OTPM groups emit input/output token headers, but they may also set + # tpm/rpm, so build both sets rather than returning early - clients and + # prometheus gauges that read the standard headers still get data. + if model_group_info is not None and (model_group_info.itpm is not None or model_group_info.otpm is not None): + current_itpm, current_otpm = await self.get_model_group_io_token_usage(model_group) + returned_dict.update( + build_io_token_rate_limit_headers( + itpm_limit=model_group_info.itpm, + otpm_limit=model_group_info.otpm, + current_itpm=current_itpm, + current_otpm=current_otpm, + ) + ) + + tpm_limit: Final = model_group_info.tpm if model_group_info is not None else None + rpm_limit: Final = model_group_info.rpm if model_group_info is not None else None + + if tpm_limit is None and rpm_limit is None: + return returned_dict + + current_tpm, current_rpm = await self._core_get_model_group_usage(model_group) + + if tpm_limit is not None: + returned_dict["x-ratelimit-remaining-tokens"] = tpm_limit - (current_tpm or 0) + returned_dict["x-ratelimit-limit-tokens"] = tpm_limit + if rpm_limit is not None: + returned_dict["x-ratelimit-remaining-requests"] = rpm_limit - (current_rpm or 0) + returned_dict["x-ratelimit-limit-requests"] = rpm_limit + + return returned_dict + + async def set_response_headers( + self, + response: object, + model_group: str | None = None, + request_kwargs: dict[str, object] | None = None, # mutable-ok: ported from the Python core unchanged + ) -> Any: # noqa: ANN401 # the core's signature, ported unchanged + """ + Add the most accurate rate limit headers for a given model response. + + ## TODO: add model group rate limit headers + # - if healthy_deployments > 1, return model group rate limit headers + # - else return the model's rate limit headers + """ + response = prepare_response_for_header_attachment(response) # rebind-ok: ported from the Python core unchanged + if response is None: + return response + + additional_headers: Final = ensure_response_additional_headers(response) + apply_response_model_id( + response, find_deployment_metadata(request_kwargs) if request_kwargs is not None else None + ) + additional_headers["x-litellm-model-group"] = model_group + apply_quality_router_decision_headers(additional_headers, request_kwargs) + additional_headers.update(complexity_router_decision_headers(request_kwargs)) + + if model_group is not None: + remaining_usage: Final = await self._core_get_remaining_model_group_usage(model_group) + apply_remaining_usage_headers(additional_headers, remaining_usage) + return response diff --git a/litellm/router_rust/surface/model_list.py b/litellm/router_rust/surface/model_list.py new file mode 100644 index 00000000000..405a4b47d7b --- /dev/null +++ b/litellm/router_rust/surface/model_list.py @@ -0,0 +1,375 @@ +""" +``HostedModelList``: the Rust core's live ``model_list``, read back from router-core. + +router-core holds the deployments. This list is what ``registry().all()`` and the deprecated +``model_list`` attribute hand out: a ``list`` (so callers that check for one, serialize it or +validate it keep working) whose contents are router-core's deployment records, read through the +handle after every write. Every write to it is a changeset instead of a Python-side mutation: + +- ``append`` / ``extend`` / an ``insert`` at the end add at the end (``append``) +- removing one deployment whose id no other entry shares is a ``delete`` +- any other write (an assignment, a slice, a removal of one of several entries sharing an id, + ``clear``, ``sort``, ...) hands router-core the resulting list (``replace``) + +A build serves the deployments it added once it ends (or fails), and the list reads them back +then. After writes that only appended, it reads just the appended records +(``RouterCoreHandle.deployments_from``), so adding deployments one at a time costs each one its own +record rather than the whole list. These writes are the raw ones a caller makes: like the Python +router's list, they reindex nothing. + +An entry read back from router-core is a dict whose writes (its own, or a nested dict's) hand +router-core the list again, so an edit in place reaches it as the Python router's list would show +it. Each deployment keeps one Python object while router-core's record for it is unchanged, keyed by +its id and its place among entries sharing that id, so identity checks on stored entries (the +model info caches) mean what they did: the same object until a write changes that deployment. +""" + +import copy +import datetime +from collections.abc import Callable, Iterable, Mapping, Sequence +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + SupportsIndex, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # any value a caller wrote into the list + overload, +) + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.types.router import ModelAccessWindow + +if TYPE_CHECKING: + from litellm.rust_bridge._native import RouterCoreHandle + +Entry: TypeAlias = dict[str, object] +# (id, its place among the entries sharing it) -> (the record it was read as, the entry object) +_Stable: TypeAlias = dict[tuple[str, int], tuple[object, Entry]] + + +_DATETIME: Final = TypeAdapter(datetime.datetime) +_Field = TypeVar("_Field") # rebind-ok: a TypeVar cannot be Final + + +def _dumped_as(adapter: TypeAdapter[_Field]) -> Callable[[object], object]: + """A value validated by ``adapter``, then dumped back to its Python form.""" + + def dump(value: object) -> object: + return adapter.dump_python(adapter.validate_python(value), exclude_none=True) + + return dump + + +# The ``ModelInfo`` fields whose Python value is no JSON value, which router-core holds as strings. +_TYPED_MODEL_INFO: Final[Mapping[str, Callable[[object], object]]] = MappingProxyType( + { + "created_at": _dumped_as(_DATETIME), + "updated_at": _dumped_as(_DATETIME), + "ptu_effective_from": _dumped_as(_DATETIME), + "ptu_effective_to": _dumped_as(_DATETIME), + "access_windows": _dumped_as(TypeAdapter(tuple[ModelAccessWindow, ...])), + } +) + + +def typed_deployment(record: Mapping[str, object]) -> Mapping[str, object]: + """A deployment record as ``Deployment.to_json`` dumped it: its ``model_info`` typed again. One + with none of those fields comes back as it is, and so does a value a raw write left unreadable.""" + info: Final = as_str_mapping(record.get("model_info")) + if info is None or info.keys().isdisjoint(_TYPED_MODEL_INFO): + return record + typed: Final[dict[str, object]] = {} # mutable-ok: filled here + for key, dumped in _TYPED_MODEL_INFO.items(): + value = info.get(key) + if value is None: + continue + try: + typed[key] = dumped(value) + except ValidationError: + continue + return {**record, "model_info": {**info, **typed}} + + +def _record(value: object) -> Mapping[str, object]: + return cast(Mapping[str, object], value) # cast-ok: a JSON object router-core wrote, or a dict inside one + + +def _entry_id(entry: object) -> str | None: + info: Final = as_str_mapping((as_str_mapping(entry) or {}).get("model_info")) or {} + value: Final = info.get("id") + return None if value is None else str(value) + + +def _keys( + entries: Iterable[object], seen: Mapping[str, int] = MappingProxyType({}) +) -> tuple[list[tuple[str, int] | None], Mapping[str, int]]: # mutable-ok: a fresh list + """Each entry's identity key (its id and how many earlier entries share it, ``seen`` counting + the ids of entries before these), and the counts once past them.""" + counts: Final[dict[str, int]] = dict(seen) # mutable-ok: counted here + keys: Final[list[tuple[str, int] | None]] = [] # mutable-ok: filled here + for entry in entries: + entry_id = _entry_id(entry) + if entry_id is None: + keys.append(None) + continue + nth = counts.get(entry_id, 0) + counts[entry_id] = nth + 1 + keys.append((entry_id, nth)) + return keys, counts + + +class _HostedEntry(dict[str, object]): + """A deployment entry read back from router-core: a write to it, or to a dict inside it, hands + router-core the list again (``replace``), so it learns what the entry now says. A copy is a + plain dict.""" + + def __init__(self, record: Mapping[str, object], written: Callable[[], None]) -> None: + super().__init__( + (key, value if (nested := as_str_mapping(value)) is None else _HostedEntry(nested, written)) + for key, value in record.items() + ) + self._written: Final = written + + def __setitem__(self, key: str, value: object) -> None: + super().__setitem__(key, value) + self._written() + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + self._written() + + def pop(self, key: str, *default: object) -> object: + value: Final = super().pop(key, *default) + self._written() + return value + + def popitem(self) -> tuple[str, object]: + item: Final = super().popitem() + self._written() + return item + + def clear(self) -> None: + super().clear() + self._written() + + def update(self, *args: object, **kwargs: object) -> None: # kwargs-ok: dict.update's own arguments + super().update(*args, **kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # dict.update's own arguments + self._written() + + def setdefault(self, key: str, default: object = None) -> object: + if key in self: + return self[key] + self[key] = default + return default + + def __ior__(self, other: object) -> "_HostedEntry": # pyright: ignore[reportIncompatibleMethodOverride] # dict's in-place merge, as a write + self.update(other) + return self + + def __copy__(self) -> dict[str, object]: # mutable-ok: a plain copy + return dict(self) + + def __deepcopy__(self, memo: dict[int, object]) -> dict[str, object]: # mutable-ok: a plain copy + return copy.deepcopy(dict(self), memo) + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + return (dict, (copy.deepcopy(dict(self)),)) + + +class HostedModelList(list[Entry]): + """The live ``model_list``: router-core's deployments, every write a changeset.""" + + def __init__( + self, + handle: Callable[[], "RouterCoreHandle | None"], + apply: Callable[[Mapping[str, object]], None], + ) -> None: + super().__init__() + self._handle: Final = handle + self._apply: Final = apply + # The handle and registry version the contents were last read at. + self._read_at: tuple[object, int] | None = None + self._stable: _Stable = {} + # How many of the shown entries hold each id, for the keys of entries appended next. + self._id_counts: Mapping[str, int] = {} + # While a build runs: the length, id counts and read to go back to when it ends. + self._building: tuple[int, Mapping[str, int], tuple[object, int] | None] | None = None + + # -- reading ---------------------------------------------------------------------------------- + + def refresh(self) -> None: + """Read router-core's deployments again when its registry moved since the last read.""" + handle: Final = self._handle() + if handle is None or self._building is not None: + # While a build runs, the list shows what it accepted until router-core serves it. + return + version: Final = handle.registry_version() + if self._read_at is not None and self._read_at[0] is handle and self._read_at[1] == version: + return + read_at: Final = self._read_at + self._read_at = (handle, version) + if read_at is not None and read_at[0] is handle and handle.registry_rewritten() <= read_at[1]: + # Only appends since the last read: the shown entries stand, and the new ones follow. + super().extend(self._kept(handle.deployments_from(len(self)), appended=True)) + return + self._show(self._kept(handle.deployments())) + + def _kept(self, records: Sequence[object], *, appended: bool = False) -> list[Entry]: # mutable-ok: a fresh list + """``records`` as entries, each the object it was while its record is unchanged; when + ``appended``, they follow the shown entries, which keep theirs.""" + stable: Final[_Stable] = self._stable if appended else {} + keys, counts = _keys(records, self._id_counts if appended else {}) + entries: Final[list[Entry]] = [] # mutable-ok: filled here + for key, record in zip(keys, records, strict=True): + typed = typed_deployment(_record(record)) + held = self._stable.get(key) if key is not None else None + kept = held is not None and held[0] == typed + entry = held[1] if held is not None and kept else _HostedEntry(typed, self._entry_written) + if key is not None: + stable[key] = (held[0] if held is not None and kept else copy.deepcopy(typed), entry) + entries.append(entry) + self._stable = stable + self._id_counts = counts + return entries + + # -- a build ---------------------------------------------------------------------------------- + + def begin_build(self, *, replacing: bool) -> None: + """A build starts: router-core serves what it adds once it ends, so until then the list + shows each deployment it accepts here (``built``), after the entries it keeps, or alone + when it replaces them, as the Python router's list grows while it adds them.""" + self.refresh() + if replacing: + self._show([]) + self._id_counts = {} + self._building = (len(self), self._id_counts, self._read_at) + + def built(self, entry: Mapping[str, object]) -> None: + """A deployment the build accepted, shown until the build ends.""" + if self._building is not None: + super().extend(self._kept([entry], appended=True)) + + def end_build(self) -> None: + """The build ended (or failed): router-core serves what it added, read back in place of + the entries shown meanwhile, each kept as the object it was shown as while unchanged.""" + if self._building is None: + return + start, counts, read_at = self._building + self._building = None + super().__delitem__(slice(start, None)) + self._id_counts = counts + self._read_at = read_at + self.refresh() + + def _seed(self, entries: Sequence[object]) -> None: + """Keep ``entries``, the list router-core now holds, as the objects its records for them + read back as.""" + for key, entry in zip(_keys(entries)[0], entries, strict=True): + self._seed_one(key, entry) + + def _seed_appended(self, entries: Sequence[object]) -> None: + """Keep ``entries``, appended after the shown ones, as the objects they read back as.""" + for key, entry in zip(_keys(entries, self._id_counts)[0], entries, strict=True): + self._seed_one(key, entry) + + def _seed_one(self, key: tuple[str, int] | None, entry: object) -> None: + if key is not None and isinstance(entry, dict): + stored: Final = cast(Entry, entry) # cast-ok: a model_list entry is a dict + self._stable[key] = (copy.deepcopy(stored), stored) + + def _entry_written(self) -> None: + """A write inside an entry: router-core takes the list as it now reads.""" + self._replace(list(self)) + + def _show(self, entries: list[Entry]) -> None: # mutable-ok: the entries shown + super().__setitem__(slice(None), entries) + + # -- writing ---------------------------------------------------------------------------------- + + def _replace(self, entries: list[Entry]) -> None: # mutable-ok: the list router-core takes + # Seeded first: router-core's records for them read back as these objects however soon + # the list is read again. + self._seed(entries) + self._apply({"replace": entries}) + self.refresh() + + def replace(self, entries: Iterable[object]) -> None: + """Every deployment at once (``router.model_list = [...]``).""" + self._replace(cast(list[Entry], list(entries))) # cast-ok: the entries as assigned, which router-core validates + + def append(self, value: Entry) -> None: + self.refresh() + self._seed_appended([value]) + self._apply({"append": [value]}) + self.refresh() + + def extend(self, values: Iterable[Entry]) -> None: + for value in tuple(values): + self.append(value) + + def __iadd__(self, values: Iterable[Entry]) -> "HostedModelList": # pyright: ignore[reportIncompatibleMethodOverride] # list's in-place add, as a changeset + self.extend(values) + return self + + def insert(self, index: SupportsIndex, value: Entry) -> None: + if int(index) >= len(self): + self.append(value) + return + changed: Final = list(self) + changed.insert(index, value) + self._replace(changed) + + @overload + def __setitem__(self, index: SupportsIndex, value: Entry) -> None: ... + @overload + def __setitem__(self, index: slice, value: Iterable[Entry]) -> None: ... + def __setitem__(self, index: SupportsIndex | slice, value: object) -> None: + changed: Final = list(self) + changed[index] = value # pyright: ignore[reportArgumentType, reportCallIssue] # list's own item and slice assignment + self._replace(changed) + + def __delitem__(self, index: SupportsIndex | slice) -> None: + changed: Final = list(self) + if isinstance(index, slice): + del changed[index] + self._replace(changed) + return + removed: Final = changed.pop(index) + removed_id: Final = _entry_id(removed) + if removed_id is None or any(_entry_id(entry) == removed_id for entry in changed): + # A delete removes every deployment with the id; the list removes only this one. + self._replace(changed) + return + self._apply({"delete": [removed_id]}) + self.refresh() + + def pop(self, index: SupportsIndex = -1) -> Entry: + entry: Final = self[index] + del self[index] + return entry + + def remove(self, value: Entry) -> None: + del self[self.index(value)] + + def clear(self) -> None: + self._replace([]) + + def sort(self, *args: object, **kwargs: object) -> None: # kwargs-ok: list.sort's own arguments + changed: Final = list(self) + changed.sort(*args, **kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # list.sort's own arguments + self._replace(changed) + + def reverse(self) -> None: + self._replace(list(reversed(self))) + + def __imul__(self, count: SupportsIndex) -> "HostedModelList": # pyright: ignore[reportIncompatibleMethodOverride] # list's in-place repeat, as a changeset + self._replace(list(self) * int(count)) + return self + + def __reduce_ex__(self, protocol: SupportsIndex) -> tuple[object, ...]: + # A copy or pickle of the live list is a plain list of its entries. + return (list, (list(self),)) diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py new file mode 100644 index 00000000000..a55a1cfc6d6 --- /dev/null +++ b/litellm/router_rust/surface/registry.py @@ -0,0 +1,2191 @@ +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAny=false +# (the ported core code reads untyped litellm helpers: PatternMatchRouter, Deployment.to_json, host.invoke) +""" +``HostRegistry``: the Rust core's live ``model_list`` and the indexes ``registry()`` reads. + +router-core builds the deployments (``litellm-rust/crates/router-core/src/build.rs``): it walks +the list, decides which values are ``os.environ/`` secret references, assigns the id of a +deployment that declares none and adds one deployment per ``organization``. What needs live Python +objects is a host step this registry answers, one deployment at a time (``_answer_build_step``): +``secrets`` (``get_secret``), ``validate`` (the PTU, access-window and strict param checks, and the +pricing each deployment writes into ``litellm.model_cost``), ``provider`` (``get_llm_provider``) +and ``added`` (pass-through credentials, pre-routing strategies, the patterns and indexes below). + +router-core holds the deployments. ``_core_model_list`` is ``HostedModelList`` +(litellm/router_rust/surface/model_list.py): router-core's records read back through the handle, every +write a changeset (a build serves what it added once it ends, a removal is a delete, any other +write a replace), each deployment one Python object while its record is unchanged. + +The indexes are router-core's too (``host_index.rs``), kept by the Python router's history rules: +the id, group and team position maps, ``model_names``, the only-growing ``deployment_names``, +the team public names, the provider default ids and the wildcard pattern routers. The registry +hands them out as the live views of litellm/router_rust/surface/indexes.py, whose writes are changesets. + +The lookups run in router-core (``lookup.rs``): the deployments by name (team public name first), +the model list with its alias, routing group and wildcard rows, the ids, names and access groups +it lists, the candidate ids for a route, the routable deployments, the credential deployment picks +and a team model's alternatives. Each answers with rows, positions in the list: this registry +reads its stored entries at them (``_rows``), so identity and in-place reads stay its own, and a +position router-core could not read (an index a caller wrote junk into) is read here and raises +as the Python router's read raised. What stays here needs live Python objects: the alias map and +routing groups a lookup is scoped by (``_list_scope``), the provider a wildcard request is tried +under, pydantic ``Deployment`` values, the credentials, the PTU ceilings, search tool support, a +team model's provider pick (``provider_for_generic_call``) and the SDK clients. + +Model info and group info are router-core's rules too (``model_info.rs``, ``group_info.rs``), +answered from this process through ``HostModelInfo`` (litellm/router_rust/surface/model_info.py). + +The attribute and method names follow the router core's (``_core_model_list``, +``_core_get_model_list`` ...), so ``litellm.router_api.RouterRegistry`` reads this object as its +router and every ``registry()`` query dispatches the way it does on the Python core. +""" + +import copy +import dataclasses +import os +import re +import threading +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import lru_cache, partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast # noqa: TID251 # entries as a caller wrote them + +from pydantic import TypeAdapter + +from litellm import exceptions as litellm_exceptions +from litellm import get_secret_str +from litellm._internal_context import service_target +from litellm._logging import verbose_router_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE +from litellm.litellm_core_utils.access_windows import access_windows_config_error +from litellm.litellm_core_utils.core_helpers import as_str_mapping, without +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_llm_provider_logic import ( + declared_authenticating_provider, + is_registered_custom_provider, + resolve_model_and_provider_for_metadata, +) +from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter +from litellm.litellm_core_utils.ptu_pricing import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + declares_ptu, + declares_ptu_shares, + is_ptu_cost_attribution_enabled, + ptu_config_error, + ptu_identity_error, + ptu_terms, + zeroed_ptu_pricing, +) +from litellm.litellm_core_utils.ptu_shares import ptu_capacity_warning +from litellm.litellm_core_utils.routing_common_utils import ( + provider_for_generic_call, + resolve_model_group_alias, + team_may_use_deployment, + warn_on_ignored_model_capability_flags, + warn_on_provider_credential_mismatch, + with_admin_api_key_secret_reference, + without_model_capability_flags, +) +from litellm.litellm_core_utils.routing_groups import apply_routing_group_priority +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.responses.bridge_check import keep_registered_mode +from litellm.router_api import RouterRegistry +from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET +from litellm.router_rust.surface.indexes import HostedIndexes +from litellm.router_rust.surface.model_info import HostModelInfo +from litellm.router_rust.surface.model_list import HostedModelList +from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model +from litellm.router_utils.deployment_numbers import refuse_unservable_numbers, refuse_wide_numbers +from litellm.router_utils.endpoint_eligibility import warn_on_unenforced_endpoint_flag +from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import deployment_has_io_token_limits +from litellm.types.router import ( + VALID_LITELLM_ENVIRONMENTS, + CredentialLiteLLMParams, + Deployment, + DeploymentTypedDict, + LiteLLM_Params, + ModelGroupInfo, + RouterModelGroupAliasItem, + RoutingGroup, + RoutingPlugin, + TaggedPreRoutingStrategy, + validate_litellm_params_strictly, +) +from litellm.types.utils import ( + DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS, + CustomPricingLiteLLMParams, + LlmProviders, + LlmProvidersSet, + is_server_derived_pricing_key, + shared_backend_model_info, +) +from litellm.utils import ( + ModelInfoLookupMiss, + catalog_with_runtime_registrations, + get_potential_model_names, + get_secret, + lookup_model_info_for_registration, + model_info_with_pricing_overrides, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router_python_host import PythonHost + from litellm.router_rust.steps.callbacks import OptionalChecks + from litellm.router_rust.steps.prerouting import PreRoutingStrategies + from litellm.router_strategy.budget_limiter import RouterBudgetLimiting + from litellm.rust_bridge._native import RouterCoreHandle + +# The PTU share teams and the team ceilings read so far. +_PTUMemo: TypeAlias = tuple[ + frozenset[str], + dict[tuple[str, str], "PTUTeamCeiling | None"], +] +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +_CACHE_PRICING_FIELDS: Final = ( + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_read_input_token_cost", + "cache_read_input_token_cost_above_200k_tokens", +) +_OFF_PEAK_BASE_RATE_FIELDS: Final = ( + "input_cost_per_token", + "input_cost_per_second", + "cost_per_second", + "tiered_pricing", +) + +# A stored model_list entry: the live dict the router core keeps. +UntypedEntry: TypeAlias = dict[str, object] +_TeamIndex: TypeAlias = dict[tuple[str, str], list[int]] + +# How a build's deployments join router-core's registry (`build::Install`). +BuildInstall: TypeAlias = Literal["replace", "append", "upsert"] +# Runs one router-core build (`RouterCoreHandle.build`): the entries, the path, whether it resolves +# secrets, how it installs, and the registry's answer to each step. ``RustCore`` supplies it. +BuildRunner: TypeAlias = Callable[ + [Sequence[object], str, bool, BuildInstall, Callable[[Mapping[str, object]], object]], None +] + + +@dataclasses.dataclass(slots=True) +class _BuildEntry: + """One ``set_model_list`` entry as its loop holds it (the router core pops ``model_name``, + ``litellm_params`` and ``model_info`` off the entry and keeps the rest), which router-core's + build steps name by index.""" + + extras: UntypedEntry + model_name: str + litellm_params: UntypedEntry + model_info: UntypedEntry + declared_id: str | None + # `organization` is a list, which router-core adds one deployment for each of. + per_organization: bool = False + # A retried entry keeps whether its id was declared twice in the list it came from. + duplicate: bool | None = None + + +@dataclasses.dataclass(slots=True) +class _Build: + """The state of one router-core build, which ``_answer_build_step`` reads and writes.""" + + path: Literal["create", "add"] + # `create`: the entries (a deep copy, which the build consumes), and each one once router-core + # reaches it. + raw: Sequence[UntypedEntry] = () + # Filled as the build reaches each entry. + entries: dict[int, _BuildEntry] = dataclasses.field(default_factory=dict) # mutable-ok: filled in step order + # `add`: the deployment to add, and the entry it is stored as. + deployment: Deployment | None = None + stored: UntypedEntry | None = None + # The deployment the build is on, its provider and its data-source values. + pending: Deployment | None = None + provider: str | None = None + data_source_values: tuple[tuple[int, str, str], ...] = () + # `add`: the deployment it added. + result: Deployment | None = None + # `create`: the error a deployment raised. The Python core's loop stops there, so every later + # step is answered as invalid and the build ends, then this is raised. + failure: Exception | None = None + + +def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: + """The environment-gating rule (``model_info_is_active_for_environment``): a deployment whose + model_info names ``supported_environments`` loads only where LITELLM_ENVIRONMENT is in it.""" + if model_info is None: + return True + supported_environments: Final = model_info.get("supported_environments") + if supported_environments is None: + return True + if not isinstance(supported_environments, list | tuple): + raise ValueError( # noqa: TRY004 # the core raises ValueError here + f"supported_environments must be a list of {VALID_LITELLM_ENVIRONMENTS}. " + f"but set as: {supported_environments} for model_info: {model_info}" + ) + litellm_environment: Final = get_secret_str(secret_name="LITELLM_ENVIRONMENT") + if litellm_environment is None: + raise ValueError("Set 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .env") + if litellm_environment not in VALID_LITELLM_ENVIRONMENTS: + raise ValueError( + f"LITELLM_ENVIRONMENT must be one of {VALID_LITELLM_ENVIRONMENTS}. but set as: {litellm_environment}" + ) + for environment in supported_environments: # pyright: ignore[reportUnknownVariableType] # a configured list + if environment not in VALID_LITELLM_ENVIRONMENTS: + raise ValueError( + f"supported_environments must be one of {VALID_LITELLM_ENVIRONMENTS}. but set as: {environment} " + f"for model_info: {model_info}" + ) + return litellm_environment in supported_environments + + +# -- pricing written into litellm.model_cost ----------------------------------------------------- + + +def _backend_model_info( + host: "PythonHost", backend_model: str, custom_llm_provider: str | None +) -> Mapping[str, object]: + """``get_model_info`` for the backend model; raises like it for an unmapped model.""" + info: Final = host.invoke("get_model_info")(model=backend_model, custom_llm_provider=custom_llm_provider) + return _MODEL_INFO_ADAPTER.validate_python(info) + + +def _backend_cost_entry( + host: "PythonHost", backend_model: str, custom_llm_provider: str | None +) -> Mapping[str, object] | None: + """The backend model's raw ``litellm.model_cost`` entry, or None when it has none.""" + try: + info: Final = _backend_model_info(host, backend_model, custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model + return None + entry: Final[object] = host.dynamic().model_cost.get(str(info.get("key") or "")) + return _MODEL_INFO_ADAPTER.validate_python(entry) if isinstance(entry, dict) else None + + +def _inherited_cache_pricing( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The cache prices a custom-priced entry inherits (``_inherit_builtin_cache_pricing``).""" + inherited_fields: Final = ( + _CACHE_PRICING_FIELDS + if model_info.get("cache_read_input_token_cost") is not None + else (*_CACHE_PRICING_FIELDS, "explicit_cache_read_input_token_cost") + ) + missing_fields: Final = tuple(field for field in inherited_fields if model_info.get(field) is None) + if not missing_fields: + return {} + try: + backend_info: Final = _backend_model_info(host, backend_model, provider) + except Exception: # noqa: BLE001 # an unmapped backend model contributes nothing + return {} + return {field: backend_info[field] for field in missing_fields if backend_info.get(field) is not None} + + +def _inherited_service_tier_pricing( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The service-tier rates a custom-priced entry inherits (``_inherit_builtin_service_tier_pricing``).""" + if ptu_terms(model_info) is not None and is_ptu_cost_attribution_enabled(): + return {} + if all(model_info.get(field) is None for field in ("input_cost_per_token", "output_cost_per_token")): + return {} + backend_entry: Final = _backend_cost_entry(host, backend_model, provider) + if backend_entry is None: + return {} + return { + field: copy.deepcopy(value) + for field, value in backend_entry.items() + if field.endswith(SERVICE_TIER_COST_KEY_SUFFIXES) and model_info.get(field) is None and value is not None + } + + +def _inherited_off_peak_base_rates( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The whole backend entry an off-peak-only entry copies (``_inherit_builtin_base_rates_for_off_peak``).""" + if not model_info.get("off_peak_pricing"): + return {} + if any(model_info.get(field) is not None for field in _OFF_PEAK_BASE_RATE_FIELDS): + return {} + backend_entry: Final = _backend_cost_entry(host, backend_model, provider) + if backend_entry is None: + return {} + return { + field: copy.deepcopy(value) + for field, value in backend_entry.items() + if model_info.get(field) is None and value is not None + } + + +def _inherited_tiered_output_rate( + host: "PythonHost", model_info: Mapping[str, object], backend_model: str, provider: str | None +) -> Mapping[str, object]: + """The flat output rate a tier table without one inherits (``_inherit_builtin_tiered_output_rate``).""" + tiers: Final = model_info.get("tiered_pricing") + if not isinstance(tiers, list) or not tiers: + return {} + if model_info.get("output_cost_per_token") is not None: + return {} + if all(isinstance(tier, dict) and "output_cost_per_token" in tier for tier in tiers): # pyright: ignore[reportUnknownVariableType] # a configured tier table + return {} + try: + backend_info: Final = _backend_model_info(host, backend_model, provider) + except Exception: # noqa: BLE001 # an unmapped backend model contributes nothing + return {} + backend_rate: Final = backend_info.get("output_cost_per_token") + return {"output_cost_per_token": backend_rate} if backend_rate else {} + + +def _bedrock_profile(params: Mapping[str, object], model: str) -> bool: + return params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/") + + +def _search_tool_support(deployment: Mapping[str, object]) -> bool | None: + """A deployment's answer to Codex's client ``tool_search``.""" + from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request + from litellm.utils import ProviderConfigManager + + configured: Final = _model_info_of(deployment).get("supports_search_tool") + if isinstance(configured, bool): + return configured + params: Final = _params_of(deployment) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return None + try: + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=resolved_model, provider=LlmProviders(resolved_provider) + ) + except (litellm_exceptions.BadRequestError, ValueError): + return None + return True if chat_config_builds_anthropic_request(chat_config) else None + + +def cost_map_backend_model(deployment: Deployment) -> str: + """The model a deployment's prices are inherited from (``_cost_map_backend_model``).""" + model_info_base_model: Final = deployment.model_info.base_model + if isinstance(model_info_base_model, str) and model_info_base_model: + return model_info_base_model + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + params_base_model: Final = params.get("base_model") + if isinstance(params_base_model, str) and params_base_model: + return params_base_model + model: Final = deployment.litellm_params.model + invoked_model: Final = params.get("model_id") + if isinstance(invoked_model, str) and invoked_model and _bedrock_profile(params, model): + profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}" + profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock") + if not isinstance(profile_info, ModelInfoLookupMiss): + return profile_model + return model + + +def deployment_model_cost_payload(host: "PythonHost", deployment: Deployment) -> Mapping[str, object]: + """The ``model_info`` a deployment contributes to ``litellm.model_cost`` + (``_deployment_model_cost_payload``): its model_info, its custom pricing, then the inherited rates.""" + declared: Final = dict(_MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True))) + custom_pricing: Final = { + field: value + for field in CustomPricingLiteLLMParams.model_fields + if (value := deployment.litellm_params.get(field)) is not None + } + backend_model: Final = cost_map_backend_model(deployment) + provider: Final = deployment.model_info.pricing_provider or deployment.litellm_params.custom_llm_provider + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + priced: Final = {**declared, **custom_pricing} + profile_pricing: Final[Mapping[str, object]] = ( + _bedrock_profile_pricing(backend_model, priced) + if params.get("model_id") + and backend_model != deployment.litellm_params.model + and _bedrock_profile(params, deployment.litellm_params.model) + else {} + ) + with_profile: Final = {**priced, **profile_pricing} + off_peak: Final = {**with_profile, **_inherited_off_peak_base_rates(host, with_profile, backend_model, provider)} + cached: Final = ( + {**off_peak, **_inherited_cache_pricing(host, off_peak, backend_model, provider)} + if off_peak.get("input_cost_per_token") is not None + else off_peak + ) + tiers: Final = {**cached, **_inherited_service_tier_pricing(host, cached, backend_model, provider)} + return {**tiers, **_inherited_tiered_output_rate(host, tiers, backend_model, provider)} + + +def _bedrock_profile_pricing(backend_model: str, priced: Mapping[str, object]) -> Mapping[str, object]: + profile_info: Final = lookup_model_info_for_registration(model=backend_model, custom_llm_provider="bedrock") + if isinstance(profile_info, ModelInfoLookupMiss): + return {} + explicit_info: Final = _MODEL_INFO_ADAPTER.validate_python(priced) + priced_info: Final = model_info_with_pricing_overrides(profile_info, explicit_info) # pyright: ignore[reportArgumentType] # the explicit model_info, as the core passes it + return {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} + + +def _backend_cost_map_keys(model: str, custom_llm_provider: str | None) -> tuple[str, ...]: + """The ``litellm.model_cost`` keys a deployment's shared backend info is registered under.""" + backend_key: Final = model if custom_llm_provider is None else f"{custom_llm_provider}/{model}" + if "responses/" in backend_key: + return (backend_key, backend_key.replace("responses/", "")) + return (backend_key,) + + +def _catalog_entry_has_token_limit(catalog_entry: Mapping[str, object], field: str) -> bool: + return catalog_entry.get(field) is not None or ( + field == "max_tokens" and catalog_entry.get("max_output_tokens") is not None + ) + + +def _shared_backend_model_info_for_keys( + catalog: Mapping[str, Mapping[str, object]], + backend_keys: tuple[str, ...], + shared_model_info: Mapping[str, object], +) -> Mapping[str, object]: + """``shared_model_info`` without a deployment's token limit the catalog already sets on one of + ``backend_keys`` (``_shared_backend_model_info_for_keys``).""" + catalog_token_limit_fields: Final = frozenset( + field + for field in DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS + if any(_catalog_entry_has_token_limit(catalog.get(key) or {}, field) for key in backend_keys) + ) + return {field: value for field, value in shared_model_info.items() if field not in catalog_token_limit_fields} + + +def _catalog_backend_keys( + model: str, custom_llm_provider: str | None, backend_keys: tuple[str, ...] +) -> tuple[str, ...]: + """Every name the catalog may list the backend model under. The provider is the declared one: + resolving it would reach provider auth (``catalog_backend_keys``).""" + model_provider_prefix: Final = model.partition("/")[0] + catalog_provider: Final = ( + custom_llm_provider + if custom_llm_provider is not None + else model_provider_prefix + if model_provider_prefix in LlmProvidersSet + else None + ) + if catalog_provider is None: + return (*backend_keys, model) + potential_names: Final = get_potential_model_names(model, catalog_provider) + return ( + *backend_keys, + model, + potential_names["split_model"], + potential_names["stripped_model_name"], + potential_names["combined_model_name"], + potential_names["combined_stripped_model_name"], + potential_names["provider_prefixed_model_name"], + potential_names["region_free_combined_model_name"], + ) + + +def register_deployment_in_model_cost( + host: "PythonHost", + *, + model_id: str | None, + model_info: Mapping[str, object], + model: str, + custom_llm_provider: str | None, + pricing_provider: str | None = None, +) -> None: + """Write a deployment's metadata into ``litellm.model_cost`` (``_register_deployment_in_model_cost``): + the whole entry under its id, the schema fields under its backend model key(s).""" + registered: Final = ( + without(model_info, *CustomPricingLiteLLMParams.model_fields) + if classify_strategy_router_model(model) is not None + else dict(model_info) + ) + register_model: Final = host.invoke("register_model") + if model_id is not None: + register_model( + model_cost={model_id: registered}, + persist_across_reloads=False, + warning_display_name=model, + custom_llm_provider=pricing_provider or custom_llm_provider, + add_to_provider_model_sets=False, + ) + backend_keys: Final = _backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) + shared_model_info: Final = _shared_backend_model_info_for_keys( + catalog=catalog_with_runtime_registrations(), + backend_keys=_catalog_backend_keys(model, custom_llm_provider, backend_keys), + shared_model_info=shared_backend_model_info(registered), + ) + register_model( + model_cost={key: keep_registered_mode(key, shared_model_info) for key in backend_keys}, + persist_across_reloads=False, + ) + + +def register_deployment_pricing(host: "PythonHost", deployment: Deployment) -> None: + """Register a deployment's custom and inherited pricing (``_register_deployment_pricing``).""" + register_deployment_in_model_cost( + host, + model_id=deployment.model_info.id, + model_info=deployment_model_cost_payload(host, deployment), + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + pricing_provider=deployment.model_info.pricing_provider, + ) + + +# -- the registry -------------------------------------------------------------------------------- + + +def _model_info_of(entry: Mapping[str, object]) -> Mapping[str, object]: + model_info: Final[object] = entry.get("model_info") + return _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else {} + + +def _params_of(entry: Mapping[str, object]) -> Mapping[str, object]: + params: Final[object] = entry.get("litellm_params") + return _LITELLM_PARAMS_ADAPTER.validate_python(params) if isinstance(params, dict) else {} + + +def _renamed(entry: UntypedEntry, model_name: str) -> UntypedEntry: + return {**entry, "model_name": model_name} + + +def _as_routing_group_row(deployment: Mapping[str, object]) -> UntypedEntry: + """A member deployment under its group's name, without the member's ``access_groups``.""" + model_info: Final = without(_model_info_of(deployment), "access_groups") + return {**deployment, "model_info": model_info} + + +def _ref_env_only(ref: object) -> bool: + return bool((as_str_mapping(ref) or {}).get("env_only")) + + +def _duplicate_of(step: Mapping[str, object], entry: _BuildEntry) -> bool: + """Whether the entry's declared id is declared twice in its list: a retried entry keeps the + answer its first build had.""" + return entry.duplicate if entry.duplicate is not None else step.get("duplicate_id") is True + + +def _index_key(value: object) -> str | None: + """``value`` as a key of the router's name and id indexes: a dict lookup hashes any key and only + a string can match, so another hashable key finds nothing (``None``), and an unhashable one + raises as the lookup raises. The native lookups take strings alone.""" + _ = hash(value) + return value if isinstance(value, str) else None + + +class HostRegistry(HostModelInfo): + """The live model_list and its indexes, kept the way the Python router core keeps them.""" + + def __init__( + self, + host: "PythonHost", + *, + model_group_alias: "dict[str, str | RouterModelGroupAliasItem]", # mutable-ok: the router's live alias map + ignore_invalid_deployments: bool, + on_change: Callable[[], None], + ) -> None: + self._host: Final = host + self._on_change: Final = on_change + self.ignore_invalid_deployments: bool = ignore_invalid_deployments + self._registry_version: int = 0 + self._deployment_index_lock: Final = threading.RLock() + self._core_model_group_alias: dict[str, str | RouterModelGroupAliasItem] = model_group_alias # mutable-ok: the live alias map # fmt: skip + # The live model_list: router-core's deployments read back, every write a changeset. + self._core_model_list: Final[HostedModelList] = HostedModelList(self._live_handle, self._hand_changes) + # The indexes the registry hands out: router-core's, read back, every write a changeset. + self._indexes: Final = HostedIndexes(self._live_handle, self._hand_changes) + self._core_deployment_latency_map: dict[str, float] = {} # mutable-ok: the live latency map + self._access_groups_cache: Mapping[str, Sequence[str]] | None = None + # The routing groups by name; RustCore sets them from its resolved settings. + self._core_routing_groups: dict[str, RoutingGroup] = {} # mutable-ok: the routing_groups attribute + self._routing_group_rows: tuple[UntypedEntry, ...] | None = None + self._provider_unresolved_deployments: tuple[Callable[[], None], ...] = () + # Runs router-core's deployment build; RustCore sets it before the first build. + self.build_runner: BuildRunner | None = None + self.missing_attribute: Callable[[str], AttributeError] | None = None + # Hands router-core a write its build did not make (a removal, a swapped list) as a + # changeset, and reads its current handle; RustCore sets both before the first build. + self.changes_runner: Callable[[Mapping[str, object]], None] | None = None + self.handle_lookup: Callable[[], RouterCoreHandle | None] | None = None + # The pre-routing strategy registries (``registry().plugins``) and the hooks that keep them; + # RustCore installs its ``PreRoutingStrategies`` (litellm/router_rust/steps/prerouting.py). + self._core_auto_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: the live registry + self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self.strategies: PreRoutingStrategies | None = None + # The router's optional pre-call checks (deployment budgets, prompt caching pins); + # RustCore installs its ``OptionalChecks`` (litellm/router_rust/steps/callbacks.py). + self.checks: OptionalChecks | None = None + # The Rust core's resolution for a request (`_core_get_routable_deployments`), set by the core. + self.routable_lookup: ( + Callable[[str, Mapping[str, object], str | None], Sequence[Mapping[str, object]]] | None + ) = None + # A deployment's cached `max_parallel_requests` limit (litellm/router_rust/invoke/slots.py), set by the core. + self.max_parallel_requests_lookup: Callable[[Mapping[str, object], Mapping[str, object]], object] | None = None + # `plugins("routing")`: the router's routing plugins, set by the core. + self._core_routing_plugins: list[RoutingPlugin] = [] # mutable-ok: the Python core hands out its live list + # `zero_cost()`: the per-deployment zero-cost lookup the auth checks fill, emptied with the + # group info caches. + self._zero_cost_cache: dict[str, bool] = {} # mutable-ok: the auth checks write into it + # The routing cache usage reads come from; RustCore sets it to the one its Rust core writes. + self._core_cache: DualCache = DualCache() + self._discovered_model_info_cache: InMemoryCache = InMemoryCache( + max_size_in_memory=1, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + # The PTU share teams and team ceilings, until the model list next changes. + self._ptu_cache: _PTUMemo | None = None + self._cached_get_model_group_info: Callable[[str], ModelGroupInfo | None] = lru_cache( # pyright: ignore[reportIncompatibleMethodOverride] # the per-router cache shadows the uncached method, as the core keeps it + maxsize=DEFAULT_MAX_LRU_CACHE_SIZE + )(self._core_get_model_group_info) + self._core_cached_deployment_model_info: Callable[[str, str], object] = lru_cache( + maxsize=DEFAULT_MAX_LRU_CACHE_SIZE + )(self._core_get_deployment_model_info) + self.cached_deployment_tracks_usage: Callable[[str, str], bool] = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._deployment_tracks_usage + ) + + def __getattr__(self, name: str) -> object: + from litellm.router_rust.core import RustCoreUnsupported + + missing: Final = self.__dict__.get("missing_attribute") + if name == "_core_model_group_alias" and missing is not None: + # `del router.model_group_alias`: every read raises, as on the Python core. + raise missing(name) + raise RustCoreUnsupported(f"registry().{name.removeprefix('_core_')} is not served by the Rust router core") + + # -- building --------------------------------------------------------------------------------- + + def build(self, model_list: Sequence[object] | None) -> None: + """The constructor's registry: ``set_model_list`` and the latency map, as the core builds them.""" + if model_list is None: + return + self._discovered_model_info_cache = InMemoryCache( + max_size_in_memory=max(len(model_list), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + self.set_model_list(model_list) + for entry in model_list: + params = _params_of(_LITELLM_PARAMS_ADAPTER.validate_python(entry)) + if "model" in params: + self._core_deployment_latency_map[str(params["model"])] = 0 + + async def arefresh_model_info(self, client: "AsyncHTTPHandler | None") -> None: + """``_core_arefresh_model_info``: token limits the served OpenAI-compatible deployments + advertise on their models endpoint, read with a few workers over a snapshot of the list.""" + import asyncio + + from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY + + deployments: Final = iter(tuple(self._core_model_list)) + + async def refresh_worker() -> None: + for raw_deployment in deployments: + try: + await self._arefresh_deployment_model_info(raw_deployment, client=client) + except Exception: # noqa: BLE001 # one invalid deployment must not prevent refreshing the others + verbose_router_logger.debug("Could not refresh deployment model info") + + _ = await asyncio.gather(*(refresh_worker() for _ in range(MODEL_INFO_REFRESH_CONCURRENCY))) + self._invalidate_model_group_info_cache() + + async def _arefresh_deployment_model_info( + self, raw_deployment: Mapping[str, object], *, client: "AsyncHTTPHandler | None" + ) -> None: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.llms.openai_like.model_info import ( + MODEL_INFO_DISCOVERY_PROVIDERS, + get_openai_compatible_model_info, + ) + from litellm.types.router import DiscoveredDeploymentModelInfo + from litellm.utils import aresolve_api_key_secret_reference + + deployment: Final = Deployment.model_validate(raw_deployment) + params: Final = LiteLLM_Params.model_validate( + MappingProxyType( + { + **deployment.litellm_params.model_dump(exclude_none=True), + **( + self._core_get_deployment_credentials_with_provider(deployment.model_info.id or "") + or MappingProxyType({}) + ), + } + ) + ) + model, provider, dynamic_api_key, api_base = self._host.invoke("get_llm_provider")( # pyright: ignore[reportAny] # litellm's untyped tuple + model=params.model, litellm_params=params + ) + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return + if api_base is None or "*" in model or params.get("use_clientside_credentials"): # pyright: ignore[reportUnknownMemberType] # the params' untyped get + return + api_key: Final = await aresolve_api_key_secret_reference( + params.api_key or dynamic_api_key, # pyright: ignore[reportAny] # see above + model=model, # pyright: ignore[reportAny] # see above + llm_provider=provider, # pyright: ignore[reportAny] # see above + upstream=MappingProxyType({"api_base": params.api_base}), + ) + headers: Final = TypeAdapter(Mapping[str, str]).validate_python( + params.get("extra_headers") or params.get("headers") or MappingProxyType({}) # pyright: ignore[reportUnknownMemberType] # see above + ) + auth_headers: Final = ( + MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) + ) + limits: Final = await get_openai_compatible_model_info( + model=model, # pyright: ignore[reportAny] # see above + api_base=api_base, # pyright: ignore[reportAny] # see above + headers=MappingProxyType( + {**auth_headers, **MappingProxyType({key.lower(): value for key, value in headers.items()})} + ), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self._core_cache.in_memory_cache, + ) + model_id: Final = deployment.model_info.id + if not limits or model_id is None or self._core_get_model_info(model_id) is not raw_deployment: + return + self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1) + self._discovered_model_info_cache.delete_cache(model_id) + self._discovered_model_info_cache.set_cache( + model_id, + DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits), # pyright: ignore[reportArgumentType] # the model_list entry, as the core keeps it + ) + self._invalidate_model_group_info_cache() + + def _deployment_tracks_usage(self, deployment_id: str, model_group: str) -> bool: + """``_deployment_tracks_usage``: a limit on the deployment or in its merged model info, where + the cost map's own ``tpm`` and ``rpm`` count too.""" + deployment_info: Final = self._core_get_deployment(model_id=deployment_id) + if deployment_info is None: + return False + deployment_model_info: Final = self._core_get_router_model_info( + deployment=deployment_info, + received_model_name=model_group, + ) + configured_limits: Final[tuple[object, ...]] = ( + deployment_info.get("tpm", None), + deployment_info.get("rpm", None), + deployment_info.litellm_params.tpm, + deployment_info.litellm_params.rpm, + deployment_model_info.get("tpm", None), + deployment_model_info.get("rpm", None), + ) + return any(limit is not None for limit in configured_limits) or deployment_has_io_token_limits( + deployment_info.model_dump() + ) + + def _invalidate_model_group_info_cache(self) -> None: + self._ptu_cache = None + self._cached_get_model_group_info.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self._core_cached_deployment_model_info.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self.cached_deployment_tracks_usage.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache + self._zero_cost_cache.clear() + self._routing_group_rows = None + + def set_routing_groups(self, groups: Sequence[RoutingGroup]) -> None: + """``_init_routing_groups``' registry side: the groups by name, and the caches they feed.""" + self._core_routing_groups = {group.group_name: group for group in groups} + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + + def _invalidate_access_groups_cache(self) -> None: + self._access_groups_cache = None + self._registry_version += 1 + + def set_model_list(self, model_list: Sequence[object]) -> None: + """Replace every deployment (``set_model_list``). All or nothing: when an entry raises, + the deployments served before the call are put back and the error is raised.""" + previous: Final = [copy.deepcopy(dict(entry)) for entry in self._core_model_list] + try: + self._replace_model_list(model_list) + except Exception: + self._replace_model_list(previous) + raise + + def _replace_model_list(self, model_list: Sequence[object]) -> None: + self._discovered_model_info_cache.flush_cache() + original_model_list: Final[list[UntypedEntry]] = copy.deepcopy(list(model_list)) # pyright: ignore[reportAssignmentType] # model_list entries # mutable-ok: a deep copy the build consumes + if self.strategies is not None: + self.strategies.reset() + self._provider_unresolved_deployments = () + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + ptu_declared: Final = tuple( + str(entry.get("model_name")) + for entry in original_model_list + if (ptu_info := _model_info_of(entry)).get("db_model") is not True and declares_ptu(ptu_info) + ) + if ptu_declared and not is_ptu_cost_attribution_enabled(): + verbose_router_logger.warning( + "PTU fields are set on config.yaml deployment(s) %s, but PTU cost attribution is disabled, so no " + "flat cost accrues and this traffic is billed per token. Set %s=True to enable it", + ", ".join(ptu_declared), + PTU_COST_ATTRIBUTION_ENV_VAR, + ) + # router-core walks the list (one deployment per organization, secrets resolved first, + # ids assigned), asking this registry about each deployment in turn; it starts the list + # and its id, name and team indexes over, and takes the given names as `model_names`. + self._run_build( + _Build(path="create", raw=original_model_list), + original_model_list, + resolve_secrets=True, + install="replace", + ) + if self.strategies is not None: + self.strategies.finalize() + self._on_change() + + # -- the build's steps (router-core's `build::build`) ------------------------------------------ + + def _run_build( + self, build: _Build, entries: Sequence[object], *, resolve_secrets: bool, install: BuildInstall + ) -> None: + """Have router-core build ``entries``, answering its steps from this registry.""" + if self.build_runner is None: + raise RuntimeError("the host registry builds deployments through its Rust core") + for entry in entries: + refuse_wide_numbers(entry) + self._core_model_list.begin_build(replacing=install == "replace") + try: + self.build_runner(entries, build.path, resolve_secrets, install, partial(self._answer_shown, build)) + finally: + self._core_model_list.end_build() + if build.failure is not None: + # router-core served what it built before the entry that raised (`set_model_list` + # cleared the list first), so the router is left as the Python core leaves it. + raise build.failure + + def _answer_shown(self, build: _Build, step: Mapping[str, object]) -> object: + """A build step's answer; a deployment the ``added`` step hands router-core shows in the + live list from then on (router-core serves it when the build ends).""" + answer: Final = self._answer_build_step(build, step) + stored: Final = as_str_mapping(answer).get("entry") if step["kind"] == "added" else None # pyright: ignore[reportOptionalMemberAccess] # an added step answers a mapping + if isinstance(stored, Mapping): + self._core_model_list.built(cast(Mapping[str, object], stored)) # cast-ok: the entry it stores + return answer + + def _answer_build_step(self, build: _Build, step: Mapping[str, object]) -> object: + """The answer to one of router-core's build steps. A ``set_model_list`` deployment that + fails is left out when the router ignores invalid deployments (``_create_deployment``), + and retried on the next price data reload when its provider did not resolve.""" + if build.failure is not None: + return {"invalid": True} + kind: Final = step["kind"] + index: Final = int(str(step["index"])) + refs: Final = step.get("refs") + if kind == "secrets" and isinstance(refs, list) and not any(_ref_env_only(ref) for ref in refs): + return self._build_secrets(build, index, refs) + if kind == "validate": + # A new deployment: the entry is popped where the loop pops it, outside the error + # handling, and nothing of the last deployment carries over. + build.pending = None # rebind-ok: the build's state, which each step advances + build.provider = None # rebind-ok: the build's state, which each step advances + build.data_source_values = () # rebind-ok: the build's state, which each step advances + if build.path == "create": + _ = self._build_entry(build, index) + try: + if kind == "secrets": + return self._build_data_sources(build, refs if isinstance(refs, list) else []) + if kind == "validate": + return self._build_validate(build, index, step) + if kind == "provider": + build.provider = self._build_provider(build) # rebind-ok: the build's state, which each step advances + return {"provider": build.provider} + return self._build_added(build) + except Exception as e: + if build.path != "create": + raise + if not self.ignore_invalid_deployments: + build.failure = e # rebind-ok: raised once router-core ends the build + return {"invalid": True} + if isinstance(e, litellm_exceptions.BadRequestError): + entry: Final = build.entries[index] + retried: Final = dataclasses.replace(entry, duplicate=_duplicate_of(step, entry)) + self._provider_unresolved_deployments = ( + *self._provider_unresolved_deployments, + partial(self._retry_create, retried), + ) + verbose_router_logger.exception( + "Error creating deployment: %s, ignoring and continuing with other deployments.", e + ) + return {"invalid": True} + + def _build_entry(self, build: _Build, index: int) -> _BuildEntry: + """Entry ``index`` as ``set_model_list``'s loop holds it, popped off the first time a step + names it (so a malformed entry raises where the loop raises).""" + held: Final = build.entries.get(index) + if held is not None: + return held + model: Final = build.raw[index] + model_name: Final = model.pop("model_name") + raw_params: Final = model.pop("litellm_params") + litellm_params: Final[UntypedEntry] = raw_params if isinstance(raw_params, dict) else {} # pyright: ignore[reportAssignmentType] # configured params + model_info: Final = model.pop("model_info", {}) + info_dict: Final[UntypedEntry] = model_info if isinstance(model_info, dict) else {} # pyright: ignore[reportAssignmentType] # a configured model_info + entry: Final = _BuildEntry( + extras=model, + model_name=str(model_name), + litellm_params=litellm_params, + model_info=info_dict, + declared_id=None if info_dict.get("id") is None else str(info_dict["id"]), + per_organization=isinstance(litellm_params.get("organization"), list), + ) + build.entries[index] = entry # rebind-ok: the build's state, which each step advances + return entry + + def _build_secrets(self, build: _Build, index: int, refs: Sequence[object]) -> Sequence[object]: + """``set_model_list``'s ``get_secret`` for each top-level ``os.environ/`` value router-core + found, written into the entry's params.""" + entry: Final = self._build_entry(build, index) + references: Final = tuple(as_str_mapping(ref) or {} for ref in refs) + values: Final = tuple(get_secret(str(reference["name"])) for reference in references) + entry.litellm_params.update( + {str(reference["key"]): value for reference, value in zip(references, values, strict=True)} + ) + return values + + def _build_data_sources(self, build: _Build, refs: Sequence[object]) -> Sequence[object]: + """``_add_deployment``'s environment reads for ``dataSources[].parameters``, applied to the + deployment where it applied them (``_build_added``).""" + references: Final = tuple(as_str_mapping(ref) or {} for ref in refs) + build.data_source_values = tuple( # rebind-ok: the build's state, which each step advances + ( + int(str(reference["source"])), + str(reference["key"]), + os.environ.get(str(reference["name"]).replace("os.environ/", ""), ""), + ) + for reference in references + ) + return tuple(value for _, _, value in build.data_source_values) + + def _build_validate(self, build: _Build, index: int, step: Mapping[str, object]) -> Mapping[str, object]: + if build.path == "add": + return self._validate_added_deployment(build) + entry: Final = self._build_entry(build, index) + routed: Final = as_str_mapping(step["entry"]) or {} + routed_info: Final = as_str_mapping(routed.get("model_info")) or {} + if "id" not in entry.model_info: + entry.model_info["id"] = routed_info.get("id") + if entry.per_organization: + entry.litellm_params["organization"] = (as_str_mapping(routed.get("litellm_params")) or {}).get( + "organization" + ) + duplicate: Final = _duplicate_of(step, entry) + deployment: Final = self._validated_deployment(entry, taken=duplicate) + if deployment is None: + return {"skip": True} + served: Final = with_admin_api_key_secret_reference(deployment) + build.pending = served # rebind-ok: the build's state, which each step advances + # A dump of this core's own: a `mock_response` ModelResponse warns, and only Python's dumps may. + return {"entry": build.pending.to_json(exclude_none=True, warnings=False)} + + def _validated_deployment(self, entry: _BuildEntry, *, taken: bool) -> Deployment | None: + """``_create_deployment`` up to ``_add_deployment``: the config checks, the deployment, its + pricing, and None when the environment gate leaves it out.""" + model_name: Final = entry.model_name + litellm_params: Final = entry.litellm_params + model_info: Final = entry.model_info + config_sourced: Final = model_info.get("db_model") is not True + identity_error: Final = ( + ptu_identity_error( + declared_id=entry.declared_id, + taken=taken, + current_id=model_info.get("id"), # pyright: ignore[reportArgumentType] # the configured id + model_name=model_name, + ) + if config_sourced and ptu_terms(model_info) is not None + else None + ) + ptu_error: Final = ( + (ptu_config_error(model_info, model_name=model_name) or identity_error) if config_sourced else None + ) + if ptu_error is not None and (declares_ptu_shares(model_info) or is_ptu_cost_attribution_enabled()): + raise ValueError(ptu_error) + access_windows_error: Final = access_windows_config_error(model_info, model_name=model_name) + if access_windows_error is not None: + raise ValueError(access_windows_error) + capacity_warning: Final = ( + ptu_capacity_warning( + model_name, MappingProxyType({"model_info": model_info, "litellm_params": litellm_params}) + ) + if is_ptu_cost_attribution_enabled() + else None + ) + if capacity_warning is not None: + verbose_router_logger.warning(capacity_warning) + zeroed_pricing: Final = zeroed_ptu_pricing(model_info, litellm_params) if config_sourced else None + merged_params: Final[Mapping[str, object]] = ( + litellm_params if zeroed_pricing is None else MappingProxyType({**litellm_params, **zeroed_pricing}) + ) + params: Final[LiteLLM_Params] = validate_litellm_params_strictly(LiteLLM_Params, dict(**merged_params)) + declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(litellm_params) + warn_on_provider_credential_mismatch(model_name=model_name, litellm_params=declared_params) + warn_on_ignored_model_capability_flags(model_name=model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag(model_name=model_name, model_info=model_info) + deployment: Final = without_model_capability_flags( + Deployment(**entry.extras, model_name=model_name, litellm_params=params, model_info=model_info) # pyright: ignore[reportArgumentType] # the entry's extra fields, as the core passes them + ) + # The values as typed: a cost written as "inf", a share written as 1e20. + refuse_unservable_numbers(deployment.to_json(exclude_none=True, warnings=False)) + register_deployment_pricing(self._host, deployment) + if model_info_is_active_for_environment(model_info=deployment.model_info) is not True: # pyright: ignore[reportArgumentType] # ModelInfo reads like a mapping + verbose_router_logger.warning( + "Ignoring deployment %s as it is not active for environment %s", + deployment.model_name, + deployment.model_info["supported_environments"], + ) + return None + for pattern in deployment.litellm_params.get("tag_regex") or []: + try: + re.compile(pattern) + except re.error as exc: + raise ValueError( + f"Invalid regex in tag_regex for model '{deployment.model_name}': {pattern!r} — {exc}" + ) from exc + return deployment + + def _validate_added_deployment(self, build: _Build) -> Mapping[str, object]: + """``add_deployment`` up to ``_add_deployment``: an id the registry holds is skipped.""" + deployment: Final = build.deployment + if deployment is None: + raise RuntimeError("an add build holds its deployment") + if deployment.model_info.id and self._core_has_model_id(deployment.model_info.id): + return {"skip": True} + declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) + warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag( + model_name=deployment.model_name, model_info=deployment.model_info.model_dump() + ) + served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) + build.pending = served # rebind-ok: the build's state, which each step advances + build.stored = served.to_json(exclude_none=True) # rebind-ok: the build's state, which each step advances + refuse_unservable_numbers(build.stored) + return {"entry": build.stored} + + def _build_provider(self, build: _Build) -> str | None: + """``_add_deployment``'s provider: None for a prompt-management model, else the provider + ``get_llm_provider`` resolves, refused when litellm does not serve it.""" + deployment: Final = build.pending + if deployment is None: + raise RuntimeError("a provider step follows a validated deployment") + litellm_model: Final = deployment.litellm_params.model + if isinstance(deployment.litellm_params.drop_params, str): + verbose_router_logger.warning( + "model=%s drop_params=%r is not a flag value, treating it as unset", + deployment.model_name, + deployment.litellm_params.drop_params, + ) + dynamic: Final = self._host.dynamic() + if "/" in litellm_model and litellm_model.split("/")[0] in dynamic.known_custom_logger_compatible_callbacks: + return None + if declared_authenticating_provider(litellm_model, deployment.litellm_params.custom_llm_provider) is not None: + _, provider = self._model_group_llm_provider(deployment.litellm_params) + return provider + _, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.get("custom_llm_provider", None), + api_base=deployment.litellm_params.api_base, + litellm_params=deployment.litellm_params, + ) + if ( + custom_llm_provider not in dynamic.provider_list + and not JSONProviderRegistry.exists(custom_llm_provider) + and not is_registered_custom_provider(custom_llm_provider) + ): + raise Exception(f"Unsupported provider - {custom_llm_provider}") # noqa: TRY002 # the core raises a plain Exception + return custom_llm_provider + + def _build_added(self, build: _Build) -> Mapping[str, object]: + """The rest of ``_add_deployment``: the entry router-core stores, which joins the list and + the indexes (``host_index.rs``) when the step returns.""" + deployment: Final = build.pending + if deployment is None: + raise RuntimeError("an added step follows a validated deployment") + self._register_deployment(deployment, build.provider, build.data_source_values) + build.pending = None # rebind-ok: the build's state, which each step advances + build.data_source_values = () # rebind-ok: the build's state, which each step advances + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + if build.path == "create": + return {"entry": deployment.to_json(exclude_none=True)} + entry: Final = build.stored if build.stored is not None else deployment.to_json(exclude_none=True) + register_deployment_pricing(self._host, deployment) + build.result = deployment # rebind-ok: the build's state, which each step advances + return {"entry": entry} + + def _retry_create(self, entry: _BuildEntry) -> None: + """A ``set_model_list`` deployment left out when its provider did not resolve, built again + and added at the end.""" + build: Final = _Build(path="create") + build.entries[0] = entry + self._run_build( + build, + [ + { + **entry.extras, + "model_name": entry.model_name, + "litellm_params": entry.litellm_params, + "model_info": entry.model_info, + } + ], + resolve_secrets=False, + install="append", + ) + + @staticmethod + def _model_group_llm_provider(litellm_params: LiteLLM_Params) -> tuple[str, str]: + try: + return resolve_model_and_provider_for_metadata( + model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider + ) + except litellm_exceptions.BadRequestError as e: + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + return "", "" + + def _register_deployment( + self, + deployment: Deployment, + custom_llm_provider: str | None, + data_source_values: Sequence[tuple[int, str, str]], + ) -> None: + """``_add_deployment`` after the provider: its data-source values, pass-through + credentials and pre-routing strategies (router-core registers its names and patterns).""" + if deployment.litellm_params.rpm is None and getattr(deployment, "rpm", None) is not None: + deployment.litellm_params.rpm = getattr(deployment, "rpm") # noqa: B009 # an extra field of the entry # rebind-ok: the core sets it on the deployment it adds + if deployment.litellm_params.tpm is None and getattr(deployment, "tpm", None) is not None: + deployment.litellm_params.tpm = getattr(deployment, "tpm") # noqa: B009 # an extra field of the entry # rebind-ok: see above + data_sources: Final = deployment.litellm_params.get("dataSources", []) or [] + for source, key, value in data_source_values: + data_sources[source]["parameters"][key] = value + if custom_llm_provider is not None: + self._initialize_deployment_for_pass_through(deployment, custom_llm_provider) + if self.strategies is not None: + self.strategies.register(deployment) + + def _initialize_deployment_for_pass_through(self, deployment: Deployment, custom_llm_provider: str) -> None: + """Register vertex credentials for a pass-through deployment (``_initialize_deployment_for_pass_through``).""" + if deployment.litellm_params.use_in_pass_through is not True or custom_llm_provider != "vertex_ai": + return + credential_name: Final = deployment.litellm_params.litellm_credential_name + credential_values: Final = ( + CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {} + ) + project: Final = credential_values.get("vertex_project") or deployment.litellm_params.vertex_project + location: Final = credential_values.get("vertex_location") or deployment.litellm_params.vertex_location + credentials: Final = credential_values.get("vertex_credentials") or deployment.litellm_params.vertex_credentials + if project is None or location is None: + raise ValueError( + "vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints." + ) + from litellm.router_host import router_host + + router_host().add_vertex_pass_through_credentials( + project_id=str(project), + location=str(location), + vertex_credentials=credentials, # pyright: ignore[reportArgumentType] # the configured credentials + ) + + # -- writes ----------------------------------------------------------------------------------- + + def add_deployment(self, deployment: Deployment) -> Deployment | None: + """Add one deployment unless its id exists (``add_deployment``), built by router-core.""" + build: Final = _Build(path="add", deployment=deployment) + self._run_build(build, [deployment.to_json(exclude_none=True)], resolve_secrets=False, install="upsert") + if build.result is None: + return None + if self.checks is not None: + self.checks.sync_deployment_budget(build.result) + self._on_change() + return build.result + + def _core_upsert_deployment(self, deployment: Deployment) -> Deployment | None: + """Add or replace one deployment (``_upsert_deployment``).""" + model_id: Final = deployment.model_info.id or "" + on_router: Final = self._core_get_deployment(model_id=model_id) + served: Final = without_model_capability_flags(deployment) + try: + if on_router is not None: + if ( + deployment.model_name == on_router.model_name + and served.litellm_params == on_router.litellm_params + and deployment.model_info == on_router.model_info + ): + return None + with self._deployment_index_lock: + removal_idx: Final = self.model_id_to_deployment_index_map.get(model_id) + if removal_idx is not None: + _ = self._core_model_list[removal_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does + self._remove(model_id) + if self.strategies is not None: + self.strategies.release(on_router) + self._on_change() + self.add_deployment(deployment=deployment) + if self.strategies is not None: + self.strategies.upserted(deployment, on_router) + except Exception as e: + if not self.ignore_invalid_deployments: + raise + verbose_router_logger.warning( + "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", + deployment.model_name, + deployment.model_info.id, + e, + ) + self._restore_deployment_after_failed_upsert(on_router, model_id) + return None + else: + return deployment + + def _restore_deployment_after_failed_upsert(self, previous: Deployment | None, model_id: str) -> None: + if previous is None or self._core_has_model_id(model_id): + return + try: + if self.strategies is None: + self.add_deployment(deployment=previous) + else: + self.strategies.without_capability_limit(lambda: self.add_deployment(deployment=previous)) + verbose_router_logger.info( + "Restored deployment %s (id=%s); it keeps serving its previous configuration.", + previous.model_name, + model_id, + ) + except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload + verbose_router_logger.warning( + "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", + previous.model_name, + model_id, + restore_error, + ) + + def _live_handle(self) -> "RouterCoreHandle | None": + return None if self.handle_lookup is None else self.handle_lookup() + + def _hand_changes(self, changes: Mapping[str, object]) -> None: + if self.changes_runner is None: + raise RuntimeError("the host registry writes deployments through its Rust core") + self.changes_runner(changes) + self._core_model_list.refresh() + + def refresh(self) -> None: + """Read router-core's list again after a write (only what was appended, when that is all + that changed). The index views are read again when an entry point hands them out, which + keeps one ``add_deployment`` from reading every index back.""" + self._core_model_list.refresh() + + def _remove(self, model_id: str) -> None: + """Remove the deployment at the id map's position for ``model_id``, every index shifted + (``_update_deployment_indices_after_removal``, in router-core).""" + self._hand_changes({"remove": model_id}) + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() + + # -- the indexes (router-core's, read back as live views) ------------------------------------- + + @property + def model_id_to_deployment_index_map(self) -> dict[str, int]: # mutable-ok: the live id index + self._indexes.refresh() + return cast(dict[str, int], self._indexes.id_map) # cast-ok: id to position, as router-core keeps it + + @property + def _core_model_name_to_deployment_indices(self) -> dict[str, list[int]]: # mutable-ok: the live name index + self._indexes.refresh() + return cast(dict[str, list[int]], self._indexes.model_name) # cast-ok: name to positions + + @_core_model_name_to_deployment_indices.setter + def _core_model_name_to_deployment_indices(self, value: object) -> None: + self._indexes.assign("model_name", value) + + @property + def _core_team_model_to_deployment_indices( + self, + ) -> dict[tuple[str, str], list[int]]: # mutable-ok: the live team index + self._indexes.refresh() + return cast(_TeamIndex, self._indexes.team_model) # cast-ok: (team, public name) to positions + + @_core_team_model_to_deployment_indices.setter + def _core_team_model_to_deployment_indices(self, value: object) -> None: + self._indexes.assign("team_model", value) + + @property + def _core_model_names(self) -> set[str]: # mutable-ok: the live group set + self._indexes.refresh() + return cast(set[str], self._indexes.model_names) # cast-ok: group names + + @_core_model_names.setter + def _core_model_names(self, value: object) -> None: + self._indexes.assign("model_names", value) + + @property + def _core_deployment_names(self) -> list[str]: # mutable-ok: the live name list + self._indexes.refresh() + return cast(list[str], self._indexes.deployment_names) # cast-ok: litellm model names + + @_core_deployment_names.setter + def _core_deployment_names(self, value: object) -> None: + self._indexes.assign("deployment_names", value) + + @property + def _core_team_public_model_names(self) -> frozenset[str]: + self._indexes.refresh() + return self._indexes.team_public_model_names + + @_core_team_public_model_names.setter + def _core_team_public_model_names(self, value: object) -> None: + self._indexes.assign("team_public_model_names", value) + + @property + def _core_provider_default_deployment_ids(self) -> list[str]: # mutable-ok: the live id list + self._indexes.refresh() + return cast(list[str], self._indexes.provider_default_ids) # cast-ok: deployment ids + + @_core_provider_default_deployment_ids.setter + def _core_provider_default_deployment_ids(self, value: object) -> None: + self._indexes.assign("provider_default_ids", value) + + @property + def _core_pattern_router(self) -> PatternMatchRouter: + self._indexes.refresh() + return self._indexes.pattern_router + + @_core_pattern_router.setter + def _core_pattern_router(self, value: object) -> None: + self._indexes.pattern_router.patterns = getattr(value, "patterns", {}) # pyright: ignore[reportAttributeAccessIssue] # a router a caller assigned + + @property + def _core_team_pattern_routers(self) -> dict[str, PatternMatchRouter]: # mutable-ok: the live per-team routers + self._indexes.refresh() + return cast(dict[str, PatternMatchRouter], self._indexes.team_pattern_routers) # cast-ok: team id to its router + + @_core_team_pattern_routers.setter + def _core_team_pattern_routers(self, value: object) -> None: + routers: Final = as_str_mapping(value) or {} + self._indexes.write( + { + "team_patterns": [ + [ + str(team), + [ + [regex, deployments] + for regex, deployments in (as_str_mapping(getattr(router, "patterns", None)) or {}).items() + ], + ] + for team, router in routers.items() + ] + } + ) + + def replace_model_list(self, model_list: "list[UntypedEntry]") -> None: # mutable-ok: the list as assigned + """``router.model_list = [...]``: the list is swapped as given, without reindexing, and + router-core takes it whole.""" + self._core_model_list.replace(model_list) + self._registry_version += 1 + self._on_change() + + def _core_delete_deployment(self, id: str) -> UntypedEntry | None: + """Remove one deployment and return its stored entry (``_delete_deployment``).""" + try: + with self._deployment_index_lock: + deployment_idx: Final = self.model_id_to_deployment_index_map.get(id) + if deployment_idx is None: + return None + item: Final = self._core_model_list[deployment_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does + self._remove(id) + if self.checks is not None: + self.checks.unregister_deployment_budget(id) + if self.strategies is not None: + self.strategies.release_entry(item) + except Exception: # noqa: BLE001 # the core answers None for any failed delete + return None + self._on_change() + return item + + def _core_get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": + """The router's deployment-budget callback (``registry().plugins("budget_limiter")``).""" + return self.checks.budget_limiter() if self.checks is not None else None + + @property + def _core_prompt_caching_pin_cache(self) -> DualCache: + """The prompt caching pins (``registry().plugins("prompt_caching_pins")``).""" + if self.checks is None: + return self.__getattr__("_core_prompt_caching_pin_cache") # pyright: ignore[reportReturnType] # raises RustCoreUnsupported + return self.checks.prompt_caching_pin_cache + + def replay_model_cost_registrations(self) -> None: + """Re-register every served deployment's cost-map entries (``replay_model_cost_registrations``).""" + provider_unresolved: Final = self._provider_unresolved_deployments + self._provider_unresolved_deployments = () + for create_deployment in provider_unresolved: + create_deployment() + for entry in tuple(self._core_model_list): + try: + deployment = Deployment(**entry) # pyright: ignore[reportArgumentType] # a stored entry + except Exception: # noqa: BLE001 # a malformed entry must not abort the rest of the rebuild + verbose_router_logger.exception( + "Router: could not rebuild cost-map entry for a deployment during a price data reload" + ) + continue + register_deployment_pricing(self._host, deployment) + self._invalidate_model_group_info_cache() + if provider_unresolved: + self._on_change() + + # -- one deployment --------------------------------------------------------------------------- + + def _core_has_model_id(self, candidate_id: str) -> bool: + handle: Final = self._live_handle() + key: Final = _index_key(candidate_id) + return handle is not None and key is not None and handle.has_model_id(key) + + def _core_get_model_info(self, id: str) -> UntypedEntry | None: + handle: Final = self._live_handle() + key: Final = _index_key(id) + idx: Final[object] = None if handle is None or key is None else handle.id_position(key) + return None if idx is None else self._core_model_list[idx] # pyright: ignore[reportArgumentType, reportCallIssue] # a position, or what a caller wrote, read as Python reads it + + def _core_get_deployment(self, model_id: str) -> Deployment | None: + entry: Final = self._core_get_model_info(model_id) + return None if entry is None else Deployment(**entry) # pyright: ignore[reportArgumentType] # a stored entry + + def _core_get_deployment_by_model_group_name(self, model_group_name: str) -> Deployment | None: + handle: Final = self._live_handle() + key: Final = _index_key(model_group_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + model: Final = self._core_model_list[indices[0]] if indices else None # pyright: ignore[reportIndexIssue, reportArgumentType, reportCallIssue] # positions, or what a caller wrote, read as Python reads them + return None if model is None else Deployment(**model) # pyright: ignore[reportArgumentType] # a stored entry + + def _at(self, indices: object) -> tuple[UntypedEntry, ...]: + """The entries at ``indices`` (what an index holds), read as Python's list reads them.""" + return tuple(self._core_model_list[idx] for idx in indices) # pyright: ignore[reportGeneralTypeIssues, reportArgumentType, reportCallIssue] # positions, or what a caller wrote + + def _core_deployments_named(self, model_name: str) -> tuple[UntypedEntry, ...] | None: + handle: Final = self._live_handle() + key: Final = _index_key(model_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + return None if indices is None else self._at(indices) + + def team_public_model_names_of(self, team_id: object) -> frozenset[str]: + handle: Final = self._live_handle() + if handle is None or not isinstance(team_id, str): + return frozenset() + return frozenset(handle.team_public_names_of(team_id)) + + def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[UntypedEntry, ...] | None: + handle: Final = self._live_handle() + team: Final = _index_key(team_id) + public: Final = _index_key(public_model_name) + indices: Final[object] = ( + None if handle is None or team is None or public is None else handle.team_positions(team, public) + ) + return None if indices is None else self._at(indices) + + # -- lookups (router-core `lookup.rs`): rows read at the stored entries -------------------- + + def _rows(self, rows: object) -> list[UntypedEntry]: # mutable-ok: a fresh list + """The stored entries router-core's lookup ``rows`` name, read as Python's list reads them: + a row it could not read is indexed here, and raises as the Python router's read raised.""" + entries: Final[list[UntypedEntry]] = [] # mutable-ok: filled per row + for row in cast(Sequence[Mapping[str, object]], rows): # cast-ok: router-core's lookup rows + if "raw" in row: + entries.append(self._core_model_list[row["raw"]]) # pyright: ignore[reportArgumentType, reportCallIssue] # what a caller wrote, read as Python reads it + elif "raw_positions" in row: + entries.extend(self._at(row["raw_positions"])) + elif "pattern" in row: + entries.append({**DeploymentTypedDict(**row["pattern"]), "model_name": row["name"]}) # pyright: ignore[reportCallIssue] # a pattern router's stored entry + else: + entries.append(self._row_entry(row)) + return entries + + def _row_entry(self, row: Mapping[str, object]) -> UntypedEntry: + held: Final[UntypedEntry] = self._core_model_list[int(str(row["at"]))] + name: Final = row.get("name") + named: Final = held if name is None else _renamed(held, str(name)) + group: Final = row.get("group") + if group is None: + return named + return _as_routing_group_row( + apply_routing_group_priority(self._core_routing_groups[str(group)], str(row["member"]), named) # pyright: ignore[reportArgumentType] # a stored entry + ) + + def _raise_at(self, raw: object) -> None: + """Read the list at the row router-core could not read it at, which raises.""" + if raw is not None: + _ = self._rows([raw]) + + def _list_scope(self, model_name: str | None, *, only: str | None = None) -> Mapping[str, object]: + """What ``get_model_list`` lists beyond the deployments: the aliases (read here, where an + alias item that is no mapping or lacks ``hidden`` raises as the Python router's read + does), the callable routing groups and the wildcard requests for ``model_name``.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + aliases: Final[list[Mapping[str, str]]] = [] # mutable-ok: filled per alias + alias_map: Final = self._core_model_group_alias or {} + if model_name is None or model_name in alias_map: + items: Final[Sequence[tuple[str, object]]] = ( + [(model_name, alias_map[model_name])] if model_name is not None else list(alias_map.items()) + ) + for model_alias, model_value in items: + if isinstance(model_value, str): + aliases.append({"alias": model_alias, "target": model_value}) + elif isinstance(model_value, dict): + item = RouterModelGroupAliasItem(**model_value) # pyright: ignore[reportCallIssue, reportArgumentType] # an alias item dict + if item["hidden"] is True and model_name is None: + continue + aliases.append({"alias": model_alias, "target": item["model"]}) + groups: Final = [ + {"name": name, "members": list(group.models)} + for name in (self._core_routing_groups or {}) + if (group := self.get_routing_group(name)) is not None + ] + wildcard: Final = ( + [model_name, f"{inferred_provider(model_name)}/{model_name}"] if model_name is not None else [] + ) + return {"aliases": aliases, "groups": groups, "wildcard": wildcard, "only": only} + + def _lookup(self) -> "RouterCoreHandle": + handle: Final = self._live_handle() + if handle is None: + raise RuntimeError("the host registry's lookups run in its Rust core") + return handle + + # -- deployments by name ---------------------------------------------------------------------- + + @staticmethod + def _core_should_include_deployment( + model_name: str, model: Mapping[str, object], team_id: str | None = None + ) -> bool: + info: Final = _model_info_of(model) + if team_id is not None and info.get("team_id") == team_id and model_name == info.get("team_public_model_name"): + return True + if model_name is not None and model.get("model_name") == model_name: # pyright: ignore[reportUnnecessaryComparison] # the core checks it + model_team_id: Final = info.get("team_id") + return team_id is None or model_team_id is None or model_team_id == team_id + return False + + @staticmethod + def _core_deployment_usable_by_team(model: Mapping[str, object] | Deployment, team_id: str | None) -> bool: + info: Final[object] = ( + model.model_info.model_dump() if isinstance(model, Deployment) else model.get("model_info") + ) + owner: Final = _MODEL_INFO_ADAPTER.validate_python(info).get("team_id") if isinstance(info, Mapping) else None + return team_may_use_deployment(owner, team_id) + + def _core_deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[Mapping[str, object]]: + """The deployments ``model`` names for this caller: alias, then team-first, then global, then + admin-across-teams (``_core_deployments_for_request``).""" + from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, is_proxy_admin_request + + registered: Final = self._core_get_model_from_alias(model=model) or model + team_id: Final = get_request_team_id(request_kwargs) + deployments: Final = self._core_get_all_deployments(model_name=registered, team_id=team_id) + if deployments or team_id is not None or not is_proxy_admin_request(request_kwargs): + return deployments + return self._team_deployments_across_teams(registered) + + def _team_deployments_across_teams(self, model: str) -> list[UntypedEntry]: # mutable-ok: a fresh list + """Every team's deployments under public name ``model``, for a proxy admin calling without a team.""" + rows, teams = self._lookup().lookup_across_teams(model) + team_deployments: Final = self._rows(rows) + if teams > 1: + raise litellm_exceptions.BadRequestError( + message=( + f"Model name '{model}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=model, + llm_provider="", + ) + return team_deployments + + def _core_get_all_deployments( + self, model_name: str, model_alias: str | None = None, team_id: str | None = None + ) -> list[UntypedEntry]: # mutable-ok: a fresh list, as the core returns it + key: Final = _index_key(model_name) + return [] if key is None else self._rows(self._lookup().lookup_all(key, team_id, model_alias)) + + def get_routing_group(self, model_name: str) -> RoutingGroup | None: + """The routing group callable as ``model_name``: a deployment model_name or an alias of the + same name shadows it.""" + if not self._core_routing_groups: + return None + group: Final = self._core_routing_groups.get(model_name) + handle: Final = self._live_handle() + if ( + group is None + or (handle is not None and handle.named_positions(model_name) is not None) + or model_name in (self._core_model_group_alias or {}) + ): + return None + return group + + def _core_routing_group_has_alternatives(self, model_group: str | None) -> bool: + """A callable routing group whose member union spans more than one deployment.""" + if model_group is None: + return False + resolved: Final = self._core_get_model_from_alias(model=model_group) or model_group + group: Final = self.get_routing_group(resolved) + if group is None: + return False + handle: Final = self._lookup() + + def held(member: str) -> Sequence[object]: + return cast(Sequence[object], handle.named_positions(member) or ()) # cast-ok: positions, or junk + + return sum(len(held(member)) for member in group.models) > 1 + + def _core_get_model_list_from_model_alias( + self, model_name: str | None = None + ) -> list[UntypedEntry]: # mutable-ok: a fresh list + return self._rows( + self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="aliases")) + ) + + def _core_get_model_list_from_routing_groups(self, model_name: str | None = None) -> Sequence[UntypedEntry]: + """Callable routing groups as model-list rows: each member deployment under the group's name.""" + if model_name is None and self._routing_group_rows is not None: + return self._routing_group_rows + rows: Final = tuple( + self._rows(self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="groups"))) + ) + if model_name is None: + self._routing_group_rows = rows + return rows + + def _core_get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> list[UntypedEntry] | None: # mutable-ok: a fresh list, as get_model_list returns + if model_name is not None and _index_key(model_name) is None: + # No index holds it, so `_get_wildcard_deployments` reads it: what the pattern routers raise. + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + return [ + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), + *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), + ] + return self._rows(self._lookup().lookup_model_list(model_name, team_id, self._list_scope(model_name))) + + def _core_wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model reads the base routers + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_owned: Final = team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + return team_owned or self._core_pattern_router.get_deployments_by_pattern(model=model) + rows: Final = self._lookup().lookup_wildcard_route( + model, team_id, [model, f"{inferred_provider(model)}/{model}"] + ) + routed: Final = cast(Sequence[Mapping[str, object]], rows) # cast-ok: router-core's wildcard rows + return [cast(Mapping[str, object], row["pattern"]) for row in routed] # cast-ok: each carries its routed copy + + def _core_wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_owned: Final = ( + team_router.get_models_by_pattern(model=model) if team_router is not None else frozenset[str]() + ) + return team_owned or self._core_pattern_router.get_models_by_pattern(model=model) + + def _core_config_deployments(self) -> Iterator[Mapping[str, object]]: + """A generator, as the Python core's: the rows are read when iteration starts.""" + yield from self._rows(self._lookup().lookup_config()) + + # -- names ------------------------------------------------------------------------------------ + + def _core_get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: a fresh list + names, raw = self._lookup().lookup_model_names(team_id, self._list_scope(None)) + self._raise_at(raw) + return names + + def _core_get_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> list[str]: # mutable-ok: a fresh list + if model_name is not None and _index_key(model_name) is None: + return [] + ids, raw = self._lookup().lookup_model_ids(model_name, exclude_team_models) + self._raise_at(raw) + return ids + + # -- resolving a name ------------------------------------------------------------------------- + + def _core_get_model_from_alias(self, model: str) -> str | None: + return resolve_model_group_alias(self._core_model_group_alias, model) + + def _core_routable_model_group(self, model_name: str) -> str: + target: Final = self._core_get_model_from_alias(model_name) + return target if target is not None else model_name + + def _core_is_recognized_model(self, model: str) -> bool: + handle: Final = self._live_handle() + key: Final = _index_key(model) + return ( + (handle is not None and key is not None and handle.has_model_name(key)) + or self._core_has_model_id(model) + or model in self._core_model_group_alias + or self.get_routing_group(model) is not None + ) + + def _core_get_model_group(self, id: str) -> list[UntypedEntry] | None: # mutable-ok: a fresh list + model_info: Final = self._core_get_model_info(id=id) + if model_info is None: + return None + return self._core_get_model_list(model_name=str(model_info["model_name"])) + + def _core_resolve_model_name_from_model_id( + self, model_id: str | None, custom_llm_provider: str | None = None + ) -> str | None: + if not model_id: + return None + handle: Final = self._lookup() + if handle.has_model_name(model_id): + return model_id + if self._core_has_model_id(model_id): + deployment: Final = self._core_get_deployment(model_id=model_id) + if deployment is not None and deployment.model_name: + return deployment.model_name + return model_id + listed, serving, raw = handle.lookup_name_serving(model_id, self._list_scope(None)) + self._raise_at(raw) + if not listed: + return None + if serving is not None: + return serving + if self._core_is_recognized_model(model_id): + return model_id + if custom_llm_provider is None: + return None + if self._core_pattern_router.routes_to_provider(model_id, custom_llm_provider): + return model_id + provider_prefixed: Final = f"{custom_llm_provider}/{model_id}" + if not model_id.startswith(f"{custom_llm_provider}/") and self._core_pattern_router.routes_to_provider( + provider_prefixed, custom_llm_provider + ): + return provider_prefixed + return None + + def _get_deployment_by_litellm_model(self, model: object) -> list[UntypedEntry]: # mutable-ok: a fresh list + handle: Final = self._live_handle() + if handle is None or not isinstance(model, str): + return [] + return list(self._at(handle.litellm_model_positions(model))) + + # -- credentials ------------------------------------------------------------------------------ + + @staticmethod + def _core_is_deployment_blocked(deployment: Deployment) -> bool: + model_info: Final[object] = getattr(deployment, "model_info", None) + return model_info is not None and getattr(model_info, "blocked", None) is True + + def _core_get_deployment_credentials(self, model_id: str) -> UntypedEntry | None: + deployment: Final = self._core_get_deployment(model_id=model_id) + if deployment is None or self._core_is_deployment_blocked(deployment): + return None + return CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + + def _get_model_group_deployment_usable_by_team( + self, model_group_name: str, team_id: str | None + ) -> Deployment | None: + if _index_key(model_group_name) is None: + return None + position, raw = self._lookup().lookup_group_usable_by_team(model_group_name, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + + def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None: + if team_id is None or _index_key(model_id) is None: + return None + position, raw = self._lookup().lookup_team_public_name(model_id, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + + def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_models: Final = team_router.route(model_id) if team_router else None + global_models: Final = tuple( + model + for model in (self._core_pattern_router.route(model_id) or ()) + if self._core_deployment_usable_by_team(model, team_id) + ) + candidates: Final = team_models or global_models + if not candidates: + return None + return Deployment(**candidates[0]) # a pattern router holds stored entry dicts + + def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None: + deployment: Final = ( + self._core_get_deployment(model_id=model_id) + or self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id) + or self._get_team_public_name_deployment(model_id=model_id, team_id=team_id) + or self._get_wildcard_deployment_usable_by_team(model_id=model_id, team_id=team_id) + ) + if deployment is None or self._core_is_deployment_blocked(deployment): + return None + return deployment + + def _core_get_deployment_credentials_with_provider( + self, model_id: str, team_id: str | None = None + ) -> UntypedEntry | None: + deployment: Final = self._core_get_credential_deployment(model_id=model_id, team_id=team_id) + if deployment is None: + return None + base: Final = CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) + credential_name: Final = deployment.litellm_params.litellm_credential_name + resolved: Final[UntypedEntry] = dict(base) + if credential_name is not None: + credential_values: Final = CredentialAccessor.get_credential_values(credential_name) + if not credential_values: + verbose_router_logger.warning("Credential '%s' not found in credential_list", credential_name) + resolved.update(credential_values) + _ = resolved.pop("litellm_credential_name", None) + model: Final = deployment.litellm_params.model + resolved["model"] = model + resolved["custom_llm_provider"] = ( + deployment.litellm_params.custom_llm_provider or (model.split("/")[0] if "/" in model else None) or "openai" + ) + return resolved + + # -- routable names --------------------------------------------------------------------------- + + def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[UntypedEntry, ...]: + return tuple(self._rows(self._lookup().lookup_routable(self._core_routable_model_group(model_name), team_id))) + + def _core_team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, + else None (``team_ptu_ceiling``), kept until the model list next changes.""" + from litellm.litellm_core_utils.ptu_shares import ptu_share_teams, team_ptu_ceiling + + listed: Final = tuple(self._core_get_model_list() or ()) + if self._ptu_cache is None: + self._ptu_cache = (ptu_share_teams(listed), {}) + share_teams, ceilings = self._ptu_cache + if team_id not in share_teams: + return None + key: Final = (team_id, model_group) + if key not in ceilings: + ceilings[key] = team_ptu_ceiling( + listed, + self._core_model_list, + team_id, + model_group, + lambda name: self._core_wildcard_route_deployments(name, team_id), + ) + return ceilings[key] + + def _get_client( + self, deployment: Mapping[str, object], kwargs: Mapping[str, object], client_type: str | None = None + ) -> object: + """The SDK client cached in the routing cache for a deployment (``_get_client``); a pod + discovery deployment has none. ``max_parallel_requests`` is the deployment's slot limit, + built on first use.""" + from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs + from litellm.router_rust.core import RustCoreUnsupported + + # `deployment["model_info"]["id"]`: the Python router reads the id first, raising for a + # deployment without one whatever the client type. + _ = deployment["model_info"]["id"] # pyright: ignore[reportIndexIssue] # read as the Python router reads it + if client_type == "max_parallel_requests": + limit_lookup: Final = self.max_parallel_requests_lookup + if limit_lookup is None: + raise RustCoreUnsupported( + "registry().client(client_type='max_parallel_requests') is not served by the Rust router core" + ) + return limit_lookup(deployment, kwargs) + if _params_of(deployment).get("kubernetes_pod_discovery") is True: + return None + model_id: Final = _model_info_of(deployment).get("id") + span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) + stream: Final = "_stream" if kwargs.get("stream") is True else "" + with service_target(ROUTER_CLIENTS_TARGET): + if client_type == "async": + return self._core_cache.get_cache( + key=f"{model_id}{stream}_async_client", local_only=True, parent_otel_span=span + ) # pyright: ignore[reportUnknownMemberType] # DualCache's reads are untyped + return self._core_cache.get_cache(key=f"{model_id}{stream}_client", parent_otel_span=span) # pyright: ignore[reportUnknownMemberType] # see above + + def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + """Per routable deployment: its ``supports_search_tool`` when a bool, else True when its route + builds an Anthropic Messages request, else None (``get_search_tool_support``).""" + return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id)) + + def _core_get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: + """The deployment ids that could serve ``model`` for ``team_id``: the alias target, then a + routing group, then a team public name or wildcard route for a name that is no model + group, then the model group (``get_candidate_model_ids_for_route``). Read-only.""" + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + resolved: Final = self._core_get_model_from_alias(model=model) or model + group: Final = self.get_routing_group(resolved) if self._core_routing_groups else None + ids, raw = self._lookup().lookup_candidate_ids( + resolved, + team_id, + None if group is None else list(group.models), + [resolved, f"{inferred_provider(resolved)}/{resolved}"], + ) + self._raise_at(raw) + return frozenset(ids) + + def _core_team_model_has_alternatives(self, deployment_id: str) -> bool: + """Whether a team deployment's public name has another unpaused deployment of that team.""" + found, raw = self._lookup().lookup_team_model_has_alternatives(deployment_id) + self._raise_at(raw) + return found + + def _core_resolve_served_model_name(self, model_name: str, served_model: str) -> str: + """``served_model`` when ``model_name`` routes to it (the alias target, a routing group + member, or the name itself), else ``model_name``.""" + target: Final = self._core_get_model_from_alias(model_name) or model_name + group: Final = self.get_routing_group(target) + members: Final = group.models if group is not None else () + return served_model if served_model == target or served_model in members else model_name + + def _core_resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: + """The provider model strings a call to ``model_name`` can be served by (``resolved_litellm_models``).""" + listed: Final = self._core_get_model_list(model_name=model_name, team_id=team_id) or () + if use_base_model: + return tuple( + backend for entry in listed if (backend := cost_map_backend_model(Deployment.model_validate(entry))) + ) + return tuple(model for entry in listed if isinstance(model := _params_of(entry).get("model"), str) and model) + + def _core_get_routable_deployments( + self, model: str, request_kwargs: Mapping[str, object] = MappingProxyType({}), call_type: str | None = None + ) -> tuple[Mapping[str, object], ...]: + """The deployments routing would consider for ``request_kwargs``, as the Rust core resolves + them, the endpoint of ``call_type`` included.""" + lookup: Final = self.routable_lookup + if lookup is None: + from litellm.router_rust.core import RustCoreUnsupported + + raise RustCoreUnsupported("registry().get_routable_deployments is not served by the Rust router core") + return tuple(lookup(model, request_kwargs, call_type)) + + def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: + deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None) + if deployment is None: + return None + model: Final = _params_of(deployment).get("model") + return None if model is None else str(model) + + def _core_map_team_model( + self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None + ) -> str | None: + """A team's public name to its group, or a deployment for a model-less call (``map_team_model``).""" + models: Final = self._core_get_model_list(model_name=team_model_name, team_id=team_id) + if not models: + return None + if team_model_name is None and custom_llm_provider is not None: + return self._team_model_for_provider(models, team_id, custom_llm_provider) + for model in models: + info = _model_info_of(model) + if info.get("team_id") == team_id: + if team_model_name is None: + public = info.get("team_public_model_name") or model.get("model_name") + return None if public is None else str(public) + return team_model_name + return None + + @staticmethod + def _team_model_for_provider(models: Sequence[UntypedEntry], team_id: str, custom_llm_provider: str) -> str | None: + def public_name(deployment: UntypedEntry) -> str | None: + info: Final = _model_info_of(deployment) + if info.get("team_id") != team_id: + return None + name: Final = info.get("team_public_model_name") + return name if isinstance(name, str) and name else str(deployment["model_name"]) + + def provider(deployment: UntypedEntry) -> object: + return provider_for_generic_call(_params_of(deployment)) # pyright: ignore[reportArgumentType] # the deployment's params + + matching: Final = next( + (m for m in models if public_name(m) is not None and provider(m) == custom_llm_provider), None + ) + if matching is None: + return None + name: Final = public_name(matching) + if name is None: + return None + if all(provider(m) == custom_llm_provider for m in models if public_name(m) == name): + return name + matched_id: Final = _model_info_of(matching).get("id") + return matched_id if isinstance(matched_id, str) else None + + # -- access groups ---------------------------------------------------------------------------- + + def _core_get_model_access_groups( + self, model_name: str | None = None, model_access_group: str | None = None, team_id: str | None = None + ) -> Mapping[str, Sequence[str]]: + use_cache: Final = model_name is None and model_access_group is None and team_id is None + if use_cache and self._access_groups_cache is not None: + return self._access_groups_cache + pairs, raw = self._lookup().lookup_access_groups( + model_name, model_access_group, team_id, self._list_scope(model_name) + ) + self._raise_at(raw) + groups: Final[dict[str, list[str]]] = {g: list(n) for g, n in pairs} # mutable-ok: as a defaultdict + if use_cache: + self._access_groups_cache = groups + return groups + + def _core_is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: + groups: Final = self._core_get_model_access_groups(model_access_group=model_access_group) + return any( + self._core_pattern_router.route(request=model) is not None for model in groups.get(model_access_group, []) + ) + + +# -- the registry() view ------------------------------------------------------------------------- + + +class RustRegistry(RouterRegistry): + """``registry()`` of the Rust core: ``RouterRegistry``'s queries over the host registry.""" + + __slots__ = () + + def includes( + self, + deployment: "DeploymentTypedDict | dict[str, object]", # mutable-ok: the RouterRegistry signature + *, + model_name: str | None = None, + team_id: str | None = None, + ) -> bool: + hosted: Final = self._router + if not isinstance(hosted, HostRegistry): + return super().includes(deployment, model_name=model_name, team_id=team_id) + if model_name is None: + return HostRegistry._core_deployment_usable_by_team(deployment, team_id) # pyright: ignore[reportPrivateUsage, reportArgumentType] # the view reads its own registry + return HostRegistry._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) # pyright: ignore[reportPrivateUsage, reportArgumentType] # see above + + +def hosted_registry_view(hosted: HostRegistry) -> RouterRegistry: + """The ``registry()`` view over ``hosted``.""" + return RustRegistry(hosted) # pyright: ignore[reportArgumentType] # the host registry answers every _core_ query the view makes + + +# The registry indexes a deprecated attribute write replaces (``router.model_names = ...`` reaches +# the host registry as ``_core_model_names``), by the field router-core reads each one as. +INDEX_ATTRIBUTES: Final[Mapping[str, str]] = MappingProxyType( + { + "_core_model_names": "model_names", + "_core_model_name_to_deployment_indices": "model_name", + "_core_team_model_to_deployment_indices": "team_model", + } +) + + +def _positions(indices: object, length: int) -> list[int]: # mutable-ok: a JSON list + """The model-list positions ``indices`` read, as Python's ``model_list[index]`` reads them: a + negative one counts from the end, and one before the start becomes ``length``, a position past + the end, so the lookup raises ``IndexError`` as Python's does.""" + if not isinstance(indices, Iterable) or isinstance(indices, str | bytes): + return [] + return [ + index if index >= 0 else (index + length if index + length >= 0 else length) + for index in indices + if isinstance(index, int) and not isinstance(index, bool) + ] + + +def index_document(hosted: HostRegistry, written: Iterable[str]) -> Mapping[str, object]: + """The written registry indexes, as the host registry holds them now, in the shape router-core + reads (``IndexOverrides``): names a lookup finds, and the deployment positions each model name + (or ``(team_id, public name)`` pair) indexes. Entries Python could not read as such are left out.""" + document: Final[dict[str, object]] = {} # mutable-ok: built here + length: Final = len(hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # positions read against the live list + for attribute in written: + value: object = getattr(hosted, attribute, None) + field: str = INDEX_ATTRIBUTES[attribute] + if field == "model_names": + names = value if isinstance(value, Iterable) and not isinstance(value, str | bytes | Mapping) else () + document[field] = sorted({name for name in names if isinstance(name, str)}) + elif field == "model_name": + entries = value if isinstance(value, Mapping) else {} + document[field] = { + name: _positions(indices, length) for name, indices in entries.items() if isinstance(name, str) + } + else: + pairs = value if isinstance(value, Mapping) else {} + document[field] = [ + [key[0], key[1], _positions(indices, length)] + for key, indices in pairs.items() + if isinstance(key, tuple) and len(key) == 2 and all(isinstance(part, str) for part in key) + ] + return document + + +DEFAULT_DEPLOYMENT_FAILING_ID: Final = "__litellm_default_deployment_failing_call__" +"""The id router-core serves a ``default_deployment`` under when every call to it fails.""" + +DEFAULT_DEPLOYMENT_COPIED_ID: Final = "__litellm_default_deployment_copied__" +"""The id router-core serves a callable ``default_deployment`` under when its ``model_info.id`` is +not a string: the host picks and calls Python's copy of it, so its id stays what was assigned.""" + + +def default_deployment_copy(value: object, model: str) -> object: + """``_try_early_resolve_deployments_for_model_not_in_names``' copy of the default deployment + for ``model``, raising what it raises for a value it cannot copy.""" + updated: Final = value.copy() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # any assigned value, read as Python reads it + updated["litellm_params"] = value["litellm_params"].copy() # pyright: ignore[reportIndexIssue, reportUnknownMemberType] # same + updated["litellm_params"]["model"] = model # pyright: ignore[reportUnknownMemberType] # same + return updated # pyright: ignore[reportUnknownVariableType] # same + + +def default_deployment_copy_error(value: object, model: str) -> BaseException: + """The error the default deployment's copy raises for ``model``.""" + try: + _ = default_deployment_copy(value, model) + except Exception as raised: # noqa: BLE001 # the error the pick raises + return raised + return RuntimeError(f"router-core: the default deployment copies for {model!r}") + + +def default_deployment_call_error(value: object) -> BaseException | None: + """The error every call to the default deployment raises, from its first reads + (``_update_kwargs_with_deployment``'s ``deployment["model_name"]``, ``_get_client``'s + ``deployment["model_info"]["id"]``); None when they read.""" + try: + served: Final = default_deployment_copy(value, "") + _ = served["model_name"] # pyright: ignore[reportIndexIssue] # read as Python reads it + _ = served["model_info"]["id"] # pyright: ignore[reportIndexIssue] # same + except Exception as raised: # noqa: BLE001 # the error the call raises + return raised + return None + + +def served_default_deployment(value: object) -> tuple[Mapping[str, object] | None, bool]: + """The ``default_deployment`` as router-core serves it, and whether resolving to it raises + (a value Python cannot copy). A deployment every call to fails is served under + ``DEFAULT_DEPLOYMENT_FAILING_ID``, one whose id is not a string under + ``DEFAULT_DEPLOYMENT_COPIED_ID``; names that are not strings are served as text.""" + if value is None: + return None, False + try: + copied: Final = default_deployment_copy(value, "") + except Exception: # noqa: BLE001 # any copy error: the pick raises it + return None, True + entry: Final = as_str_mapping(copied) or {} + model_info: Final = dict(as_str_mapping(entry.get("model_info")) or {}) + failing: Final = default_deployment_call_error(value) is not None + assigned_id: Final = model_info.get("id") + model_info["id"] = ( + DEFAULT_DEPLOYMENT_FAILING_ID + if failing + else assigned_id + if isinstance(assigned_id, str) + else DEFAULT_DEPLOYMENT_COPIED_ID + ) + name: Final = entry.get("model_name") + return { + **entry, + "model_name": name if isinstance(name, str) else str(name or ""), + "litellm_params": dict(as_str_mapping(entry.get("litellm_params")) or {}), + "model_info": model_info, + }, False diff --git a/litellm/router_rust/surface/replay.py b/litellm/router_rust/surface/replay.py new file mode 100644 index 00000000000..6c770e07883 --- /dev/null +++ b/litellm/router_rust/surface/replay.py @@ -0,0 +1,30 @@ +"""The Rust core's live routers, whose deployments a price data reload re-registers in +``litellm.model_cost`` until they retire (``_live_routers`` / ``_replay_live_router_model_cost``).""" + +from __future__ import annotations + +import weakref +from typing import TYPE_CHECKING, Final + +from litellm.utils import add_live_deployment_replay + +if TYPE_CHECKING: + from litellm.router_rust.surface.registry import HostRegistry + +_live_registries: Final[weakref.WeakSet[HostRegistry]] = weakref.WeakSet() + + +def track_live_registry(registry: HostRegistry) -> None: + _live_registries.add(registry) + + +def untrack_live_registry(registry: HostRegistry) -> None: + _live_registries.discard(registry) + + +def _replay_live_registries() -> None: + for registry in tuple(_live_registries): + registry.replay_model_cost_registrations() + + +add_live_deployment_replay(_replay_live_registries) diff --git a/litellm/router_rust/surface/retries.py b/litellm/router_rust/surface/retries.py new file mode 100644 index 00000000000..4a9c6429ffa --- /dev/null +++ b/litellm/router_rust/surface/retries.py @@ -0,0 +1,295 @@ +""" +The retry members ``RustCore`` serves: ``should_retry_this_error`` and +``get_num_retries_from_retry_policy``. router-core decides both; this module only describes the +Python exception the way an attempt failure is described and reads the answer back. +""" + +import dataclasses +import operator +from collections.abc import Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.types.router import RetryPolicy + +# The most retries a request's `num_retries` asks router-core for (its count is a u32). +_MOST_RETRIES: Final = 2**32 - 1 + +if TYPE_CHECKING: + from litellm.router_rust.core import Resolved + from litellm.rust_bridge._native import ( # pyright: ignore[reportPrivateUsage] # the stub's handle + RouterCoreHandle, + ) + + +def resolved_retry_policy(value: object) -> RetryPolicy | None: + """The router's ``retry_policy`` as the Python router's constructor keeps it: a dict is read + into a ``RetryPolicy``, a ``RetryPolicy`` is kept, anything else is no policy.""" + if isinstance(value, RetryPolicy): + return value + if isinstance(value, Mapping): + fields: Final[Mapping[str, int | None]] = value # pyright: ignore[reportUnknownVariableType] # the policy's fields + return RetryPolicy(**fields) # pyright: ignore[reportArgumentType] # validated by the model + return None + + +def _is_policy(entry: object) -> bool: + return isinstance(entry, RetryPolicy | Mapping) + + +def reads_as_policy(entry: object) -> bool: + """Whether ``resolve_retry_policy`` reads ``entry`` without raising: no entry, a ``RetryPolicy``, + or a mapping ``RetryPolicy(**entry)`` accepts.""" + if entry is None or isinstance(entry, RetryPolicy): + return True + if not isinstance(entry, Mapping): + return False + try: + _ = RetryPolicy(**entry) # pyright: ignore[reportUnknownArgumentType] # validated by the model + except Exception: # noqa: BLE001 # any error the lookup would raise + return False + return True + + +def is_policy_per_group(policies: object) -> bool: + """Whether ``model_group_retry_policy`` is what router-core's settings hold: a mapping of + group names to policies (or nothing).""" + if policies is None: + return True + if not isinstance(policies, Mapping): + return False + entries: Final[Mapping[object, object]] = policies # pyright: ignore[reportUnknownVariableType] # read as objects + return all(isinstance(group, str) and _is_policy(entry) for group, entry in entries.items()) + + +def readable_retry_policy(value: object) -> RetryPolicy | None: + """A router ``retry_policy`` written as a plain attribute, as router-core's settings hold it. + The attribute keeps what was written; a value the lookup raises for reaches router-core as + ``retry_policy_raises`` instead.""" + return resolved_retry_policy(value) if reads_as_policy(value) else None + + +@dataclasses.dataclass(frozen=True, slots=True) +class GroupPolicies: + """A ``model_group_retry_policy`` as router-core reads it, resolved the way + ``resolve_retry_policy`` reads it for each group the call may evaluate. + + ``document`` holds a group's policy where the lookup finds one, and an empty policy (which + grants nothing) where it finds ``None``; a group it does not find is absent, so the router's + ``retry_policy`` applies. ``raising`` names the groups whose lookup raises (an ``in``/``[]`` + the value does not support, or an entry that is not a policy, whose fields + ``get_num_retries_from_retry_policy`` then fails to read): the Python retry loop raises that + error in place of the attempt's on a hop's first failure. + """ + + document: Mapping[str, object] + raising: frozenset[str] + + +def _looked_up(policies: object, group: str) -> tuple[bool, object]: + """``(found, entry)`` as ``resolve_retry_policy`` looks ``group`` up; raises as it raises.""" + found: Final = group in policies # pyright: ignore[reportOperatorIssue] # any value, as the Python router reads it + return (True, policies[group]) if found else (False, None) # pyright: ignore[reportIndexIssue, reportUnknownVariableType] # see above + + +def _policy_document(entry: object) -> object: + """A readable entry as router-core reads it: no entry grants nothing, a mapping is read the way + ``RetryPolicy(**entry)`` reads it.""" + if entry is None: + return {} + if isinstance(entry, Mapping): + return RetryPolicy(**entry) # pyright: ignore[reportUnknownArgumentType] # checked by reads_as_policy + return entry + + +def group_policies(policies: object, groups: Iterable[str]) -> GroupPolicies: + """``policies`` resolved for ``groups`` (a mapping is read by its own keys).""" + if policies is None: + return GroupPolicies({}, raising=frozenset()) + if isinstance(policies, Mapping): + entries: Final[Mapping[object, object]] = policies # pyright: ignore[reportUnknownVariableType] # read as objects + # A key that is no string never matches a group name. + named: Final = {group: entry for group, entry in entries.items() if isinstance(group, str)} + readable: Final = {group: entry for group, entry in named.items() if reads_as_policy(entry)} + return GroupPolicies( + {group: _policy_document(entry) for group, entry in readable.items()}, + raising=frozenset(named) - frozenset(readable), + ) + document: Final[dict[str, object]] = {} # mutable-ok: built here; router-core reads it as plain JSON + raising: Final[set[str]] = set() # mutable-ok: the groups whose lookup raises + for group in groups: + try: + found, entry = _looked_up(policies, group) + except Exception: # noqa: BLE001 # the Python retry loop raises it on a hop's first failure + raising.add(group) + continue + if not found: + continue + if not reads_as_policy(entry): + raising.add(group) + continue + document[group] = _policy_document(entry) + return GroupPolicies(document, raising=frozenset(raising)) + + +def static_group_policies(policies: object) -> object: + """The router's ``model_group_retry_policy`` as router-core's static settings hold it. A value + that is not a mapping of policies is resolved per request instead (``group_policies``).""" + return group_policies(policies, ()).document if isinstance(policies, Mapping) else None # pyright: ignore[reportUnknownArgumentType] # read as objects + + +def raising_groups(request: Mapping[str, object], resolved: GroupPolicies) -> Sequence[str]: + """The groups whose ``model_group_retry_policy`` lookup raises on a hop's first failure, unless + the retry loop never reads it: the request opted out with ``num_retries=0``. A streamed Anthropic + messages call is no exception: only the stream's own retry rules read such an entry as no policy + (``_anthropic_messages_resolved_retry_policy``); a failure before the stream opened reaches the + retry loop.""" + if not resolved.raising or request.get("num_retries") == 0: + return () + return sorted(resolved.raising) + + +def policy_lookup_error(original: BaseException, model_group: str, retry_policy: object, policies: object) -> Exception: + """The error ``get_num_retries_from_retry_policy`` raises for ``model_group`` after ``original`` + failed (the Python retry loop raises it in the attempt's place, chained to it); a lookup that + unexpectedly reads is reported as such.""" + from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy + + traceback: Final = original.__traceback__ + try: + # Looked up while `original` is being handled, as in the retry loop's `except` block, so + # the error chains to it the way Python chains it. + try: + raise original + except BaseException: # noqa: BLE001 # re-raised only to be handled here + _ = get_num_retries_from_retry_policy( + exception=original, # pyright: ignore[reportArgumentType] # any exception the hop raised + retry_policy=retry_policy, # pyright: ignore[reportArgumentType] # the router's resolved policy + model_group=model_group, + model_group_retry_policy=policies, # pyright: ignore[reportArgumentType] # read as the Python router reads it + ) + except Exception as raised: # noqa: BLE001 # the error the hop ends with + return raised + finally: + original.__traceback__ = traceback # rebind-ok: put back as the hop left it + return RuntimeError(f"the model_group_retry_policy lookup for {model_group!r} did not raise") + + +def router_policy_retries( + exception: Exception, retry_policy: object, policies: object, model_group: str | None +) -> int | None: + """``get_num_retries_from_retry_policy`` for a router ``model_group_retry_policy`` that is not + a mapping of policies, read as the Python router reads it (raising as it raises).""" + from litellm.litellm_core_utils.get_retry_from_policy import get_num_retries_from_retry_policy + + return get_num_retries_from_retry_policy( + exception=exception, + retry_policy=retry_policy, # pyright: ignore[reportArgumentType] # the router's resolved policy + model_group=model_group, + model_group_retry_policy=policies, # pyright: ignore[reportArgumentType] # read as the Python router reads it + ) + + +def request_retry_fields(value: object) -> Mapping[str, object]: + """A request's ``num_retries`` as router-core reads it. ``async_function_with_retries`` never + casts it: it compares it with 0, and counts with ``range`` only once it is positive.""" + if value is None: + return {} + if isinstance(value, int): + count: Final = int(value) + return {"num_retries": min(count, _MOST_RETRIES)} if count >= 0 else {"uncounted_retries": "negative"} + try: + if value == 0: + return {"num_retries": 0} + if not value > 0: # pyright: ignore[reportOperatorIssue] # any value, compared as the retry loop compares it + return {"uncounted_retries": "not_positive"} + return {"num_retries": min(operator.index(value), _MOST_RETRIES)} # pyright: ignore[reportArgumentType] # see above + except Exception: # noqa: BLE001 # the retry loop raises it on a hop's first failure + return {"uncounted_retries": "raises"} + + +def retry_count_error(original: BaseException, value: object) -> Exception: + """The error the retry loop raises reading a request's ``num_retries`` after ``original`` + failed, chained to it; a value that unexpectedly reads is reported as such.""" + traceback: Final = original.__traceback__ + try: + try: + raise original + except BaseException: # noqa: BLE001 # re-raised only to be handled here + if value > 0: # pyright: ignore[reportOperatorIssue] # `if num_retries > 0:` + _ = range(value) # pyright: ignore[reportArgumentType] # `for current_attempt in range(num_retries):` + except Exception as raised: # noqa: BLE001 # the error the hop ends with + return raised + finally: + original.__traceback__ = traceback # rebind-ok: put back as the hop left it + return RuntimeError(f"the request's num_retries {value!r} did not raise") + + +def _count(deployments: object) -> int: + return len(deployments) if isinstance(deployments, list) else 0 # pyright: ignore[reportUnknownArgumentType] # only the length is read + + +def _filled(fallbacks: object) -> bool: + return isinstance(fallbacks, Sequence) and len(fallbacks) > 0 # pyright: ignore[reportUnknownArgumentType] # only the length is read + + +class RustRetries: + """Retry members read from the Rust core's handle.""" + + _handle: "RouterCoreHandle" + _resolved: "Resolved" + + @property + def _written_retry_policy(self) -> object: + """The ``retry_policy`` attribute as written (``Resolved`` holds what router-core reads).""" + raise NotImplementedError + + def should_retry_this_error( + self, + error: Exception, + healthy_deployments: list[object] | None = None, # mutable-ok: the Python router's signature + all_deployments: list[object] | None = None, # mutable-ok: same + context_window_fallbacks: list[object] | None = None, # mutable-ok: same + content_policy_fallbacks: list[object] | None = None, # mutable-ok: same + regular_fallbacks: list[object] | None = None, # mutable-ok: same + ) -> bool: + """``True`` when the error is retried in the same group; otherwise the error is raised.""" + from litellm.router_rust.core import attempt_failure + + retried: Final = self._handle.should_retry( + dict(attempt_failure(error)), + (_count(healthy_deployments), _count(all_deployments)), + (context_window_fallbacks is not None, content_policy_fallbacks is not None, _filled(regular_fallbacks)), + ) + if not retried: + raise error + return True + + def get_num_retries_from_retry_policy(self, exception: Exception, model_group: str | None = None) -> int | None: + from litellm.router_rust.core import attempt_failure + + resolved: Final = self._resolved + written: Final = self._written_retry_policy + if not is_policy_per_group(resolved.model_group_retry_policy) or not reads_as_policy(written): + return router_policy_retries(exception, written, resolved.model_group_retry_policy, model_group) + # A dict lookup hashes any key and only a string can match; the native call takes strings alone. + _ = hash(model_group) + return self._handle.retry_policy_retries( + dict(attempt_failure(exception)), model_group if isinstance(model_group, str) else None + ) + + +__all__ = [ + "GroupPolicies", + "RustRetries", + "group_policies", + "is_policy_per_group", + "policy_lookup_error", + "raising_groups", + "readable_retry_policy", + "reads_as_policy", + "request_retry_fields", + "resolved_retry_policy", + "retry_count_error", + "router_policy_retries", + "static_group_policies", +] diff --git a/litellm/router_rust/surface/settings.py b/litellm/router_rust/surface/settings.py new file mode 100644 index 00000000000..c8635593de5 --- /dev/null +++ b/litellm/router_rust/surface/settings.py @@ -0,0 +1,236 @@ +""" +The Rust core's settings semantics on the host side: how ``ChangeSet.settings`` is read (the rules +of ``update_settings``), the ``settings().as_dict()`` view (``get_settings``), the router attributes +the Python core keeps as plain attributes, deleted settings, and the construction-time side effects +of the logging arguments. + +The rules are the Python router core's, ported (never imported). Settings change only through a +changeset; a direct attribute write is turned into one by ``RustCore.__setattr__``. +""" + +import logging +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger +from litellm.constants import RUNTIME_UPDATABLE_ROUTER_SETTINGS +from litellm.litellm_core_utils.routing_groups import validate_routing_strategy +from litellm.types.router import RetryPolicy, RoutingStrategy + +if TYPE_CHECKING: + from litellm.router_config import RouterConfig + from litellm.router_python_host import PythonHost + +MISSING: Final = object() + +# `update_settings` casts these with `int(value)`. +_INT_SETTINGS: Final = frozenset({"timeout", "num_retries", "retry_after", "allowed_fails", "cooldown_time"}) +CONTINUATION_SETTING: Final = "enable_mid_stream_fallback_continuation" + +# Settings the Python core keeps as plain router attributes and `update_settings` sets as given, +# with their constructor defaults. ``max_retries`` has none: it exists once a changeset sets it. +PLAIN_DEFAULTS: Final[Mapping[str, object]] = MappingProxyType( + { + "retry_policy": None, + "model_group_retry_policy": {}, + "enable_weighted_failover": False, + "fallback_on_unsupported_params": True, + } +) +# `get_settings`: the settings it lists, in its order (`routing_groups` comes last). +_VIEW_ORDER: Final = ( + "routing_strategy_args", + "routing_strategy", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "model_group_retry_policy", + "retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", + "enable_tag_filtering", + "tag_routing_prefix", + "fallback_on_unsupported_params", +) + +# Router attributes that are their constructor argument as given, else this default. Arguments +# the Rust core declines are only ever at their default here. +_ARGUMENT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType( + { + "set_verbose": False, + "debug_level": "INFO", + "client_ttl": 3600, + "stream_timeout": None, + "assistants_config": None, + "ignore_invalid_deployments": False, + "alerting_config": None, + "auto_router_capability_limit": None, + "deployment_affinity_ttl_seconds": 3600, + "session_affinity_repin_on_failover": False, + "model_group_affinity_config": None, + } +) +# Class-level and constant attributes of the Python core. +_CONSTANT_ATTRIBUTES: Final[Mapping[str, object]] = MappingProxyType( + {"default_cache_time_seconds": 1 * 60 * 60, "tenacity": None, "access_groups": None} +) + + +@dataclass(frozen=True) +class SettingsUpdate: + """What a ``ChangeSet.settings`` mapping changes, read with ``update_settings``' rules.""" + + values: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) # fields and plain settings + plain: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) # plain attributes, as stored + continuation: object = MISSING # the continuation flag, when given + routing_groups: object = MISSING # the routing groups input, when given + optional_pre_call_checks: object = MISSING # the optional pre-call checks, when given + failure: Exception | None = None # what stopped the reading; the settings before it stay applied + + +def read_settings_update(settings: Mapping[str, object], routing_strategy: object) -> SettingsUpdate: + """``_core_update_settings``' reading of each setting, in order: names that are not + runtime-updatable are ignored, the integer settings are cast with ``int``, a retry policy dict + becomes a ``RetryPolicy`` (any other type but ``None`` is dropped), and a strategy other than + ``routing_strategy``, the router's, is validated. The loop has no rollback: a setting that + raises ends it, and the ones before it are applied.""" + values: Final[dict[str, object]] = {} # mutable-ok: built here, then frozen into the update + plain: Final[dict[str, object]] = {} # mutable-ok: same + continuation: object = MISSING # rebind-ok: set when the flag is given + routing_groups: object = MISSING # rebind-ok: set when routing groups are given + optional_checks: object = MISSING # rebind-ok: set when the optional pre-call checks are given + failure: Exception | None = None # rebind-ok: set when a setting raises + try: + for name, value in settings.items(): + if name not in RUNTIME_UPDATABLE_ROUTER_SETTINGS: + verbose_router_logger.debug("Setting %s is not allowed", name) + elif name in _INT_SETTINGS: + values[name] = int(value) # pyright: ignore[reportArgumentType] # `int(kwargs[var])`, as given: a bad value raises int's own error + elif name == "routing_groups": + routing_groups = value + elif name == "optional_pre_call_checks": + # `set_optional_pre_call_checks` reads a list it is given as a set. + _ = frozenset[object](value) if value is not None else None # pyright: ignore[reportArgumentType] # as given: a bad value raises frozenset's own error + optional_checks = value + elif name == CONTINUATION_SETTING: + continuation = value + elif name == "retry_policy": + policy = RetryPolicy.model_validate(value) if isinstance(value, dict) else value + if policy is None or isinstance(policy, RetryPolicy): + plain[name] = policy + elif name in PLAIN_DEFAULTS or name == "max_retries": + plain[name] = value + else: + if name == "routing_strategy" and _normalized(value) != routing_strategy: + validate_routing_strategy(value) # pyright: ignore[reportArgumentType] # validated as given + values[name] = value + except Exception as raised: # noqa: BLE001 # raised by `apply` once the settings before it are applied + failure = raised # rebind-ok: None until a setting fails + return SettingsUpdate( + values=MappingProxyType(values), + plain=MappingProxyType(plain), + continuation=continuation, + routing_groups=routing_groups, + optional_pre_call_checks=optional_checks, + failure=failure, + ) + + +def _normalized(strategy: object) -> object: + """``_normalize_strategy``, which leaves anything but the enum as given.""" + return strategy.value if isinstance(strategy, RoutingStrategy) else strategy + + +def initial_plain(config: "RouterConfig") -> dict[str, object]: # mutable-ok: the core's own plain attributes + """The plain attributes as the constructor sets them: a ``retry_policy`` dict is read into a + ``RetryPolicy`` and any other type but a policy is no policy.""" + from litellm.router_rust.surface.retries import resolved_retry_policy + + plain: Final = {name: config.get(name, default) for name, default in PLAIN_DEFAULTS.items()} + plain["retry_policy"] = resolved_retry_policy(plain["retry_policy"]) + return plain + + +def settings_view( + values: Mapping[str, object], plain: Mapping[str, object], deleted: frozenset[str] +) -> dict[str, object]: # mutable-ok: the settings view is built fresh per call, as `get_settings` builds it + """``get_settings``: the listed settings the router has, then the routing groups. ``values`` + holds the settings fields and plain attributes the Rust core resolves.""" + view: Final[dict[str, object]] = {} # mutable-ok: see above + for name in _VIEW_ORDER: + if name in deleted: + continue + if name in plain: + view[name] = plain[name] + elif name in values: + view[name] = values[name] + view["routing_groups"] = values.get("routing_groups", []) + return view + + +def argument_attribute(config: "RouterConfig", name: str) -> object: + """A router attribute that is its constructor argument (or a constant), else ``MISSING``.""" + if name in _ARGUMENT_ATTRIBUTES: + return config.get(name, _ARGUMENT_ATTRIBUTES[name]) + if name in _CONSTANT_ATTRIBUTES: + return _CONSTANT_ATTRIBUTES[name] + return MISSING + + +def default_litellm_params(config: "RouterConfig") -> dict[str, object]: # mutable-ok: the router's own dict + """``default_litellm_params`` as the constructor completes it: the router timeout, no client + retries and the caching groups.""" + given: Final = _as_dict(config.get("default_litellm_params")) or {} + params: Final[dict[str, object]] = dict(given) # mutable-ok: see above + params.setdefault("timeout", config.get("timeout")) + params.setdefault("max_retries", 0) + metadata: Final = _as_dict(params.setdefault("metadata", {})) + if metadata is not None: + params["metadata"] = {**metadata, "caching_groups": config.get("caching_groups")} + return params + + +def _as_dict(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed + + +def request_timeout(config: "RouterConfig") -> float | None: + """The per-attempt ``request_timeout``: only stored when a router timeout is also set.""" + from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout + + return get_configured_request_timeout() if config.get("timeout") is not None else None + + +def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None: + """What the constructor's logging arguments do: the host stops its debug info, and + ``set_verbose`` sets the router logger to ``debug_level``.""" + host.suppress_debug_info() + if config.get("set_verbose", False) is True: + level: Final = config.get("debug_level", "INFO") + if level == "INFO": + verbose_router_logger.setLevel(logging.INFO) + elif level == "DEBUG": + verbose_router_logger.setLevel(logging.DEBUG) + + +__all__ = [ + "CONTINUATION_SETTING", + "MISSING", + "PLAIN_DEFAULTS", + "SettingsUpdate", + "apply_construction_effects", + "argument_attribute", + "default_litellm_params", + "initial_plain", + "read_settings_update", + "request_timeout", + "settings_view", +] diff --git a/litellm/router_strategy/adaptive_router/signals.py b/litellm/router_strategy/adaptive_router/signals.py index 72e8d27d2bf..c7fffd05632 100644 --- a/litellm/router_strategy/adaptive_router/signals.py +++ b/litellm/router_strategy/adaptive_router/signals.py @@ -198,7 +198,7 @@ def _detect_loop(history: list[str], new_calls: Sequence[Mapping[str, object]]) return False -_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504} +_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504, 529} _EXHAUSTION_KEYWORDS: Final = ( "context length", diff --git a/litellm/router_strategy/auto_router/litellm_encoder.py b/litellm/router_strategy/auto_router/litellm_encoder.py index 743a0b97fd2..a402f9c8ecf 100644 --- a/litellm/router_strategy/auto_router/litellm_encoder.py +++ b/litellm/router_strategy/auto_router/litellm_encoder.py @@ -30,7 +30,7 @@ class CustomDenseEncoder(DenseEncoder): litellm_router_instance: ClassVar = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx`` @renamed_router_param("litellm_router_instance") - def __init__(self, ctx: Optional["RoutingContext"] = None, **kwargs): + def __init__(self, ctx: Optional["RoutingContext"] = None, **kwargs) -> None: if "ctx" in kwargs: ctx = kwargs.pop("ctx") @@ -55,7 +55,7 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin): model_name: str, score_threshold: float | None = None, max_input_chars: int = 0, - ): + ) -> None: """Initialize the LiteLLMEncoder. :param ctx: The routing context of the owning router. diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 1d502dd8d95..98dd39b40e3 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -36,7 +36,12 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once +from litellm.litellm_core_utils.batch_utils import ( + ChargeOnceJob, + defers_batch_cost, + job_charged_once_marker_key, + job_to_charge_once, +) from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) @@ -52,7 +57,7 @@ from litellm.types.utils import BudgetConfig as GenericBudgetInfo DEFAULT_REDIS_SYNC_INTERVAL: Final = 1 _CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object]) -_BATCH_CHARGE_VALUES: Final = TypeAdapter(tuple[object, float]) +_JOB_CHARGE_VALUES: Final = TypeAdapter(tuple[object, float]) class _LiteLLMParamsDictView: @@ -545,13 +550,16 @@ class RouterBudgetLimiting(CustomLogger): if standard_logging_payload is None: raise ValueError("standard_logging_payload is required") - batch_response, response_cost = _BATCH_CHARGE_VALUES.validate_python( + polled_response, response_cost = _JOB_CHARGE_VALUES.validate_python( (response_obj, _CALLBACK_FIELDS.validate_python(standard_logging_payload).get("response_cost", 0)) ) model_id: Final[str] = str(standard_logging_payload.get("model_id", "")) custom_llm_provider: Final[str | None] = standard_logging_payload.get("custom_llm_provider") - batch_id: Final = batch_id_to_charge_once(callback_fields.get("call_type"), batch_response, response_cost) + if defers_batch_cost(callback_fields, polled_response): + return + + job: Final = job_to_charge_once(callback_fields, polled_response, response_cost) budget_config: Final = ( self._get_budget_config_for_provider(custom_llm_provider) if custom_llm_provider is not None else None @@ -565,7 +573,7 @@ class RouterBudgetLimiting(CustomLogger): spend_key=spend_key, start_time_key=start_time_key, response_cost=response_cost, - batch_id=batch_id, + job=job, ) deployment_budget_config: Final = self._get_budget_config_for_deployment(model_id) @@ -578,7 +586,7 @@ class RouterBudgetLimiting(CustomLogger): spend_key=deployment_spend_key, start_time_key=deployment_start_time_key, response_cost=response_cost, - batch_id=batch_id, + job=job, ) request_tags: Final = get_tags_from_request_kwargs( @@ -596,7 +604,7 @@ class RouterBudgetLimiting(CustomLogger): spend_key=_tag_spend_key, start_time_key=_tag_start_time_key, response_cost=response_cost, - batch_id=batch_id, + job=job, ) async def _increment_spend_for_key( @@ -605,14 +613,21 @@ class RouterBudgetLimiting(CustomLogger): spend_key: str, start_time_key: str, response_cost: float, - batch_id: str | None = None, + job: ChargeOnceJob | None = None, ): if budget_config.budget_duration is None: return current_time: Final = self._clock() new_window_seconds: Final = budget_window_seconds(current_time, budget_config.budget_duration) - if batch_id is not None and not await self._claim_batch_charge(spend_key, batch_id, new_window_seconds): + if job is not None and not await self._claim_job_charge(spend_key, job, new_window_seconds): + if job.repeat_poll_cost > 0: + await self._increment_spend_for_key( + budget_config=budget_config, + spend_key=spend_key, + start_time_key=start_time_key, + response_cost=job.repeat_poll_cost, + ) return budget_start: Final = await self._get_or_set_budget_start_time( @@ -639,8 +654,8 @@ class RouterBudgetLimiting(CustomLogger): verbose_router_logger.debug("Incremented spend for %s by %s", spend_key, response_cost) - async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool: - marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id) + async def _claim_job_charge(self, spend_key: str, job: ChargeOnceJob, ttl_seconds: int) -> bool: + marker_key: Final = job_charged_once_marker_key(spend_key=spend_key, job=job) redis_cache: Final = self.dual_cache.redis_cache if redis_cache is not None: polls: Final = await redis_cache.async_increment(key=marker_key, value=1, ttl=ttl_seconds, refresh_ttl=True) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4f4ed76c170..05dd4706b62 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1219,9 +1219,11 @@ def _with_classifier_forecast( forecasted_decision: Final[StandardLoggingRoutingDecision] = { **with_input, "classifier_probabilities": outcome.jev_verdict.probabilities, - "classifier_confidence": outcome.jev_verdict.confidence, } - return forecasted_decision + confidence: Final = outcome.jev_verdict.confidence + return ( + forecasted_decision if confidence is None else {**forecasted_decision, "classifier_confidence": confidence} + ) if outcome.llm_v2_forecast is not None: return _with_llm_v2_forecast(with_input, outcome.llm_v2_forecast) forecast: Final = outcome.capability_forecast @@ -2468,7 +2470,7 @@ class ComplexityRouter(CustomLogger): score=None, signals=( f"{config.provider}-classifier:{tier_name}", - f"{config.provider}-confidence={answer.confidence:.6f}", + *(() if answer.confidence is None else (f"{config.provider}-confidence={answer.confidence:.6f}",)), *( f"tier-probability:{label}={probability:.6f}" for label, probability in answer.probabilities.items() diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index bb7d2e91cd3..5cd4f00a941 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -210,7 +210,7 @@ class JevChoiceAnswer(LiteLLMBaseModel): type: Literal["choice"] choice: str probabilities: Mapping[str, JevProbability] - confidence: JevProbability + confidence: JevProbability | None = None class JevUsage(LiteLLMBaseModel): @@ -425,7 +425,7 @@ class HttpJevClassifierClient: class JevVerdict(NamedTuple): label: str probabilities: Mapping[str, float] - confidence: float + confidence: float | None model: str cost: float | None provider: str = "typesafe" diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 8e0c3d6a7e7..22159079f98 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -1,6 +1,11 @@ +import contextlib import logging -from collections.abc import Mapping, Sequence -from typing import Final +import threading +import time +import uuid +from collections.abc import Callable, Generator, Mapping, Sequence +from contextvars import ContextVar +from typing import Final, Protocol, TypeAlias from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -8,11 +13,31 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import log_redis_failure from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.types.utils import CallTypes IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60 +SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0 +SELECTION_RESERVATION_MAX_KEYS: Final = 10_000 +_RESERVATIONS_LOCK: Final = threading.Lock() +_PICK_RESERVES: Final[ContextVar[bool]] = ContextVar("least_busy_pick_reserves") +_PICK_OWNER: Final[ContextVar[str | None]] = ContextVar("least_busy_pick_owner", default=None) + + +@contextlib.contextmanager +def picks_reserve(reserve: bool) -> Generator[None, None, None]: + token: Final = _PICK_RESERVES.set(reserve) + try: + yield + finally: + _PICK_RESERVES.reset(token) + + +def pick_reserves(default: bool) -> bool: + return _PICK_RESERVES.get(default) class _ModelInfo(TypedDict, total=False): @@ -21,10 +46,12 @@ class _ModelInfo(TypedDict, total=False): class _Metadata(TypedDict, total=False): model_group: ReadOnly[str | None] + model_info: ReadOnly[_ModelInfo | None] class _LitellmParams(TypedDict, total=False): metadata: ReadOnly[_Metadata | None] + litellm_metadata: ReadOnly[_Metadata | None] model_info: ReadOnly[_ModelInfo | None] @@ -32,6 +59,10 @@ class _CallKwargs(TypedDict, total=False): litellm_params: ReadOnly[_LitellmParams | None] +class CallIdentity(TypedDict, total=False): + litellm_call_id: ReadOnly[object] + + class _DeploymentModelInfo(TypedDict): id: ReadOnly[str | int] @@ -41,38 +72,85 @@ class _Deployment(TypedDict): _CALL_KWARGS: Final = TypeAdapter(_CallKwargs) +CALL_IDENTITY: Final = TypeAdapter(CallIdentity) _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment]) -_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None) +_MEMORY_COUNTS: Final[TypeAdapter[tuple[float | None, ...] | None]] = TypeAdapter(tuple[float | None, ...] | None) +_ATTEMPT_TIMES: Final[TypeAdapter[tuple[float, ...] | None]] = TypeAdapter(tuple[float, ...] | None) +_Pick: TypeAlias = tuple[float, str | None] +_PICKS: Final[TypeAdapter[tuple[_Pick, ...] | None]] = TypeAdapter(tuple[_Pick, ...] | None) +_PICKED_DEPLOYMENTS: Final[TypeAdapter[tuple[str, ...] | None]] = TypeAdapter(tuple[str, ...] | None) def _request_count_key(model_group: str, deployment_id: str) -> str: return f"{model_group}_request_count:{deployment_id}" +def _model_group(metadata: _Metadata | None) -> str | None: + return metadata.get("model_group") if metadata else None + + +def _deployment_id(model_info: _ModelInfo | None) -> str | int | None: + return model_info.get("id") if model_info else None + + +def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None: + return _deployment_id(metadata.get("model_info")) if metadata else None + + def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None: - if is_batch_retrieve_call_type(kwargs.get("call_type")): + call_type: Final = kwargs.get("call_type") + if is_batch_retrieve_call_type(call_type) or call_type == CallTypes.arealtime.value: return None try: call: Final = _CALL_KWARGS.validate_python(kwargs) except ValidationError: return None - litellm_params: Final = call.get("litellm_params") - metadata: Final = litellm_params.get("metadata") if litellm_params else None - model_info: Final = litellm_params.get("model_info") if litellm_params else None - model_group: Final = metadata.get("model_group") if metadata else None - deployment_id: Final = model_info.get("id") if model_info else None + litellm_params: Final = call.get("litellm_params") or _LitellmParams() + routing_metadata: Final = (litellm_params.get("litellm_metadata"), litellm_params.get("metadata")) + model_group: Final = next((group for group in map(_model_group, routing_metadata) if group is not None), None) + deployment_id: Final = next( + ( + deployment_id + for deployment_id in ( + _deployment_id(litellm_params.get("model_info")), + *map(_routed_deployment_id, routing_metadata), + ) + if deployment_id is not None + ), + None, + ) if model_group is None or deployment_id is None: return None return model_group, str(deployment_id) -def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: +def _reservation_key(deployment_id: str) -> str: + return f"least_busy_reservations:{deployment_id}" + + +def _started_attempts_key(call_id: str, ref: tuple[str, str]) -> str: + return f"least_busy_started:{call_id}:{_request_count_key(*ref)}" + + +def _picked_deployments_key(call_id: str) -> str: + return f"least_busy_picked:{call_id}" + + +def call_id_of(kwargs: Mapping[str, object]) -> str | None: + call_id: Final = kwargs.get("litellm_call_id") + return call_id if isinstance(call_id, str) else None + + +def _deployment_ids(healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]: return tuple( - _request_count_key(model_group, str(deployment["model_info"]["id"])) - for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) + str(deployment["model_info"]["id"]) for deployment in _DEPLOYMENTS.validate_python(healthy_deployments) ) +def _request_count_keys(model_group: str, deployment_ids: tuple[str, ...]) -> tuple[str, ...]: + return tuple(_request_count_key(model_group, deployment_id) for deployment_id in deployment_ids) + + def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]: return tuple(0 if value is None else int(value) for value in values) @@ -84,12 +162,21 @@ def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]: return _as_counts(values) -def _least_busy( - healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...] -) -> Mapping[str, object] | None: - if not healthy_deployments: - return None - return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])] +def _unexpired(picks: Sequence[_Pick], now: float) -> tuple[_Pick, ...]: + return tuple(pick for pick in picks if now - pick[0] < SELECTION_RESERVATION_TTL_SECONDS) + + +def _without_one_pick(picks: tuple[_Pick, ...], call_id: str | None) -> tuple[_Pick, ...]: + tags: Final = tuple(tag for _, tag in picks) + owner: Final = next((tag for tag in (call_id, _PICK_OWNER.get()) if tag is not None and tag in tags), None) + if owner not in tags: + return picks + index: Final = tags.index(owner) + return (*picks[:index], *picks[index + 1 :]) + + +def _least_busy_index(counts: tuple[int, ...]) -> int: + return min(range(len(counts)), key=lambda index: counts[index]) def _warn_unreadable(model_group: str, error: Exception) -> None: @@ -111,24 +198,156 @@ def _warn_unwritable(key: str, error: Exception) -> None: ) +class _ReservationStore(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: tuple[object, ...], *, ttl: float) -> None: ... + + def delete_cache(self, key: str) -> None: ... + + +def _picks(store: _ReservationStore, deployment_id: str) -> tuple[_Pick, ...]: + return _PICKS.validate_python(store.get_cache(_reservation_key(deployment_id))) or () + + +def _attempt_times(store: _ReservationStore, key: str) -> tuple[float, ...]: + return _ATTEMPT_TIMES.validate_python(store.get_cache(key)) or () + + +def _picked_deployments(store: _ReservationStore, call_id: str) -> tuple[str, ...]: + return _PICKED_DEPLOYMENTS.validate_python(store.get_cache(_picked_deployments_key(call_id))) or () + + +def _put(store: _ReservationStore, key: str, values: tuple[object, ...], ttl: float) -> None: + # InMemoryCache keeps a live key's first TTL, so drop the key to restart it + store.delete_cache(key) + if values: + store.set_cache(key, values, ttl=ttl) + + +def _release_call_on(store: _ReservationStore, deployment_id: str, call_id: str) -> None: + kept: Final = tuple(pick for pick in _picks(store, deployment_id) if pick[1] != call_id) + _put(store, _reservation_key(deployment_id), kept, SELECTION_RESERVATION_TTL_SECONDS) + + +def release_unstarted_picks(store: _ReservationStore, call_id: str | None) -> None: + if call_id is None: + return + if not _picked_deployments(store, call_id): + return + with _RESERVATIONS_LOCK: + for deployment_id in frozenset(_picked_deployments(store, call_id)): + _release_call_on(store, deployment_id, call_id) + store.delete_cache(_picked_deployments_key(call_id)) + + +@contextlib.contextmanager +def owning_picks(store: _ReservationStore, call_id: str | None) -> Generator[None, None, None]: + owner: Final = call_id or _PICK_OWNER.get() or uuid.uuid4().hex + token: Final = _PICK_OWNER.set(owner) + try: + yield + finally: + _PICK_OWNER.reset(token) + release_unstarted_picks(store, owner) + + class LeastBusyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 logged_failure: int = 0 - def __init__(self, router_cache: DualCache): + def __init__( + self, + router_cache: DualCache, + clock: Callable[[], float] = time.monotonic, + reservation_store: _ReservationStore | None = None, + ): self.router_cache = router_cache self.router_cache_id = str(id(router_cache)) + self._clock = clock + self._reservations: Final[_ReservationStore] = ( + reservation_store + if reservation_store is not None + else InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS, clock=clock) + ) + + def _live_reservations(self, deployment_id: str, now: float) -> tuple[_Pick, ...]: + return _unexpired(_picks(self._reservations, deployment_id), now) + + def _store_reservations(self, deployment_id: str, picks: tuple[_Pick, ...]) -> None: + _put(self._reservations, _reservation_key(deployment_id), picks, SELECTION_RESERVATION_TTL_SECONDS) + + def _take_reservation(self, deployment_id: str, call_id: str | None, now: float) -> None: + self._store_reservations(deployment_id, _without_one_pick(self._live_reservations(deployment_id, now), call_id)) + + def _record_pick(self, call_id: str, deployment_id: str) -> None: + picked: Final = (*_picked_deployments(self._reservations, call_id), deployment_id) + _put(self._reservations, _picked_deployments_key(call_id), picked, SELECTION_RESERVATION_TTL_SECONDS) + + def _select( + self, + healthy_deployments: Sequence[Mapping[str, object]], + deployment_ids: tuple[str, ...], + keys: tuple[str, ...], + shared: tuple[int, ...] | None, + reserve: bool, + call_id: str | None, + ) -> Mapping[str, object] | None: + if not healthy_deployments: + return None + with _RESERVATIONS_LOCK: + now: Final = self._clock() + counts: Final = ( + self._this_workers_counts(keys) if shared is None else self._at_least_this_workers(keys, shared) + ) + live: Final = tuple(self._live_reservations(deployment_id, now) for deployment_id in deployment_ids) + index: Final = _least_busy_index(tuple(count + len(r) for count, r in zip(counts, live, strict=True))) + if reserve: + tag: Final = call_id if call_id is not None else _PICK_OWNER.get() + self._store_reservations(deployment_ids[index], (*live[index], (now, tag))) + if tag is not None: + self._record_pick(tag, deployment_ids[index]) + return healthy_deployments[index] + + def _start_attempt(self, ref: tuple[str, str], call_id: str | None) -> None: + with _RESERVATIONS_LOCK: + now: Final = self._clock() + self._take_reservation(ref[1], call_id, now) + self._count_locally(_request_count_key(*ref), 1) + if call_id is not None: + key: Final = _started_attempts_key(call_id, ref) + started: Final = (*_attempt_times(self._reservations, key), now) + _put(self._reservations, key, started, IN_FLIGHT_COUNT_TTL_SECONDS) + + def _end_attempt(self, ref: tuple[str, str], call_id: str | None) -> bool: + if call_id is None: + with _RESERVATIONS_LOCK: + self._count_locally(_request_count_key(*ref), -1) + return True + key: Final = _started_attempts_key(call_id, ref) + with _RESERVATIONS_LOCK: + started: Final = _attempt_times(self._reservations, key) + if started: + _put(self._reservations, key, started[1:], IN_FLIGHT_COUNT_TTL_SECONDS) + self._count_locally(_request_count_key(*ref), -1) + else: + self._take_reservation(ref[1], call_id, self._clock()) + return bool(started) @with_service_target("router_usage") def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: - self._increment(kwargs, 1) + ref: Final = _deployment_ref(kwargs) + if ref is None: + return + self._start_attempt(ref, call_id_of(kwargs)) + self._count_shared(_request_count_key(*ref), 1) @with_service_target("router_usage") def log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -136,7 +355,7 @@ class LeastBusyLoggingHandler(CustomLogger): def log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_failure += 1 @@ -144,7 +363,7 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -152,15 +371,20 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_failure += 1 @with_service_target("router_usage") def get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -168,15 +392,19 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) @with_service_target("router_usage") async def async_get_available_deployments( - self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]] + self, + model_group: str, + healthy_deployments: Sequence[Mapping[str, object]], + reserve: bool = False, + call_id: str | None = None, ) -> Mapping[str, object] | None: - keys: Final = _request_count_keys(model_group, healthy_deployments) + deployment_ids: Final = _deployment_ids(healthy_deployments) + keys: Final = _request_count_keys(model_group, deployment_ids) redis_cache: Final = self.router_cache.redis_cache if redis_cache is not None: try: @@ -184,42 +412,51 @@ class LeastBusyLoggingHandler(CustomLogger): except Exception as e: _warn_unreadable(model_group, e) else: - return _least_busy(healthy_deployments, shared) - local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys) - return _least_busy(healthy_deployments, local) + return self._select(healthy_deployments, deployment_ids, keys, shared, reserve, call_id) + return self._select(healthy_deployments, deployment_ids, keys, None, reserve, call_id) - def _increment(self, kwargs: Mapping[str, object], delta: int) -> None: + def _settle(self, kwargs: Mapping[str, object]) -> None: ref: Final = _deployment_ref(kwargs) - if ref is None: + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): return - key: Final = _request_count_key(*ref) - redis_cache: Final = self.router_cache.redis_cache + self._count_shared(_request_count_key(*ref), -1) + + async def _async_settle(self, kwargs: Mapping[str, object]) -> None: + ref: Final = _deployment_ref(kwargs) + if ref is None or not self._end_attempt(ref, call_id_of(kwargs)): + return + await self._async_count_shared(_request_count_key(*ref), -1) + + def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]: + return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) + + def _at_least_this_workers(self, keys: tuple[str, ...], shared: tuple[int, ...]) -> tuple[int, ...]: + return tuple(max(counts) for counts in zip(shared, self._this_workers_counts(keys), strict=True)) + + def _count_locally(self, key: str, delta: int) -> None: try: local: Final = self.router_cache.increment_cache( key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS ) if local < 0: self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return + except TypeError as e: + _warn_unwritable(key, e) + + def _count_shared(self, key: str, delta: int) -> None: + redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return + try: redis_cache.increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) - async def _async_increment(self, kwargs: Mapping[str, object], delta: int) -> None: - ref: Final = _deployment_ref(kwargs) - if ref is None: - return - key: Final = _request_count_key(*ref) + async def _async_count_shared(self, key: str, delta: int) -> None: redis_cache: Final = self.router_cache.redis_cache + if redis_cache is None: + return try: - local: Final = await self.router_cache.async_increment_cache( - key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS - ) - if local is not None and local < 0: - await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS) - if redis_cache is None: - return await redis_cache.async_increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS) except Exception as e: _warn_unwritable(key, e) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 3f0a5412b62..6a8757f6e6e 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -2,16 +2,22 @@ # picks based on response time (for streaming, this is time to first token) from collections.abc import Mapping from datetime import datetime +from functools import partial from typing import Final import litellm -from litellm import ModelResponse, token_counter, verbose_logger +from litellm import ModelResponse, verbose_logger from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type -from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed +from litellm.litellm_core_utils.deployment_pricing import ( + deployment_backend_models, + deployment_pricing_models, + str_keyed, +) +from litellm.litellm_core_utils.token_counter import embedding_aware_token_count def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]: @@ -235,7 +241,9 @@ class LowestCostLoggingHandler(CustomLogger): } try: - input_tokens = token_counter(messages=messages, text=input) + input_tokens = embedding_aware_token_count( + messages, input, partial(deployment_backend_models, healthy_deployments) + ) except Exception: input_tokens = 0 diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index c352934db6c..981fb5341dc 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -3,18 +3,21 @@ import random from collections.abc import Sequence from datetime import datetime, timedelta +from functools import partial from math import ceil, isfinite from typing import TYPE_CHECKING, Any, Final from pydantic import Field import litellm -from litellm import ModelResponse, token_counter, verbose_logger +from litellm import ModelResponse, verbose_logger from litellm._internal_context import with_service_target from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models +from litellm.litellm_core_utils.token_counter import embedding_aware_token_count from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import LiteLLMPydanticObjectBase @@ -439,7 +442,9 @@ class LowestLatencyLoggingHandler(CustomLogger): } try: - input_tokens = token_counter(messages=messages, text=input) + input_tokens = embedding_aware_token_count( + messages, input, partial(deployment_backend_models, healthy_deployments) + ) except Exception: input_tokens = 0 diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 9636d7f009d..e9769c6f595 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -2,14 +2,16 @@ # identifies lowest tpm deployment import traceback from datetime import datetime +from functools import partial from typing import Final -from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models +from litellm.litellm_core_utils.token_counter import embedding_aware_token_count from litellm.types.utils import LiteLLMPydanticObjectBase from litellm.utils import print_verbose @@ -174,7 +176,9 @@ class LowestTPMLoggingHandler(CustomLogger): verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict) try: - input_tokens = token_counter(messages=messages, text=input) + input_tokens = embedding_aware_token_count( + messages, input, partial(deployment_backend_models, healthy_deployments) + ) except Exception: input_tokens = 0 verbose_router_logger.debug("input_tokens=%s", input_tokens) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 0719b30d36d..0574c3d7f69 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -5,18 +5,20 @@ from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass +from functools import partial from typing import TYPE_CHECKING, Any, Final import httpx import litellm -from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models +from litellm.litellm_core_utils.token_counter import embedding_aware_token_count from litellm.types.router import RouterErrors from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload from litellm.utils import get_utc_datetime, print_verbose @@ -422,7 +424,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): rpm_dict[rpm_keys[idx].split(":")[0]] = rpm_values[idx] try: - input_tokens = token_counter(messages=messages, text=input) + input_tokens = embedding_aware_token_count( + messages, input, partial(deployment_backend_models, healthy_deployments) + ) except Exception: input_tokens = 0 verbose_router_logger.debug("input_tokens=%s", input_tokens) diff --git a/litellm/router_utils/client_initalization_utils.py b/litellm/router_utils/client_initalization_utils.py index fdcead4a6b9..b7bb6d80ad4 100644 --- a/litellm/router_utils/client_initalization_utils.py +++ b/litellm/router_utils/client_initalization_utils.py @@ -50,7 +50,7 @@ class MaxParallelRequestsLimit: class InitalizeCachedClient: @staticmethod @renamed_router_param("litellm_router_instance") - def set_max_parallel_requests_client(ctx: "RoutingContext", model: dict): + def set_max_parallel_requests_client(ctx: "RoutingContext", model: dict) -> None: litellm_params: Final = model.get("litellm_params", {}) model_id: Final = model["model_info"]["id"] rpm: Final = litellm_params.get("rpm", None) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index e50d5bb47a6..03485ebd5f8 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -125,12 +125,13 @@ _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( def _first_present(*sources: Mapping[str, Any] | None, key: str) -> int | float | None: """Return *key* from the first source mapping where it's set, so callers can support a setting living in more than one deployment config location. Sources - are checked in order from most to least specific to that setting.""" + are checked in order from most to least specific to that setting. Only a number + counts: a bool or a string is skipped, so the router default applies.""" for source in sources: if source is None: continue value = source.get(key) - if value is not None: + if isinstance(value, int | float) and not isinstance(value, bool): return value return None @@ -811,7 +812,7 @@ def _is_allowed_fails_set_on_router( return False -def cast_exception_status_to_int(exception_status: str | int) -> int: +def cast_exception_status_to_int(exception_status: object) -> int: if isinstance(exception_status, str): try: exception_status = int(exception_status) @@ -820,6 +821,13 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: "Unable to cast exception status to int %s. Defaulting to status=500.", exception_status ) exception_status = 500 + # A bool or a number no HTTP status can be is read as a 500, as the Rust router core reads it. + if ( + isinstance(exception_status, bool) + or not isinstance(exception_status, int) + or not 0 <= exception_status <= 65535 + ): + return 500 return exception_status @@ -847,6 +855,45 @@ def is_stream_stall(exception: BaseException) -> bool: return isinstance(exception, litellm.Timeout) and isinstance(exception.__cause__, MidStreamFallbackError) +def is_caller_entra_credential_failure( + deployment: Deployment | None, + router_defaults: Mapping[str, object], + model_call_details: Mapping[str, object], + exception_status: str | int, + exception: BaseException, +) -> bool: + if cast_exception_status_to_int(exception_status) != 401 or deployment is None: + return False + + from litellm.llms.azure.common_utils import caused_by_entra_rejection + + if not caused_by_entra_rejection(exception): + return False + from litellm.llms.azure_ai.common_utils import AZURE_ENTRA_LITELLM_PARAM_KEYS, azure_entra_environment_defaults + + call_params: Final = _CREDENTIAL_VALUES_ADAPTER.validate_python(model_call_details.get("litellm_params") or {}) + + credential_name: Final = deployment.litellm_params.litellm_credential_name + configured_by_precedence: Final[tuple[Mapping[str, object], ...]] = ( + router_defaults, + _CREDENTIAL_VALUES_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)), + _CREDENTIAL_VALUES_ADAPTER.validate_python( + CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {} + ), + ) + environment: Final = azure_entra_environment_defaults() + return any( + call_params.get(key) is not None + and (call_params.get(key) or environment.get(key)) + != (_first_configured(configured_by_precedence, key) or environment.get(key)) + for key in AZURE_ENTRA_LITELLM_PARAM_KEYS + ) + + +def _first_configured(sources: tuple[Mapping[str, object], ...], key: str) -> object: + return next((value for source in sources if (value := source.get(key)) is not None), None) + + def is_caller_scoped_auth_failure(deployment: Deployment | None, exception_status: str | int) -> bool: if cast_exception_status_to_int(exception_status) not in (401, 403) or deployment is None: return False diff --git a/litellm/router_utils/deployment_numbers.py b/litellm/router_utils/deployment_numbers.py new file mode 100644 index 00000000000..7cee96f6a64 --- /dev/null +++ b/litellm/router_utils/deployment_numbers.py @@ -0,0 +1,90 @@ +# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false +"""Numbers a router deployment cannot be served with, refused the same way by both router cores. + +A whole number wider than 64 bits (a `ptu_shares` of `1e20`) does not fit the router's JSON +deployment record, and an infinite or NaN cost would read back as null there, so the call +would silently be priced from the model cost map. Both are refused when the deployment is +built or added, with the error naming the field. +""" + +import math +import re +from collections.abc import Iterator, Mapping +from typing import Final + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +# The whole numbers a deployment record holds (serde_json's i64 and u64). +SMALLEST_NUMBER: Final = -(2**63) +LARGEST_NUMBER: Final = 2**64 - 1 +# A pricing field: `input_cost_per_token`, `cache_read_input_token_cost`, ... +_COST_FIELD: Final = re.compile(r"cost_per|_cost$") + + +def _reads_as_non_finite(value: object) -> bool: + """Whether a cost, as written, is an infinity or a NaN (``float()`` reads ``"inf"`` and ``"1e400"``).""" + if isinstance(value, float): + return not math.isfinite(value) + if not isinstance(value, str): + return False + try: + return not math.isfinite(float(value)) + except ValueError: + return False + + +def _is_cost(key: object) -> bool: + return isinstance(key, str) and _COST_FIELD.search(key) is not None + + +def _number_leaves(value: object) -> Iterator[tuple[str, object, bool]]: + """Every value under ``value`` that is no container, in order: its path, and whether it sits in a + cost field. Each container is walked once, so an entry that holds itself ends.""" + pending: Final[list[tuple[object, str, bool]]] = [(value, "", False)] # mutable-ok: the walk's work list + walked: Final[set[int]] = set() # mutable-ok: the ids walked so far + while pending: + current, path, cost = pending.pop() + if not isinstance(current, Mapping | list | tuple): + yield path, current, cost + elif id(current) not in walked: + walked.add(id(current)) + pending.extend( + reversed( + [(item, f"{path}.{key}".removeprefix("."), cost or _is_cost(key)) for key, item in current.items()] + if isinstance(current, Mapping) + else [(item, f"{path}[{index}]", cost) for index, item in enumerate(current)] + ) + ) + + +def _refusal(entry: Mapping[str, object], reason: str) -> ValueError: + return ValueError(f"Deployment {entry.get('model_name')!r} cannot be served: {reason}") + + +def refuse_wide_numbers(entry: object) -> None: + """Refuse a deployment holding a whole number wider than 64 bits, by name.""" + deployment: Final = as_str_mapping(entry) + if deployment is None: + return + for path, value, _ in _number_leaves(deployment): + if isinstance(value, int) and not SMALLEST_NUMBER <= value <= LARGEST_NUMBER: + raise _refusal( + deployment, + f"{path} is {value}, outside what the router can hold " + f"({SMALLEST_NUMBER} to {LARGEST_NUMBER}). Use a smaller value.", + ) + + +def refuse_unservable_numbers(entry: object) -> None: + """``refuse_wide_numbers``, and a deployment with an infinite or NaN cost.""" + refuse_wide_numbers(entry) + deployment: Final = as_str_mapping(entry) + if deployment is None: + return + for path, value, cost in _number_leaves(deployment): + if cost and _reads_as_non_finite(value): + raise _refusal( + deployment, + f"{path} is {value!r}, and the router only prices with finite costs. " + "Set a finite cost, or remove it to use the model cost map's price.", + ) diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index 4a87e582c97..7f58c138bba 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -57,10 +57,14 @@ class _EndpointDeclaration(BaseModel): return (value,) if isinstance(value, str) else value +def endpoint_for_call(call_type: object, module: object = None) -> str | None: + qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type + return CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None + + @contextmanager def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]: - qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type - endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None + endpoint: Final = endpoint_for_call(call_type, module) token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None)) try: yield @@ -68,6 +72,10 @@ def endpoint_selection(call_type: object, module: object = None, model: object = _REQUEST_SELECTION.reset(token) +def selected_endpoint() -> str | None: + return _REQUEST_SELECTION.get()[0] + + def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None: try: declaration: Final = _EndpointDeclaration.model_validate(model_info) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..20f9e114747 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -28,6 +28,7 @@ from litellm.router_utils.cooldown_handlers import ( _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils cast_exception_status_to_int, is_advisor_orchestration_failure, + is_caller_entra_credential_failure, is_caller_scoped_auth_failure, is_caller_timeout_408, is_stream_stall, @@ -132,6 +133,14 @@ def _trigger_cooldown_for_failed_deployment( deployment_id, ) return + if is_caller_entra_credential_failure( + deployment, ctx.default_litellm_params(), model_call_details, exception_status, exception + ): + verbose_router_logger.debug( + "Not triggering cooldown for fallback deployment %s: Entra ID rejected a credential the caller sent.", + deployment_id, + ) + return # Priority: deployment config > response header > router default, matching # Router.deployment_callback_on_failure's precedence for the primary path. @@ -623,6 +632,8 @@ def get_fallback_model_group( ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): if isinstance(item, dict): + if not item: + continue # an empty entry names no group: the next entries still apply fallback_key = next(iter(item)) if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] diff --git a/litellm/router_utils/global_callback_holders.py b/litellm/router_utils/global_callback_holders.py new file mode 100644 index 00000000000..53a7c20cff3 --- /dev/null +++ b/litellm/router_utils/global_callback_holders.py @@ -0,0 +1,53 @@ +""" +The routers of this process that hold optional pre-call checks and routing callbacks, whatever core +serves them. + +A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving +router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at +runtime keeps it registered while another live router still has one of that type +(``live_check_owners``). Both rules look at every router in the process, so the sets live here, +outside any one router core. + +A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``, +``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners`` +serves ``optional_callbacks``. +""" + +import threading +import weakref +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final, Protocol + +if TYPE_CHECKING: + from litellm.integrations.custom_logger import CustomLogger + + +class GlobalCallbackHolder(Protocol): + """A router whose callbacks a retiring router may hand its vacated checks to.""" + + @property + def _core_adaptive_routers(self) -> Mapping[str, object]: ... + + def _global_callback_candidates(self) -> "tuple[CustomLogger, ...]": ... + + def _sync_adaptive_router_hooks(self) -> None: ... + + +class CheckOwner(Protocol): + """A live router with optional pre-call checks.""" + + @property + def optional_callbacks(self) -> "Sequence[object] | None": ... + + +routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet() +routers_holding_global_callbacks_lock: Final = threading.Lock() +live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet() + +__all__ = [ + "CheckOwner", + "GlobalCallbackHolder", + "live_check_owners", + "routers_holding_global_callbacks", + "routers_holding_global_callbacks_lock", +] diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index c759af8fdb8..1c9cb10a48b 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -6,6 +6,7 @@ and exposes it for router candidate filtering. """ import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from typing_extensions import TypedDict @@ -78,7 +79,7 @@ class DeploymentHealthCache: merged: Final = { model_id: state for model_id, state in {**existing, **states}.items() - if isinstance(state, dict) and (now - state.get("timestamp", 0)) < expiry_seconds + if isinstance(state, dict) and (now - _timestamp_of(state)) < expiry_seconds # pyright: ignore[reportUnknownArgumentType] # a cached record } self.cache.set_cache( key=self.CACHE_KEY, @@ -101,7 +102,7 @@ class DeploymentHealthCache: for model_id, state in raw.items() if isinstance(state, dict) and not state.get("is_healthy", True) - and (now - state.get("timestamp", 0)) < self.staleness_threshold + and (now - _timestamp_of(state)) < self.staleness_threshold # pyright: ignore[reportUnknownArgumentType] # a cached record } @with_service_target(HEALTH_CHECKS_TARGET) @@ -129,3 +130,10 @@ class DeploymentHealthCache: str(e), ) return set() + + +def _timestamp_of(state: Mapping[str, object]) -> float: + """A health record's timestamp; one that is no number reads as 0 (stale), so a bad record + drops out alone instead of turning health routing off for every deployment.""" + timestamp: Final = state.get("timestamp", 0) + return float(timestamp) if isinstance(timestamp, int | float) and not isinstance(timestamp, bool) else 0.0 diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 80e1f3fe29c..f887c4551f2 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -20,6 +20,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter import litellm from litellm import token_counter @@ -28,6 +29,7 @@ from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION +from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.litellm_core_utils.internal_call_metadata import ( IO_TOKEN_RESERVATION_KEYS, ITPM_CACHE_KEY, @@ -41,6 +43,8 @@ from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.router import RouterCacheEnum, RouterErrors from litellm.utils import get_utc_datetime +_MODEL_INFO_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -227,8 +231,20 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme ) -def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int: - model_max: Final = _model_max_output_tokens(model_name) if model_name else None +def _default_otpm_reservation(deployment: Mapping[str, object], model_name: str, otpm_limit: int) -> int: + model_info: Final[object] = deployment.get("model_info") + configured_model_max: Final = ( + coerce_token_limit(_MODEL_INFO_ADAPTER.validate_python(model_info).get("max_output_tokens")) + if isinstance(model_info, Mapping) + else None + ) + model_max: Final = ( + max(0, configured_model_max) + if configured_model_max is not None + else _model_max_output_tokens(model_name) + if model_name + else None + ) share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)) return min(4096 if model_max is None else model_max, share_of_limit) @@ -242,7 +258,7 @@ def _otpm_reservation( from_request: Final = _explicit_max_tokens(request_kwargs) if from_request is not None: return from_request - default: Final = _default_otpm_reservation(model_name, otpm_limit) + default: Final = _default_otpm_reservation(deployment, model_name, otpm_limit) from_deployment: Final = _deployment_output_cap(request_kwargs, deployment) return default if from_deployment is None else min(from_deployment, default) diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index f41d19cd62c..a3da0ae1674 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -134,7 +134,7 @@ class ModelRateLimitingCheck(CustomLogger): this check actively enforces those limits across ALL routing strategies. """ - def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime): + def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime) -> None: self.dual_cache = dual_cache self.clock = clock # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment, diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index f78b5982186..1d91255e28c 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js return tuple(sorted(pairs, key=lambda pair: pair[0])) +_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint")) + + +def _is_cache_marked(block: Mapping[str, object]) -> bool: + match block: + case {"cache_control": {"type": "ephemeral"}}: + return True + case _: + return block.get("prompt_cache_breakpoint") is not None + + def _canonical_bytes(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() @@ -66,7 +77,7 @@ def _block_unit( return _canonical_bytes((envelope, block)), message_run_type block_type: Final = block.get("type") block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None - stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control") + stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS) return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type @@ -264,15 +275,9 @@ class PromptCachingCache: continue for content_idx, content_block in enumerate(content): - if isinstance(content_block, dict): - cache_control = content_block.get("cache_control") - if ( - cache_control is not None - and isinstance(cache_control, dict) - and cache_control.get("type") == "ephemeral" - ): - last_cacheable_message_idx = msg_idx - last_cacheable_content_idx = content_idx + if isinstance(content_block, dict) and _is_cache_marked(content_block): + last_cacheable_message_idx = msg_idx + last_cacheable_content_idx = content_idx # If no cacheable block found, return empty list (no cacheable prefix) if last_cacheable_message_idx is None: diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index aa2b29988dc..17a56e62e41 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -52,7 +52,7 @@ class SearchAPIRouter: @staticmethod @renamed_router_param("router_instance") - async def update_router_search_tools(ctx: "RoutingContext", search_tools: list): + async def update_router_search_tools(ctx: "RoutingContext", search_tools: list) -> None: """ Update the router with search tools from the database. @@ -143,7 +143,6 @@ class SearchAPIRouter: # Set up kwargs for the fallback system kwargs["model"] = search_tool_name # Use model field for compatibility with fallback system kwargs["original_generic_function"] = original_function - # Bind ctx to the helper method using partial kwargs["original_function"] = partial( SearchAPIRouter.async_search_with_fallbacks_helper, litellm_routing_ctx=ctx, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 8177e4a5cc9..dc170116089 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,6 +1,6 @@ from asyncio import Future from collections.abc import Mapping, Sequence -from typing import Never, final +from typing import Generic, Literal, Never, NotRequired, TypedDict, TypeVar, final import httpx from pydantic import JsonValue @@ -218,6 +218,8 @@ __all__ = [ "NativeDiagnosticProcessor", "ProcessReservedForForking", "ResponsesWebSocketConnection", + "RouterCall", + "RouterCoreHandle", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", @@ -297,31 +299,492 @@ class _SecretManagerRuntime: primary_secret_name: str | None = None, ) -> Future[JsonValue]: ... +# The FailureKind names an attempt outcome's `kind` takes (router-core `FailureKind`). +_FailureKind = Literal[ + "RateLimitError", + "ContextWindowExceededError", + "ContentPolicyViolationError", + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "BadRequestError", + "Timeout", + "APIConnectionError", + "InternalServerError", + "ServiceUnavailableError", + "BadGatewayError", + "GuardrailIntervention", + "Declined", + "CallerCredentialAuthenticationError", + "CallerCredentialRateLimitError", + "APIError", +] + +# One event a call reported since the last step (router-core `RouterEvent`), carried on the next step. +class _Event(TypedDict): + kind: str + +# Every step `RouterCall.next` returns carries the events reported since the last one. +class _Step(TypedDict): + events: list[_Event] + +class _AttemptStep(_Step): + kind: Literal["attempt"] + deployment_id: str + # The deployment as a `model_list` entry, with its id in `model_info`. + deployment: dict[str, JsonValue] + # The name this attempt serves: the requested model or the fallback target. + model: str + model_group: str + target_order: int | None + candidate_ids: list[str] + retry: int + fallback_depth: int + fallback_level: int + timeout: float | None + # Request params the fallback walk's dict targets merged in. + params: dict[str, JsonValue] | None + # The walk level whose request metadata the attempt carries. + scope: int + max_retries: int + model_group_size: int + +class _SleepStep(_Step): + kind: Literal["sleep"] + seconds: float + +class _AllowFallbackStep(_Step): + kind: Literal["allow_fallback"] + target: str + +class _RefuseFallbacksStep(_Step): + kind: Literal["refuse_fallbacks"] + # The walk level whose chain the dict targets are. + level: int + targets: list[dict[str, JsonValue]] + +class _SelectStep(_Step): + kind: Literal["select"] + stage: Literal[ + "plugins", + "strategy", + "tags", + "pre_call", + "text_only", + "callback_filters", + "routing_plugins", + "provider_request", + ] + strategy: str | None + model_group: str + # The name the request resolved from, which strategies key their state by. + model: str + # Whose selector a strategy reads: "default", "group" (with `group`) or "override". + scope: str + group: str | None + # The deployments the name resolved to, before health filters. + pool: list[str] + candidates: list[str] + # `routing_plugins` only: whether the plugins run. + run: NotRequired[bool] + # `callback_filters` only: the ids the request already failed on. + excluded_ids: NotRequired[list[str]] + skipped_ids: NotRequired[list[str]] + +# Pre-routing before a pick resolves `model`: answered with the model group to resolve instead +# (a string), a custom strategy's deployment document (a mapping), or None. +class _PreRouteStep(_Step): + kind: Literal["pre_route"] + model: str + fallback_depth: int + retry: int + +# One RouterCache op against the router's DualCache. `key` is set for every op but `batch_get`, +# and `increment_usage`, which set `keys`; `value` is the JSON to store (`set`), the amount +# (`increment`) or one amount per key (`increment_usage`); `ttl` is in seconds; `local` reads or +# writes this worker's in-memory tier only. Outcomes: get -> the JSON value or None; batch_get -> +# one JSON value or None per key, in order; increment -> the new int or float; set, delete and +# increment_usage -> None. +class _CacheStep(_Step): + kind: Literal["cache"] + op: Literal["get", "set", "increment", "increment_usage", "batch_get", "delete"] + key: str | None + keys: list[str] | None + value: JsonValue + ttl: float | None + local: bool + +# One lookup of a model-info query (router-core `ModelInfoLookup`): its `lookup` name and that +# lookup's arguments, answered as the lookup documents. +class _ModelInfoStep(_Step): + kind: Literal["model_info"] + lookup: Literal[ + "model_info", + "cost_entry", + "provider", + "pattern_models", + "bedrock_base", + "discovered", + "user_model_info", + "log", + "group_params", + "group_deployment", + "group_capabilities", + "group_seed", + "reasoning_intersect", + ] + model: NotRequired[JsonValue] + tolerate: NotRequired[bool] + exact: NotRequired[bool] + key: NotRequired[str] + process: NotRequired[bool] + name: NotRequired[str] + model_id: NotRequired[str] + id: NotRequired[JsonValue] + level: NotRequired[Literal["debug", "error"]] + message: NotRequired[str] + index: NotRequired[int] + model_info: NotRequired[JsonValue] + litellm_model: NotRequired[JsonValue] + llm_provider: NotRequired[JsonValue] + mapped: NotRequired[bool] + info: NotRequired[dict[str, JsonValue]] + current: NotRequired[JsonValue] + resolved: NotRequired[JsonValue] + +class _FailedStep(_Step): + kind: Literal["failed"] + error: str + error_kind: Literal[ + "config", + "unknown_model", + "no_healthy_deployments", + "attempt", + "cache", + "retired", + "paused", + "not_pass_through", + "no_pass_through_deployments", + "only_markers", + "ptu_reserved", + "context_window_exceeded", + "unsupported_params", + "routing_plugins_left_nothing", + "routing_plugins_need_async", + "multiple_teams", + "reserved", + "invalid_router_weights", + "non_numeric_weight", + "missing_routing_group", + "default_deployment_raises", + "index_out_of_range", + "model_info", + ] + attempt_failed: bool + status: int | None + failure_kind: _FailureKind | None + error_model: NotRequired[str] + error_detail: NotRequired[str] + error_params: NotRequired[list[str]] + # `model_info` only: the builtin exception the lookup raised. + error_exception: NotRequired[str] + +# The end of a `RouterCoreHandle.start` (and the stream recovery calls). +class _DoneStep(_Step): + kind: Literal["done"] + deployment_id: str + model_group: str + attempted_retries: int + max_retries: int + attempted_fallbacks: int + total_tokens: int + +# The end of a `RouterCoreHandle.select`. +class _SelectedStep(_Step): + kind: Literal["done"] + deployment_id: str + model_group: str + deployment: dict[str, JsonValue] + +# The end of a `RouterCoreHandle.healthy`. +class _HealthyStep(_Step): + kind: Literal["done"] + # The name was a deployment id. + single: bool + deployment_ids: list[str] + deployments: list[dict[str, JsonValue]] + +# The end of a `RouterCoreHandle.report_failure`. +class _ReportedStep(_Step): + kind: Literal["done"] + cooled: bool + # The cooldown applied, in seconds; None when the deployment was not cooled down. + cooldown_time: float | None + +# The end of a `RouterCoreHandle.report_success` or `count_success`. +class _CountedStep(_Step): + kind: Literal["done"] + key: str + +# One `os.environ/` reference a `secrets` step asks for: a top-level param (`key`), or an +# env-only `dataSources[source].parameters[key]` (`env_only`). +class _SecretRef(TypedDict): + name: str + key: str + env_only: bool + source: NotRequired[int] + +# The deployment build's `os.environ/` references: answered with one value per ref, in order. +class _SecretsStep(_Step): + kind: Literal["secrets"] + index: int + refs: list[_SecretRef] + +# The deployment build's steps for one deployment. `validate` and `added` are answered with +# `{"entry": }`, `{"skip": True}` or `{"invalid": True}`; `provider` with +# `{"provider": str | None}` or `{"invalid": True}`. +class _BuildStep(_Step): + kind: Literal["validate", "provider", "added"] + path: Literal["create", "add"] + index: int + entry: dict[str, JsonValue] + declared_id: str | None + duplicate_id: bool + provider: str | None + +class _BuiltDeployment(TypedDict): + index: int + id: str | None + provider: str | None + +# The end of a `RouterCoreHandle.build`. +class _BuiltStep(_Step): + kind: Literal["done"] + built: list[_BuiltDeployment] + +# The end of a model-info query: `router_model_info` sets `info` (the merged map or None) and +# `user_keys`; `deployment_model_info` sets `info`; the others set `value`. +class _ModelInfoDoneStep(_Step): + kind: Literal["done"] + info: NotRequired[dict[str, JsonValue] | None] + user_keys: NotRequired[list[str]] + value: NotRequired[JsonValue] + +# `RouterCoreHandle.resolve`: what a name resolves to. +class _Resolution(TypedDict): + model_group: str + kind: str + ids: list[str] + +# The steps each kind of `RouterCall` returns, by the handle method that starts it. +_CallSteps = ( + _AttemptStep + | _SleepStep + | _AllowFallbackStep + | _RefuseFallbacksStep + | _SelectStep + | _PreRouteStep + | _CacheStep + | _DoneStep + | _FailedStep +) +_SelectSteps = _SelectStep | _PreRouteStep | _CacheStep | _SelectedStep | _FailedStep +_HealthySteps = _SelectStep | _PreRouteStep | _CacheStep | _HealthyStep | _FailedStep +_BuildSteps = _SecretsStep | _BuildStep | _BuiltStep | _FailedStep +_ModelInfoSteps = _ModelInfoStep | _ModelInfoDoneStep | _FailedStep +_FailureReportSteps = _CacheStep | _ReportedStep | _FailedStep +_SuccessReportSteps = _CacheStep | _CountedStep | _FailedStep + +class _AttemptSucceeded(TypedDict): + ok: Literal[True] + total_tokens: NotRequired[int] + blocked: NotRequired[Literal["content_filter", "refusal"] | None] + # A stream whose success or failure the host reports when it ends (`report_success`, + # `report_failure`): the attempt counts its request and `total_tokens` only. + deferred: NotRequired[bool] + # Whether the attempt counts against its deployment's usage; unset, its own limits decide. + tracks_usage: NotRequired[bool | None] + +class _AttemptFailed(TypedDict): + ok: Literal[False] + kind: _FailureKind + status: NotRequired[int | None] + retry_after: NotRequired[float | None] + num_retries: NotRequired[int | None] + # The deployment the exception names, which a retry skips over the attempt's own. + failed_deployment_id: NotRequired[str | None] + detail: NotRequired[str | None] + agentic_followup: NotRequired[bool] + # The exception has no `status_code`, which no cooldown rule reads. + status_absent: NotRequired[bool] + # The failure counts against no per-minute requests. + uncounted_request: NotRequired[bool] + +# Each document is a mapping or its JSON text. +_Document = Mapping[str, object] | str + @final -class NativeCacheHandle: - def __new__(cls, _uninstantiable: Never, /) -> Never: ... +class RouterCoreHandle: + def __new__(cls, config: _Document, dynamic: _Document) -> RouterCoreHandle: ... + def start(self, op: str, request: _Document) -> RouterCall[_CallSteps]: ... + # Driven like `start`; its `done` step is a `_SelectedStep`. + def select(self, model: str, request: _Document) -> RouterCall[_SelectSteps]: ... + # Driven like `select`; its `done` step is a `_HealthyStep`. + def healthy(self, model: str, request: _Document) -> RouterCall[_HealthySteps]: ... + # `{"replace" | "append" | "upsert": [deployments], "delete": [ids], "settings": {dynamic document}, "index": {...}}`. + def apply(self, changes: _Document) -> int: ... + # Build model_list entries through the host, driven like `start`: `_SecretsStep` and + # `_BuildStep` steps, then a `_BuiltStep`. `path`: "create" (`set_model_list`) or "add"; + # `install`: "replace", "append" or "upsert". + def build( + self, entries: Sequence[object], path: str, resolve_secrets: bool, install: str + ) -> RouterCall[_BuildSteps]: ... + # Keep the providers `other`'s builds resolved for the same deployments. + def adopt_providers(self, other: RouterCoreHandle) -> None: ... + # `get_router_model_info`, driven like `start`: `model_info` lookup steps (a `lookup` field + # and its arguments), then a done step with `info` (the merged map or None) and `user_keys`. + def router_model_info(self, query: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_deployment_model_info`, driven like `router_model_info`; its done step has `info`. + def deployment_model_info(self, model_id: str, model_name: object) -> RouterCall[_ModelInfoSteps]: ... + # `_set_model_group_info` for a `GroupInfoQuery` document, driven like `router_model_info`; its + # done step has `value`: None, or `{"assignments": {field: value}}`. + def model_group_info(self, query: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_configured_token_limits`, driven like `router_model_info`; `value` is `[input, output]`. + def configured_token_limits(self, id: object, configured: _Document) -> RouterCall[_ModelInfoSteps]: ... + # `get_model_listing_info`, driven like `router_model_info`; `value` has `cost_map_keys`, + # `max_input_tokens` and `max_output_tokens`. + def model_listing_info(self, deployments: list[object]) -> RouterCall[_ModelInfoSteps]: ... + # A configured string that is not blank (`get_configured_mode`, `get_configured_display_name`). @staticmethod - def memory( - *, - ttl: float = 600.0, - capacity: int = 200, - max_entry_bytes: int = 4194304, - ) -> NativeCacheHandle: ... + def configured_text(value: object) -> str | None: ... + # Each model_info's `service_tiers`, unvalidated. @staticmethod - def redis( - url: str, - *, - namespace: str, - ttl: float = 600.0, - max_entry_bytes: int = 4194304, - ) -> NativeCacheHandle: ... - def get(self, key: str) -> object: ... - def set(self, key: str, value: object, *, ttl: float | None = None) -> None: ... - def async_get(self, key: str) -> Future[object]: ... - def async_set(self, key: str, value: object, *, ttl: float | None = None) -> Future[None]: ... - def async_set_many(self, entries: Sequence[tuple[str, object]], *, ttl: float | None = None) -> Future[None]: ... - def flush(self) -> None: ... - def async_flush(self) -> Future[None]: ... - def ping(self) -> Future[bool]: ... - def disconnect(self) -> Future[None]: ... - def delete(self, keys: Sequence[str]) -> Future[None]: ... + def configured_service_tiers(model_infos: list[object]) -> list[object]: ... + # The routing-cache key a deployment counts `metric` (tpm, rpm, itpm, otpm) under in `minute`. + def usage_key(self, id: str, model: str, metric: str, minute: str) -> str: ... + def registry_version(self) -> int: ... + # The version the deployments before the newest appends last changed at. + def registry_rewritten(self) -> int: ... + # A failure the router did not see itself, driven like `start`: cache steps, then a + # `_ReportedStep`. + def report_failure( + self, + id: str, + failure: Mapping[str, object], + deployment_cooldown: bool = False, + count_only: bool = False, + requested_group: str | None = None, + cooldown_time: float | None = None, + uncounted: bool = False, + counts_request: bool = False, + usage_model: str | None = None, + callbacks: Mapping[str, bool] | None = None, + ) -> RouterCall[_FailureReportSteps]: ... + # A success a logging callback saw, driven like `report_failure`; `done` carries `key`. + def report_success( + self, + id: str, + total_tokens: float, + requests: float, + usage_model: str | None = None, + tracks_usage: bool | None = None, + callbacks: Mapping[str, bool] | None = None, + ) -> RouterCall[_SuccessReportSteps]: ... + # Count a success against any id, driven like `report_failure`; `done` carries `key`. + def count_success(self, id: str, callbacks: Mapping[str, bool] | None = None) -> RouterCall[_SuccessReportSteps]: ... + # [(id, tpm_key, rpm_key)] for this minute, or None for a group with no deployment. + def usage_keys(self, model_group: str) -> list[tuple[str, str, str]] | None: ... + # Every deployment as a `model_list` entry, in config order. + def deployments(self) -> list[dict[str, JsonValue]]: ... + def deployments_from(self, start: int) -> list[dict[str, JsonValue]]: ... + def deployment(self, id: str) -> dict[str, JsonValue] | None: ... + # Lookups over the registry indexes, answering what each index holds as written. + # Registry lookups (router-core `lookup.rs`): rows the host reads its stored entries at + # (`{"at": position, "name"?, "group"?, "member"?}`, `{"pattern": entry, "name"}`, or a + # `{"raw": value}` / `{"raw_positions": value}` its own read raises at). + def lookup_all( + self, model_name: str, team_id: str | None = None, rename: str | None = None + ) -> list[dict[str, JsonValue]]: ... + def lookup_across_teams(self, model: str) -> tuple[list[dict[str, JsonValue]], int]: ... + def lookup_config(self) -> list[dict[str, JsonValue]]: ... + def lookup_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def lookup_model_list( + self, model_name: str | None, team_id: str | None, scope: Mapping[str, object] + ) -> list[dict[str, JsonValue]]: ... + def lookup_model_names( + self, team_id: str | None, scope: Mapping[str, object] + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def lookup_access_groups( + self, model_name: str | None, access_group: str | None, team_id: str | None, scope: Mapping[str, object] + ) -> tuple[list[tuple[str, list[str]]], dict[str, JsonValue] | None]: ... + def lookup_name_serving( + self, model_id: str, scope: Mapping[str, object] + ) -> tuple[bool, str | None, dict[str, JsonValue] | None]: ... + def has_model_name(self, name: str) -> bool: ... + def lookup_team_model_has_alternatives(self, id: str) -> tuple[bool, dict[str, JsonValue] | None]: ... + def lookup_group_usable_by_team( + self, model_group: str, team_id: str | None + ) -> tuple[int | None, dict[str, JsonValue] | None]: ... + def lookup_team_public_name( + self, model_id: str, team_id: str + ) -> tuple[int | None, dict[str, JsonValue] | None]: ... + def lookup_routable(self, model_group: str, team_id: str | None) -> list[dict[str, JsonValue]]: ... + def lookup_wildcard_route( + self, model: str, team_id: str | None, requests: list[str] + ) -> list[dict[str, JsonValue]]: ... + def lookup_candidate_ids( + self, resolved: str, team_id: str | None, members: list[str] | None, requests: list[str] + ) -> tuple[list[str], dict[str, JsonValue] | None]: ... + def has_model_id(self, id: str) -> bool: ... + def id_position(self, id: str) -> JsonValue | None: ... + def id_positions(self) -> list[tuple[str, JsonValue]]: ... + def named_positions(self, name: str) -> JsonValue | None: ... + def team_positions(self, team: str, public: str) -> JsonValue | None: ... + def team_public_names_of(self, team: str) -> list[str]: ... + def litellm_model_positions(self, model: str) -> list[int]: ... + # The registry indexes the host hands out (`HostIndexes`), as JSON. + def host_indexes(self) -> dict[str, JsonValue]: ... + def provider_default_ids(self) -> list[str]: ... + # `PatternMatchRouter.route` over the wildcard router, or `team`'s. + def pattern_route( + self, team: str | None, request: str, filtered: list[str] | None = None + ) -> list[dict[str, JsonValue]] | None: ... + # `get_models_by_pattern` over the wildcard router, or `team`'s. + def models_by_pattern(self, team: str | None, requests: list[str]) -> list[str] | None: ... + # The provider the host resolved when it built deployment `id` (its `provider` build step). + def provider(self, id: str) -> str | None: ... + def resolve(self, name: str, specific_deployment: bool = False) -> _Resolution | None: ... + def routable(self, model: str, request: _Document) -> list[dict[str, JsonValue]]: ... + # `_get_routing_context` for a pick of `model`: (strategy, scope, group). + def routing_context(self, model: str, routing_strategy: str | None = None) -> tuple[str, str, str | None]: ... + def retire(self) -> None: ... + # A stream that failed before content: what may still take over it, and the retry budget for + # a failure of `kind`: {"can_retry", "can_fall_back", "refusal_fallback": bool, + # "retry_budget": int | None, "policy_applies": bool}. + def stream_policy( + self, request: _Document, origin: _Document, kind: str | None = None + ) -> dict[str, bool | int | None]: ... + # The same-group retry and the fallback walk for such a stream, driven like `start`. + def retry_stream( + self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object] + ) -> RouterCall[_CallSteps]: ... + def fallback_stream( + self, op: str, request: _Document, origin: _Document, failure: Mapping[str, object] + ) -> RouterCall[_CallSteps]: ... + def retry_policy_retries(self, failure: Mapping[str, object], model_group: str | None = None) -> int | None: ... + def allowed_fails_for(self, failure: Mapping[str, object]) -> int | None: ... + def should_retry( + self, failure: Mapping[str, object], counts: tuple[int, int], fallbacks: tuple[bool, bool, bool] + ) -> bool: ... + +_S = TypeVar("_S", covariant=True) + +@final +class RouterCall(Generic[_S]): + # The answer to the last step, which Rust validates against it: attempt: _AttemptSucceeded | + # _AttemptFailed; sleep: None; allow_fallback: bool; refuse_fallbacks: bool; select: the remaining + # candidate ids, pick first; pre_route: see _PreRouteStep; cache: see _CacheStep; secrets, build and model_info: + # see those steps; first call: None. + def next(self, outcome: object) -> _S: ... diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 9ba8cbc35a7..a12f79c6c8a 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -23,6 +23,7 @@ class Route(str, Enum): OCR = "ocr" TOKEN_COUNTER = "token_counter" TOKENIZER = "tokenizer" + ROUTER = "router" class Target(str, Enum): @@ -224,6 +225,7 @@ RULES: Final[Rules] = ( RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), + RouteRule(Route.ROUTER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), ) diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index e155680bd30..92590531cdc 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -12,7 +12,7 @@ import datetime import traceback import uuid import weakref -from collections.abc import Awaitable, Coroutine, Mapping +from collections.abc import Awaitable, Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import ( TYPE_CHECKING, @@ -22,6 +22,7 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +import httpx from pydantic import TypeAdapter if TYPE_CHECKING: @@ -243,6 +244,27 @@ def post_call( logger.post_call(original_response=original_response, api_key=api_key, additional_args=additional_args) +def record_response(logger: Logging, status: int, headers: Sequence[tuple[str, str]], body: str) -> None: + """Stores the provider response the way Python's Messages handler does, for logging to replay.""" + response: Final = httpx.Response(status_code=status, headers=list(headers), content=body.encode()) + details: Final = logger.model_call_details # the call's own logger: recording on it is the point + details["httpx_response"] = response + details["response_headers"] = dict(response.headers) + + +def record_api_window(logger: Logging, start: float, end: float) -> None: + """Records the provider exchange's window the way Python's timed `AsyncHTTPHandler.post` does.""" + from litellm.litellm_core_utils.logging_utils import ( + _set_duration_in_model_call_details, # pyright: ignore[reportPrivateUsage] # the timed post's own recorder + ) + + _set_duration_in_model_call_details( + logging_obj=logger, + start_time=datetime.datetime.fromtimestamp(start), # noqa: DTZ006 # the timed post records naive local times + end_time=datetime.datetime.fromtimestamp(end), # noqa: DTZ006 # the timed post records naive local times + ) + + def defers_async_logging(logger: LoggingSurface) -> bool: return bool(getattr(logger, "defer_async_logging", False)) @@ -359,6 +381,27 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t _STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) +class _HiddenParamsCarrier: + """Stands in for the stream `@client` stamps, so the head becomes its `_hidden_params`.""" + + def __init__(self, hidden_params: dict[str, object]) -> None: # mutable-ok: the stamps land in this dict + self._hidden_params: object = hidden_params + + +def stamp_stream_head( + head: dict[str, object], # mutable-ok: the stream's hidden params, stamped in place + logger: Logging, + kwargs: dict[str, object], # mutable-ok: `finalize` takes the call kwargs as a dict + start_time: datetime.datetime, + end_time: datetime.datetime, +) -> None: + carrier: Final = _HiddenParamsCarrier(head) + finalize(carrier, logger, kwargs, start_time, end_time) + stamped: Final = carrier._hidden_params # pyright: ignore[reportPrivateUsage] # the carrier's own field + if stamped is not head and isinstance(stamped, dict): + head.update(cast("dict[str, object]", stamped)) # cast-ok: update_response_metadata writes a str-keyed dict + + def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 017ff65e2a7..c8b53097f46 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -1,21 +1,35 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +import datetime +from collections.abc import Callable, Mapping, Sequence from dataclasses import asdict, dataclass -from typing import Final, cast # noqa: TID251 # narrows the normalized native payload to the public TypedDict +from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils import exception_mapping_utils from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.messages.native import handler +from litellm.llms.anthropic.messages.native.handler import ( + _native_messages_config as native_messages_config, # pyright: ignore[reportPrivateUsage] # the handler's own config lookup +) +from litellm.llms.anthropic.messages.native.handler import ( + _with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding +) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures +from litellm.rust_bridge.host import callbacks_legacy_python from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse _DROP_PATHS: Final = TypeAdapter(list[object]) +_METADATA: Final = TypeAdapter(dict[str, object]) _BODY_FIELDS: Final = frozenset( { "max_tokens", @@ -40,6 +54,7 @@ _BODY_FIELDS: Final = frozenset( "cache_control", "reasoning_effort", "safeguards", + "service_tier", } ) @@ -59,8 +74,40 @@ def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: ) +def inferred_provider(model: str, api_base: str | None) -> str | None: + """The provider Python's handler resolves for a call that names none, or None when the model + carries no provider `get_llm_provider` knows.""" + try: + _, provider, _, _ = litellm.get_llm_provider(model=model, api_base=api_base) + except litellm.BadRequestError: + return None + return provider + + def body_params(request: Mapping[str, object]) -> Mapping[str, object]: - return known_params(_BODY_FIELDS, request) + params: Final = known_params(_BODY_FIELDS, request) + if deployment_reasoning_effort_levels(request.get("model_info")) is None: + return params + return _with_deployment_effort(params, request) + + +def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, object]) -> Mapping[str, object]: + """Python's handler rounds the requested effort to the levels the deployment declares in + `model_info.reasoning_effort_levels`, the one place a deployment, not the model, decides.""" + explicit_provider: Final = optional_str(request.get("custom_llm_provider")) + try: + model, provider, _, _ = litellm.get_llm_provider( + model=str(request["model"]), custom_llm_provider=explicit_provider + ) + except litellm.BadRequestError: + return params + return _deployment_output_effort( + params, + model=model, + custom_llm_provider=provider, + kwargs=request, + drop_params=litellm.drop_params is True or request.get("drop_params") is True, + ) def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: @@ -71,18 +118,147 @@ def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, obj return anthropic_messages_stream_hidden_params(httpx.Headers(list(headers))) -def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: - if getattr(error, "messages_request_error", False): - return litellm.BadRequestError( - message=str(error), - model=str(request["model"]).removeprefix(f"{request_provider}/"), - llm_provider=request_provider, - ) - return failures.map_native_failure( - error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) +_UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str]) +_UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) +_ERROR_CLASS: TypeAlias = Callable[..., Exception] +_validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator + Callable[[dict[str, object]], object], handler.validate_anthropic_api_metadata +) +_exception_type: Final = cast( # cast-ok: the legacy public exception mapper is untyped + Callable[..., Exception], exception_mapping_utils.exception_type +) +_BODY_READ_TIMEOUT: Final = "Timeout on reading data from socket" # aiohttp's SocketTimeoutError text +_MISSING_KEY: Final = ( + "Missing Anthropic API Key - A call is being made to anthropic but no key is set " + "either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` " + "or `ANTHROPIC_AUTH_TOKEN` in your environment vars" +) + + +def _rejection(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: + """The error Python's handler raises for a request it refuses before sending it.""" + metadata: Final = request.get("metadata") + if isinstance(metadata, dict): + try: + _validate_metadata(_METADATA.validate_python(metadata)) + except ValidationError as invalid: + return invalid + if getattr(error, "messages_unsupported_param", False): + return litellm.UnsupportedParamsError(message=str(error)) + return litellm.BadRequestError( + message=str(error), + model=str(request["model"]).removeprefix(f"{request_provider}/"), + llm_provider=request_provider, ) +def _timeout_text(phase: str, request: Mapping[str, object]) -> str: + """`litellm.Timeout`'s text as Python's HTTP client raises it for the same timeout.""" + seconds: Final = request.get("timeout") + logging_obj: Final = request.get("litellm_logging_obj") + start_time: Final = getattr(logging_obj, "start_time", None) + time_taken: Final = ( + round( + (datetime.datetime.now() - start_time).total_seconds(), # noqa: DTZ005 # the logger's start_time is naive local time + 3, + ) + if isinstance(start_time, datetime.datetime) + else None + ) + return str( + litellm.Timeout( + message=f"{phase} timed out. Timeout passed={seconds}, time taken={time_taken} seconds", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ) + ) + + +def _upstream_failure( + error: Exception, request: Mapping[str, object] +) -> tuple[int, str, Sequence[tuple[str, str]]] | None: + """The status, text and headers Python's HTTP client reports for the same failure.""" + timeout_phase: Final = getattr(error, "messages_timeout", None) + if isinstance(timeout_phase, str): + return 408, _timeout_text(timeout_phase, request), () + try: + status, text = _UPSTREAM_ARGS.validate_python(error.args) + return status, text, _UPSTREAM_HEADERS.validate_python(getattr(error, "headers", None)) + except ValidationError: + return None + + +def _provider_error(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception | None: + """The provider exception Python's `_handle_error` builds for the same failure.""" + failure: Final = _upstream_failure(error, request) + if failure is None: + return None + status, text, headers = failure + api_base: Final = request.get("api_base") + config: Final = native_messages_config( + str(request["model"]).removeprefix(f"{request_provider}/"), + request_provider, + request.get("model_info"), + api_base if isinstance(api_base, str) else None, + ) + return ( + cast(_ERROR_CLASS, config.get_error_class)( # cast-ok: the config's untyped headers parameter + error_message=text, status_code=status, headers=dict(headers) + ) + if config is not None + else AnthropicError(status_code=status, message=text, headers=httpx.Headers(headers)) + ) + + +def _log_resolved_call(request: Mapping[str, object], request_provider: str) -> None: + """Tells the logger the resolved call, as Python's handler does before its provider config + transforms the request.""" + logger: Final = request.get("litellm_logging_obj") + if not isinstance(logger, LiteLLMLoggingObj): + return + callbacks_legacy_python.update_logging( + logger, + kwargs={name: value for name, value in request.items() if name != "metadata"}, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + optional_params=dict(body_params(request)), + litellm_params={"model_info": request.get("model_info"), **body_params(request)}, + custom_llm_provider=request_provider, + ) + + +def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: + """Raises what Python's Messages handler raises for the same failure.""" + if getattr(error, "messages_timeout", None) == "Body": + # Python reads a stream's body outside its HTTP client's error mapping, so the transport's + # own timeout reaches the caller unmapped. + return httpx.ReadTimeout(_BODY_READ_TIMEOUT) + if getattr(error, "messages_late_rejection", False): + _log_resolved_call(request, request_provider) + if getattr(error, "messages_request_error", False): + return _rejection(error, request, request_provider) + if getattr(error, "messages_missing_credential", False): + return litellm.AuthenticationError( + message=_MISSING_KEY, + llm_provider=request_provider, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + ) + provider_error: Final = _provider_error(error, request, request_provider) + if provider_error is None: + return failures.map_native_failure( + error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) + ) + try: + return _exception_type( + model=str(request["model"]), + custom_llm_provider=request_provider, + original_exception=provider_error, + extra_kwargs=dict(request), + ) + except Exception as mapped: # noqa: BLE001 # exception_type raises the mapped public error + mapped.__context__ = error + return mapped + + def _drop_params(kwargs: Mapping[str, object]) -> bool: return bool(litellm.drop_params) or normalize_drop_params(kwargs.get("drop_params")) is True diff --git a/litellm/secret_managers/aws_secret_manager.py b/litellm/secret_managers/aws_secret_manager.py index 88642ae141e..c87edc4b1dd 100644 --- a/litellm/secret_managers/aws_secret_manager.py +++ b/litellm/secret_managers/aws_secret_manager.py @@ -70,7 +70,7 @@ class AWSKeyManagementService_V2: is_litellm_license_in_env = True if is_litellm_license_in_env is False: raise ValueError( - "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) def load_aws_kms(self, use_aws_kms: bool | None): diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index 6bba56a5afe..2b5b4d52274 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -31,7 +31,7 @@ class TextCompletionStreamWrapper: def __aiter__(self): return self - def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream): + def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream) -> TextCompletionResponse: try: response: Final = TextCompletionResponse() response["id"] = chunk.get("id", None) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 33b2c677d31..3b28225f0ff 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -210,7 +210,7 @@ class NoulAnswer(LiteLLMPydanticObjectBase): class ChoiceAnswer(LiteLLMPydanticObjectBase): type: Literal["choice"] choice: str - confidence: float + confidence: float | None = None probabilities: Mapping[str, float] model_config = ConfigDict(extra="allow", frozen=True) @@ -219,7 +219,7 @@ class ChoiceAnswer(LiteLLMPydanticObjectBase): class ScoreAnswer(LiteLLMPydanticObjectBase): type: Literal["score"] score: float - confidence: float + confidence: float | None = None legend: Mapping[str, DecisionsJSON] probabilities: Mapping[str, float] @@ -431,7 +431,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): name: str | None choice: str | bool probabilities: tuple[OpenAIChoiceProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -449,7 +449,7 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): name: str | None score: float probabilities: tuple[OpenAIScoreProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -595,7 +595,7 @@ class DecisionsIRChoiceProbability: @dataclass(frozen=True, slots=True) class DecisionsIRChoiceAnswer: choice: str | bool - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRChoiceProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @@ -610,7 +610,7 @@ class DecisionsIRScoreProbability: @dataclass(frozen=True, slots=True) class DecisionsIRScoreAnswer: score: float - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRScoreProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 77e75dea030..14ed719a880 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -74,14 +74,13 @@ class AgenticLoopSafetyError(ValueError): Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so callers that already catch the broader type keep working. - - Only the anthropic messages loop raises this today. The chat completions - loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises - a plain ``ValueError`` from its own copy of the same rails, so catching - this type alone will not cover that surface until it is moved over. """ +class AgenticLoopLimitError(AgenticLoopSafetyError): + pass + + class StandardCustomLoggerInitParams(LiteLLMBaseModel): """ Params for initializing a CustomLogger. diff --git a/litellm/types/llms/custom_llm.py b/litellm/types/llms/custom_llm.py index 6b583b0a83d..bb12449617d 100644 --- a/litellm/types/llms/custom_llm.py +++ b/litellm/types/llms/custom_llm.py @@ -1,11 +1,12 @@ from typing import TYPE_CHECKING -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: + from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_llm import CustomLLM class CustomLLMItem(TypedDict): provider: str - custom_handler: "CustomLLM" + custom_handler: ReadOnly["CustomLLM | BaseVideoConfig"] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c803df04911..c9739eee91d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1247,9 +1247,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "languages", + "keywords", "prompt", "temperature", "response_format", + "stream", "timestamp_granularities", "include", ] @@ -2545,6 +2548,17 @@ class OpenAIRealtimeResponseUsage(TypedDict): output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] +class OpenAIRealtimeTranslationDurationUsage(TypedDict): + type: ReadOnly[Literal["duration"]] + output_seconds: NotRequired[ReadOnly[float]] + input_seconds: NotRequired[ReadOnly[float]] + + +class OpenAIRealtimeTranslationClosedEvent(TypedDict): + type: ReadOnly[Literal["session.closed"]] + usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2588,6 +2602,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated | OpenAIRealtimeErrorEvent + | OpenAIRealtimeTranslationClosedEvent ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 0d39b0a0616..1d3d5911eb1 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -23,6 +23,8 @@ from litellm.types.mcp import ( def _validate_mcp_protocol_metadata(value: dict[str, object]) -> dict[str, object]: if "protocol_version" in value: TypeAdapter[MCPUpstreamProtocol](MCPUpstreamProtocol).validate_python(value["protocol_version"]) + if "skip_tool_output_validation" in value: + TypeAdapter[bool](bool).validate_python(value["skip_tool_output_validation"], strict=True) return value @@ -297,6 +299,10 @@ class MCPServer(LiteLLMBaseModel): """ return self.oauth2_flow == "client_credentials" + @property + def skip_tool_output_validation(self) -> bool: + return (self.mcp_info or {}).get("skip_tool_output_validation") is True + @model_validator(mode="after") def resolve_protocol_version(self) -> Self: if "protocol_version" not in self.model_fields_set and self.mcp_info is not None: diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 19e40aa5c5d..49dd8b9ef91 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -4,7 +4,7 @@ Claude Code Marketplace endpoint types for LiteLLM Proxy from typing import Final, Literal, TypeAlias -from pydantic import AliasChoices, Field, TypeAdapter, model_validator +from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, model_validator from litellm.types.llms.base import LiteLLMBaseModel @@ -27,20 +27,11 @@ class PluginOwner(LiteLLMBaseModel): email: str | None = Field(None, description="Owner email") -class PluginSpec(LiteLLMBaseModel): +class PluginMetadata(LiteLLMBaseModel): """Mutable fields shared by plugin create and update requests.""" - source: dict[str, str] = Field( - ..., - description=( - "Plugin source reference. Supported formats:\n" - "- GitHub: {'source': 'github', 'repo': 'org/repo'}\n" - "- Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'}\n" - "- Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'}\n" - "- Zip archive on any https host (e.g. S3): " - "{'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''}" - ), - ) + model_config = ConfigDict(frozen=True) + version: str | None = Field("1.0.0", description="Semantic version") description: str | None = Field(None, description="Plugin description") author: PluginAuthor | None = Field(None, description="Plugin author") @@ -75,6 +66,20 @@ class PluginSpec(LiteLLMBaseModel): } +class PluginSpec(PluginMetadata): + source: dict[str, str] = Field( + ..., + description=( + "Plugin source reference. Supported formats:\n" + "- GitHub: {'source': 'github', 'repo': 'org/repo'}\n" + "- Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'}\n" + "- Git Subdir: {'source': 'git-subdir', 'url': 'https://github.com/org/repo.git', 'path': 'plugins/plugin-name'}\n" + "- Zip archive on any https host (e.g. S3): " + "{'source': 'archive', 'url': 'https://bucket.s3.amazonaws.com/plugin.zip', 'sha256': ''}" + ), + ) + + class RegisterPluginRequest(PluginSpec): """ Request body for registering a plugin in the marketplace. @@ -90,6 +95,10 @@ class RegisterPluginRequest(PluginSpec): ) +class UploadPluginRequest(PluginMetadata): + name: str = Field(..., pattern=r"^[a-z0-9-]+$", description="Skill name (kebab-case)") + + class UpdatePluginRequest(PluginSpec): """ Request body for replacing an existing plugin. diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py index 6f521d1fd1b..99ff7953ee2 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -16,7 +16,7 @@ class ToolPermissionRule(LiteLLMBaseModel): id: str = Field(description="Unique identifier for the rule") tool_name: str | None = Field( default=None, - description="Regex pattern applied to the tool's function name", + description="Regex pattern applied to the tool's name, or to its type when the tool has no name", ) tool_type: str | None = Field( default=None, diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 8645f89b65d..37c49d737af 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -1,9 +1,10 @@ from __future__ import annotations from datetime import date, datetime +from enum import Enum from typing import Final, Literal -from pydantic import Field +from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel @@ -16,8 +17,17 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30 DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] +class SuccessorModelSource(str, Enum): + MODEL_COST_MAP = "model_cost_map" + DEPLOYMENT_CONFIG = "deployment_config" + + class ModelDeprecationInfo(LiteLLMBaseModel): model_name: str = Field(description="The public name of the model on the proxy (model_group).") + team_public_model_name: str | None = Field( + default=None, + description="The model name visible to the team, when configured.", + ) litellm_model: str | None = Field( default=None, description="The underlying litellm model string the deprecation date is sourced from.", @@ -34,7 +44,18 @@ class ModelDeprecationInfo(LiteLLMBaseModel): litellm_provider: str | None = Field(default=None, description="The provider this model belongs to.") successor_model: str | None = Field( default=None, - description=("The model to migrate to, as set in model_info.successor_model on the deployment."), + description=( + "The model to migrate to: model_info.successor_model on the deployment when set, " + "else the successor_model the cost map names for the deprecated model." + ), + ) + + successor_model_source: SuccessorModelSource | None = Field( + default=None, + description="Whether the recommended successor comes from the model cost map or deployment configuration.", + ) + deprecation_source: str | None = Field( + default=None, description="Provider deprecation notice linked by the selected model cost-map entry." ) @@ -56,3 +77,38 @@ class ModelDeprecationResponse(LiteLLMBaseModel): ) warn_within_days: int = Field(description="The window (in days) used to bucket 'imminent' models.") checked_at: datetime = Field(description="UTC timestamp when the deprecation snapshot was generated.") + + +class DeprecationAlertStatus(str, Enum): + OPEN = "open" + RESOLVED = "resolved" + DISMISSED = "dismissed" + + +class DeprecationAlertUpdate(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str = Field(min_length=1, max_length=1024) + deprecation_date: date + status: DeprecationAlertStatus + last_seen_date: date + last_seen_requests: int = Field(ge=0) + + +class DeprecationAlertState(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + deprecation_date: date + status: DeprecationAlertStatus = DeprecationAlertStatus.OPEN + last_seen_date: date | None = None + last_seen_requests: int = 0 + updated_at: datetime | None = None + updated_by: str | None = None + + +class DeprecationTagUsage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + api_key: str + tag: str diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 6da73982291..4a9b4228bc8 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping, Sequence from typing import Any, Literal from pydantic import ConfigDict @@ -66,6 +67,41 @@ class RealtimeExpiresAfter(LiteLLMBaseModel): seconds: int | None = None +class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + model: str | None = None + delay: str | None = None + keywords: Sequence[str] | None = None + language: str | None = None + languages: Sequence[str] | None = None + prompt: str | None = None + + +class RealtimeAudioInputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + format: str | Mapping[str, object] | None = None + noise_reduction: Mapping[str, object] | None = None + transcription: RealtimeAudioTranscriptionConfig | None = None + turn_detection: Mapping[str, object] | None = None + + +class RealtimeAudioOutputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + format: str | Mapping[str, object] | None = None + language: str | None = None + voice: str | Mapping[str, object] | None = None + + +class RealtimeSessionAudioConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + input: RealtimeAudioInputConfig | None = None + output: RealtimeAudioOutputConfig | None = None + + class RealtimeSessionConfig(LiteLLMBaseModel): """ Session configuration nested inside the client_secrets request body. @@ -75,12 +111,12 @@ class RealtimeSessionConfig(LiteLLMBaseModel): Extra/unknown fields are passed through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) type: str | None = None model: str | None = None instructions: str | None = None - audio: dict[str, object] | None = None + audio: RealtimeSessionAudioConfig | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None @@ -129,17 +165,20 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel): unchanged to the provider. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None input_audio_transcription: dict[str, Any] | None = None + audio: RealtimeSessionAudioConfig | None = None def resolved_model(self) -> str | None: if self.model: return self.model if self.input_audio_transcription: return self.input_audio_transcription.get("model") + if self.audio and self.audio.input and self.audio.input.transcription: + return self.audio.input.transcription.model return None @@ -151,7 +190,7 @@ class RealtimeTranscriptionSessionResponse(LiteLLMBaseModel): ephemeral key. Unknown fields pass through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) client_secret: dict[str, object] | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 4e3e4d92b3c..3363d113fc7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1154,9 +1154,9 @@ class CustomRoutingStrategyBase: self, model: str, messages: list[dict[str, str]] | None = None, - input: str | list | None = None, + input: str | list[object] | None = None, specific_deployment: bool | None = False, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, ) -> None: """ Asynchronously retrieves the available deployment based on the given parameters. @@ -1177,9 +1177,9 @@ class CustomRoutingStrategyBase: self, model: str, messages: list[dict[str, str]] | None = None, - input: str | list | None = None, + input: str | list[object] | None = None, specific_deployment: bool | None = False, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, ) -> None: """ Synchronously retrieves the available deployment based on the given parameters. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 080ea9848b6..ba29a67a36c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -224,6 +224,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_adaptive_thinking: bool | None supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] + thinking_on_by_default: ReadOnly[bool | None] supports_tool_search: bool | None supports_mid_conversation_system: bool | None supports_url_context: bool | None @@ -383,6 +384,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None input_cost_per_audio_token_flex: ReadOnly[float | None] + input_cost_per_audio_token_above_200k_tokens: ReadOnly[float | None] + input_cost_per_audio_token_above_200k_tokens_flex: ReadOnly[float | None] + input_cost_per_audio_token_above_200k_tokens_priority: ReadOnly[float | None] + input_cost_per_audio_token_above_200k_tokens_batches: ReadOnly[float | None] input_cost_per_audio_token_priority: ReadOnly[float | None] input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models @@ -408,10 +413,12 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None] + input_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None] input_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None] input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None output_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None] + output_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None] output_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None] output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] @@ -425,6 +432,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): regional_processing_uplift_multiplier_us: ( float | None ) # OpenAI US data-residency uplift multiplier applied to all token costs (e.g. 1.10 = +10%) + fedramp_uplift_multiplier: ReadOnly[ + float | None + ] # OpenAI FedRAMP endpoint (gov.api.openai.com) uplift multiplier applied to all token costs (e.g. 1.10 = +10%) regional_endpoint_uplift_multiplier: ReadOnly[ float | None ] # Vertex AI non-global (regional) endpoint uplift multiplier applied to all token costs (e.g. 1.10 = +10%) @@ -444,6 +454,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_pixel: ReadOnly[float | None] output_cost_per_image_token: float | None output_cost_per_image_token_batches: ReadOnly[float | None] + output_cost_per_image_token_priority: ReadOnly[float | None] output_cost_per_video_token: float | None # for gemini omni models with video output output_vector_size: int | None output_cost_per_reasoning_token: float | None @@ -554,6 +565,11 @@ class CallTypes(str, Enum): aextraction = "aextraction" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" + acreate_realtime_client_secret = "acreate_realtime_client_secret" + arealtime_calls = "arealtime_calls" + acreate_realtime_transcription_session = "acreate_realtime_transcription_session" + acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret" + arealtime_translation_calls = "arealtime_translation_calls" create_batch = "create_batch" acreate_batch = "acreate_batch" aretrieve_batch = "aretrieve_batch" @@ -794,6 +810,8 @@ CallTypesLiteral = Literal[ "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ] MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -802,6 +820,10 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( # Mapping of API routes to their corresponding call types API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { + "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -1120,9 +1142,11 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API - "/realtime": [CallTypes.arealtime], - "/v1/realtime": [CallTypes.arealtime], - "/openai/v1/realtime": [CallTypes.arealtime], + "/realtime": (CallTypes.arealtime,), + "/v1/realtime": (CallTypes.arealtime,), + "/openai/v1/realtime": (CallTypes.arealtime,), + "/realtime/translations": (CallTypes.arealtime,), + "/v1/realtime/translations": (CallTypes.arealtime,), "/live/sessions": (CallTypes.arealtime,), "/v1/live/sessions": (CallTypes.arealtime,), "/openai/v1/live/sessions": (CallTypes.arealtime,), @@ -2951,9 +2975,19 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None +class TranscriptionDetectedLanguage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + code: str + + +_DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ...]) + + class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None + languages: Sequence[TranscriptionDetectedLanguage] | None = None _hidden_params: dict = {} _response_headers: dict | None = None @@ -2966,8 +3000,28 @@ class TranscriptionResponse(OpenAIObject): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __init__(self, text=None) -> None: - super().__init__(text=text) + def __init__( + self, + text: str | None = None, + usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | Mapping[str, object] | None = None, + languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None, + **kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields + ) -> None: + super().__init__( + text=text, usage=usage, languages=TranscriptionResponse.detected_languages(languages), **kwargs + ) + if self.languages is None: + del self.languages + + @staticmethod + def detected_languages(value: object) -> tuple[TranscriptionDetectedLanguage, ...] | None: + try: + return _DETECTED_LANGUAGES.validate_python(value, from_attributes=True) + except ValidationError: + return None + + def set_audio_transcription_duration(self, duration: float) -> None: + self._hidden_params["audio_transcription_duration"] = duration def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator @@ -4130,6 +4184,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_character_above_128k_tokens: float | None = None input_cost_per_audio_token: float | None = None input_cost_per_audio_token_flex: float | None = None + input_cost_per_audio_token_above_200k_tokens: float | None = None + input_cost_per_audio_token_above_200k_tokens_flex: float | None = None + input_cost_per_audio_token_above_200k_tokens_priority: float | None = None + input_cost_per_audio_token_above_200k_tokens_batches: float | None = None input_cost_per_audio_token_priority: float | None = None input_cost_per_token_cache_hit: float | None = None input_cost_per_token_above_128k_tokens: float | None = None @@ -4140,6 +4198,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_token_above_272k_tokens_flex: float | None = None input_cost_per_token_above_272k_tokens_ultrafast: float | None = None input_cost_per_token_above_200k_tokens_batches: float | None = None + input_cost_per_token_above_200k_tokens_flex: float | None = None input_cost_per_token_above_100k_tokens_batches: float | None = None input_cost_per_token_above_272k_tokens_batches: float | None = None input_cost_per_query: float | None = None @@ -4169,12 +4228,14 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_above_272k_tokens_flex: float | None = None output_cost_per_token_above_272k_tokens_ultrafast: float | None = None output_cost_per_token_above_200k_tokens_batches: float | None = None + output_cost_per_token_above_200k_tokens_flex: float | None = None output_cost_per_token_above_100k_tokens_batches: float | None = None output_cost_per_token_above_272k_tokens_batches: float | None = None output_cost_per_character_above_128k_tokens: float | None = None output_cost_per_image: float | None = None output_cost_per_image_token: float | None = None output_cost_per_image_token_batches: float | None = None + output_cost_per_image_token_priority: float | None = None output_cost_per_video_token: float | None = None output_cost_per_reasoning_token: float | None = None output_cost_per_reasoning_token_flex: float | None = None @@ -4201,6 +4262,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): annotation_cost_per_page_batches: float | None = None regional_processing_uplift_multiplier_eu: float | None = None regional_processing_uplift_multiplier_us: float | None = None + fedramp_uplift_multiplier: float | None = None regional_endpoint_uplift_multiplier: float | None = None @classmethod @@ -4241,6 +4303,10 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( } ) +DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset( + {"max_input_tokens", "max_output_tokens", "max_tokens"} +) + DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset( { "token_count_framing", @@ -4580,10 +4646,12 @@ class LlmProviders(str, Enum): CLF_AI_GATEWAY = "clf_ai_gateway" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" + PARALLEL_AI = "parallel_ai" NACE = "nace" STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + EMPIRIOLABS = "empiriolabs" EU_JEV = "eu_jev" MISTRAL = "mistral" MILVUS = "milvus" @@ -4712,6 +4780,7 @@ class LlmProviders(str, Enum): SAIL = "sail" REKA = "reka" STREAMLAKE = "streamlake" + WALLY = "wally" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" @@ -5197,11 +5266,13 @@ class DataResidency(Enum): Inferred from the OpenAI api_base host (eu.api.openai.com -> EU, us.api.openai.com -> US). Used to apply the regional-processing cost uplift (see ``regional_processing_uplift_multiplier_`` - on ModelInfo). + on ModelInfo). gov.api.openai.com maps to FEDRAMP, which applies + ``fedramp_uplift_multiplier`` instead. """ US = "us" EU = "eu" + FEDRAMP = "fedramp" LLMResponseTypes = ( diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index 0fc2b8ede1b..240e36ecdf6 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -17,10 +17,11 @@ VIDEO_ID_PREFIX: Final = "video_" VIDEO_ID_SERVED_MODEL_PREFIX: Final = "served_model:" VIDEO_ID_DEPLOYMENT_PREFIX: Final = "deployment_id:" VIDEO_ID_REQUESTED_MODEL_PREFIX: Final = "requested_model:" -VIDEO_ID_CALL_TYPES: Final = frozenset( +VIDEO_STATUS_CALL_TYPES: Final = frozenset({"avideo_status", "video_status"}) +VIDEO_COST_REQUEST_ID_SUFFIX: Final = "_video_cost" +POLLED_VIDEO_STATUS_KWARG: Final = "polled_video_status" +VIDEO_ID_CALL_TYPES: Final = VIDEO_STATUS_CALL_TYPES | frozenset( { - "avideo_status", - "video_status", "avideo_content", "video_content", "avideo_remix", @@ -31,6 +32,12 @@ VIDEO_ID_CALL_TYPES: Final = frozenset( "video_extension", } ) + + +def is_video_status_call_type(call_type: object) -> bool: + return isinstance(call_type, str) and call_type in VIDEO_STATUS_CALL_TYPES + + CHARACTER_ID_PREFIX: Final = "character_" CHARACTER_ID_TEMPLATE: Final = "litellm:custom_llm_provider:{};model_id:{};character_id:{}" @@ -94,6 +101,27 @@ def _trailing_part(parts: Sequence[str], prefix: str) -> str | None: return next((part.removeprefix(prefix) or None for part in parts[3:] if part.startswith(prefix)), None) +def _decoded_video_id_payload(encoded_video_id: str) -> str | None: + if not encoded_video_id.startswith(VIDEO_ID_PREFIX): + return None + try: + decoded_id: Final = base64.b64decode( + _add_base64_padding(encoded_video_id.removeprefix(VIDEO_ID_PREFIX)).encode("utf-8") + ).decode("utf-8") + except ValueError as e: + verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) + return None + return decoded_id if ";" in decoded_id else None + + +def canonical_video_id(video_id: str) -> str: + """One spelling per video, since a client may poll a LiteLLM video id with or without its base64 padding.""" + payload: Final = _decoded_video_id_payload(video_id) + if payload is None: + return video_id + return f"{VIDEO_ID_PREFIX}{base64.b64encode(payload.encode('utf-8')).decode('utf-8')}" + + def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: """Decode provider and model_id from encoded video_id.""" if not encoded_video_id: @@ -103,26 +131,8 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: video_id=encoded_video_id, ) - if not encoded_video_id.startswith(VIDEO_ID_PREFIX): - return DecodedVideoId( - custom_llm_provider=None, - model_id=None, - video_id=encoded_video_id, - ) - - try: - cleaned_id = encoded_video_id.replace(VIDEO_ID_PREFIX, "") - cleaned_id = _add_base64_padding(cleaned_id) - decoded_id: Final = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8") - except Exception as e: - verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) - return DecodedVideoId( - custom_llm_provider=None, - model_id=None, - video_id=encoded_video_id, - ) - - if ";" not in decoded_id: + decoded_id: Final = _decoded_video_id_payload(encoded_video_id) + if decoded_id is None: return DecodedVideoId( custom_llm_provider=None, model_id=None, diff --git a/litellm/utils.py b/litellm/utils.py index 5a0fc18dfdc..53b02ebddac 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,6 +83,7 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.allowed_tools_tool_choice import ( CHAT_ALLOWED_TOOLS_LAYOUT, parse_chat_allowed_tools_tool_choice, @@ -467,6 +468,7 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, @@ -1123,6 +1125,38 @@ def _is_streaming_response_for_correlation(result: object) -> bool: return isinstance(result, CustomStreamWrapper) +def _returns_requested_stream(kwargs: dict[str, object], call_type: CallTypes | str, result: object) -> bool: + if call_type in (CallTypes.transcription, CallTypes.atranscription): + return isinstance(result, (openai.Stream, openai.AsyncStream)) + return _is_streaming_request(kwargs=kwargs, call_type=call_type) + + +def _transcription_runs_unstreamed( + call_type: str, requested_kwargs: Mapping[str, object], call_kwargs: Mapping[str, object] +) -> bool: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested + + return ( + call_type == CallTypes.atranscription.value + and transcription_stream_requested(requested_kwargs.get("stream")) + and call_kwargs.get("stream") is False + ) + + +def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> dict[str, object]: + from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming + + return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True) + + +def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay + + if not runs_unstreamed or not isinstance(result, TranscriptionResponse): + return result + return GuardedTranscriptionReplay(result) + + def _is_converted_stream_result(result: object) -> bool: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -1558,7 +1592,6 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) - ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} if "metadata" in kwargs: @@ -1603,7 +1636,6 @@ def dispatch_success_logging( is_litellm_internal_call: bool, ) -> None: if is_litellm_internal_call: - logging_obj.capture_usage_before_logging(result, start_time, end_time) return _schedule_async_success_logging( @@ -1642,14 +1674,9 @@ def _schedule_async_success_logging( if not logging_obj.claim_async_success_log(): return - if logging_obj.defer_async_logging and not is_completion_with_fallbacks: - logging_obj.capture_usage_before_logging(result, start_time, end_time, freeze_only=True) - def _enqueue_async_logging() -> None: from litellm.responses.streaming_iterator import spawn_logging_task - if not is_completion_with_fallbacks: - logging_obj.capture_usage_before_logging(result, start_time, end_time) if logging_obj.defer_async_logging: logging_obj.handle_sync_success_callbacks_for_async_calls( result=result, start_time=start_time, end_time=end_time @@ -1806,7 +1833,28 @@ async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type if result is not None: modified_kwargs = result - return modified_kwargs + return _unstreamed_when_post_call_guardrails_apply(modified_kwargs, typed_call_type) + + +def _unstreamed_when_post_call_guardrails_apply( + kwargs: dict[str, object], call_type: CallTypes | None +) -> dict[str, object]: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested + + if ( + call_type is not CallTypes.atranscription + or not transcription_stream_requested(kwargs.get("stream")) + or _deployment_provider(kwargs) not in (LlmProviders.OPENAI.value, LlmProviders.AZURE.value) + ): + return kwargs + streamed_request: Final = _guardrail_view_of_requested_stream(kwargs) + custom_guardrail_class: Final = _get_cached_custom_guardrail() + if not any( + isinstance(callback, custom_guardrail_class) and callback.runs_post_call_deployment_hook(streamed_request) + for callback in litellm.callbacks + ): + return kwargs + return {**kwargs, "stream": False} async def async_post_call_success_deployment_hook( @@ -2154,7 +2202,7 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): + if _returns_requested_stream(kwargs, call_type, result) or _is_converted_stream_result(result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): @@ -2163,6 +2211,15 @@ def client(original_function): else: logging_obj.stream = True logging_obj.model_call_details["stream"] = True + if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, + ) + + result = wrap_transcription_stream( + result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details) + ) # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2217,7 +2274,6 @@ def client(original_function): # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated verbose_logger.info("Wrapper: Completed Call, calling success_handler") - logging_obj.capture_usage_before_logging(TypeAdapter(object).validate_python(result), start_time, end_time) if not is_internal_call.get(): # Copy the current context to propagate it to the background thread # This is essential for OpenTelemetry span context propagation @@ -2250,7 +2306,7 @@ def client(original_function): is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_completion_litellm_router_call + num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2284,7 +2340,7 @@ def client(original_function): is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_responses_litellm_router_call + num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2338,6 +2394,7 @@ def client(original_function): kwargs["litellm_logging_obj"] = logging_obj modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type) + transcription_runs_unstreamed: Final = _transcription_runs_unstreamed(call_type, kwargs, modified_kwargs) if modified_kwargs is not None: kwargs = modified_kwargs @@ -2388,7 +2445,9 @@ def client(original_function): if _is_converted_stream_result(_caching_handler_response.cached_result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True - return _caching_handler_response.cached_result + return _replayed_when_run_unstreamed( + _caching_handler_response.cached_result, transcription_runs_unstreamed + ) elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response @@ -2451,7 +2510,7 @@ def client(original_function): raise end_time = datetime.datetime.now() - streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type) + streaming_requested: Final = _returns_requested_stream(kwargs, call_type, result) if streaming_requested or _is_converted_stream_result(result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True @@ -2465,6 +2524,15 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, + ) + + result = wrap_transcription_stream( + result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details) + ) _update_response_metadata( result=result, logging_obj=logging_obj, @@ -2491,7 +2559,9 @@ def client(original_function): _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: result = await async_post_call_success_deployment_hook( - request_data=call_kwargs, + request_data=_guardrail_view_of_requested_stream(call_kwargs) + if transcription_runs_unstreamed + else call_kwargs, response=result, call_type=_call_type_enum, ) @@ -2542,7 +2612,7 @@ def client(original_function): is_litellm_internal_call=_is_litellm_internal_call, ) - return result + return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed) except Exception as e: traceback_exception: Final = traceback.format_exc() # Reuse the timestamp taken right when the deployment call itself failed, before @@ -2569,7 +2639,10 @@ def client(original_function): is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_acompletion_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2600,7 +2673,10 @@ def client(original_function): is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_aresponses_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -4150,6 +4226,54 @@ def _register_on_current_map( _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload +_runtime_registration_order: Final[dict[str, int]] = {} # mutable-ok: tells which of a row's registered names came last +_runtime_registered_names: Final[dict[str, tuple[str, ...]]] = {} # mutable-ok: names registered onto each live row +_REGISTRATION_TICKS: Final = itertools.count() + + +@dataclass(frozen=True, slots=True) +class _RegistrationsOverCatalog(Mapping[str, Mapping[str, object]]): + catalog: Mapping[str, Mapping[str, object]] + registrations: Mapping[str, Mapping[str, object]] + registration_order: Mapping[str, int] + registered_names: Mapping[str, tuple[str, ...]] + + def __getitem__(self, key: str) -> Mapping[str, object]: + catalog_entry: Final = self.catalog.get(key) + names: Final = dict.fromkeys((*self.registered_names.get(key, ()), key)) + present: Final = tuple(name for name in names if name in self.registrations) + if not present: + return self.catalog[key] + oldest_first: Final = sorted(present, key=lambda name: self.registration_order.get(name, -1)) + registered: Final = (self.registrations[name] for name in oldest_first) + return MappingProxyType( + dict(itertools.chain.from_iterable(entry.items() for entry in (catalog_entry or {}, *registered))) + ) + + def __iter__(self) -> Iterator[str]: + return iter({**dict.fromkeys(self.catalog), **dict.fromkeys(self.registrations)}) + + def __len__(self) -> int: + return len(self.catalog.keys() | self.registrations.keys()) + + +def _record_registered_name(written_key: str, registered_key: str) -> None: + names: Final = _runtime_registered_names.get(written_key, ()) + if registered_key not in names: + _runtime_registered_names[written_key] = (*names, registered_key) + + +def catalog_with_runtime_registrations() -> Mapping[str, Mapping[str, object]]: + """The loaded catalog plus ``register_model`` calls, without what routers register for their deployments.""" + catalog: Final = GetModelCostMap.loaded_model_cost_map() + if not _runtime_registered_model_cost: + return catalog + return _RegistrationsOverCatalog( + catalog, + MappingProxyType(_runtime_registered_model_cost), + MappingProxyType(_runtime_registration_order), + MappingProxyType(_runtime_registered_names), + ) def runtime_model_cost_registrations( @@ -4181,6 +4305,13 @@ class _LiveDeploymentReplay: """ callback: Callable[[], None] | None = None + # Replays other router cores add beside the slot, run after it. + extra: tuple[Callable[[], None], ...] = () + + +def add_live_deployment_replay(replay: Callable[[], None]) -> None: + """Add a callback that re-asserts another router core's live deployments after a refresh.""" + _LiveDeploymentReplay.extra = (*_LiveDeploymentReplay.extra, replay) def set_live_deployment_replay(replay: Callable[[], None]) -> None: @@ -4214,8 +4345,17 @@ def reapply_runtime_model_cost_registrations() -> None: """ if _LiveDeploymentReplay.callback is not None: _LiveDeploymentReplay.callback() + for extra_replay in _LiveDeploymentReplay.extra: + extra_replay() if _runtime_registered_model_cost: - register_model(model_cost=dict(_runtime_registered_model_cost)) + register_model( + model_cost=dict( + sorted( + _runtime_registered_model_cost.items(), + key=lambda registration: _runtime_registration_order.get(registration[0], -1), + ) + ) + ) def cost_map_omits_token_price(*keys: object) -> bool: @@ -4289,9 +4429,12 @@ def register_model( dict(_runtime_registered_model_cost.get(_registered_key, {})), _RATE_TABLE.validate_python(_registered_value), ) + _runtime_registration_order[_registered_key] = next(_REGISTRATION_TICKS) for key, value in loaded_model_cost.items(): write = _register_on_current_map(str(key), value, custom_llm_provider, warning_display_name) + if persist_across_reloads: + _record_registered_name(write.key, str(key)) # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map() @@ -4332,40 +4475,61 @@ def _get_non_default_params(passed_params: dict, default_params: dict, additiona return non_default_params +_OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS: Final = frozenset(("languages", "keywords", "stream")) + + def get_optional_params_transcription( model: str, custom_llm_provider: str, language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: str | None = None, temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, + stream: bool | None = None, drop_params: bool | None = None, **kwargs, ): from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS # retrieve all parameters passed to the function - passed_params: Final = locals() + passed_params: Final = locals().copy() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") - passed_params.pop("custom_llm_provider") - passed_params.pop("drop_params") - drop_params = normalize_drop_params(drop_params) + passed_params.pop("model") + custom_llm_provider = passed_params.pop("custom_llm_provider") + drop_params = normalize_drop_params(passed_params.pop("drop_params")) special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + transcription_stream_flag, + transcription_stream_unset, + ) + for k, v in special_params.items(): passed_params[k] = v - default_params: Final = { - "language": None, - "prompt": None, - "response_format": None, - "temperature": None, # openai defaults this to 0 - "timestamp_granularities": None, - } + sends_through_openai_sdk: Final = custom_llm_provider in ("openai", "azure") + drops_unsupported_params: Final = drop_params is True or litellm.drop_params is True + transcription_params: Final = ( + OPENAI_TRANSCRIPTION_PARAMS + if sends_through_openai_sdk or drops_unsupported_params + else [param for param in OPENAI_TRANSCRIPTION_PARAMS if param not in _OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS] + ) + default_params: Final = {param: None for param in (*transcription_params, "prompt", "temperature")} + stream_unset: Final = transcription_stream_unset(stream) + malformed_stream: Final = not stream_unset and transcription_stream_flag(stream) is None + if malformed_stream and not drops_unsupported_params: + raise UnsupportedParamsError( + status_code=400, + message="`stream` must be a boolean. To drop it from the call, set `litellm.drop_params = True`.", + ) non_default_params: Final = { - k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + k: v + for k, v in passed_params.items() + if k in default_params and v != default_params[k] and not (k == "stream" and (stream_unset or malformed_stream)) } optional_params = {} @@ -4374,9 +4538,7 @@ def get_optional_params_transcription( if len(non_default_params.keys()) > 0: keys: Final = list(non_default_params.keys()) for k in keys: - if ( - drop_params is True or litellm.drop_params is True - ) and k not in supported_params: # drop the unsupported non-default values + if drops_unsupported_params and k not in supported_params: non_default_params.pop(k, None) elif k not in supported_params: raise UnsupportedParamsError( @@ -4430,9 +4592,12 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else transcription_params, additional_drop_params=kwargs.get("additional_drop_params", None), ) + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict) and not extra_body: + optional_params.pop("extra_body") return optional_params @@ -6005,6 +6170,14 @@ EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"}) LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) +def header_auth_params(custom_llm_provider: str) -> frozenset[str]: + if custom_llm_provider != "azure_ai": + return frozenset() + from litellm.llms.azure_ai.common_utils import AZURE_ENTRA_CREDENTIAL_PARAM_KEYS + + return AZURE_ENTRA_CREDENTIAL_PARAM_KEYS + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -6039,8 +6212,10 @@ def add_provider_specific_params_to_optional_params( **extra_body, } - dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset( - param for param in (additional_drop_params or ()) if isinstance(param, str) + dropped_keys: Final = ( + EXTRA_BODY_ROUTING_KEYS + | header_auth_params(custom_llm_provider) + | frozenset(param for param in (additional_drop_params or ()) if isinstance(param, str)) ) ensure_extra_body_is_safe: Final = litellm_utils.ensure_extra_body_is_safe optional_params["extra_body"] = ensure_extra_body_is_safe( @@ -6172,7 +6347,15 @@ def get_deployment_order(deployment: Mapping[str, object]) -> int | None: resolved_order: Final = ( order if order is not None else cast(Mapping[str, object], deployment.get("model_info", {})).get("order") ) - return cast(int | None, resolved_order) + # One number for every spelling (`1`, `1.0`, `"1"`, `True`), so orders written different ways + # sort and compare together; anything else counts as no order. + if isinstance(resolved_order, bool | int): + return int(resolved_order) + if isinstance(resolved_order, float): + return int(resolved_order) if resolved_order.is_integer() else None + if isinstance(resolved_order, str) and re.fullmatch(r"[+-]?[0-9]+", resolved_order): + return int(resolved_order) + return None _get_deployment_order = get_deployment_order @@ -7181,6 +7364,7 @@ def get_model_info_helper( "cache_read_input_audio_token_cost_batches", None ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), + cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None @@ -7278,6 +7462,18 @@ def get_model_info_helper( input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None), + input_cost_per_audio_token_above_200k_tokens=_model_info.get( + "input_cost_per_audio_token_above_200k_tokens", None + ), + input_cost_per_audio_token_above_200k_tokens_flex=_model_info.get( + "input_cost_per_audio_token_above_200k_tokens_flex", None + ), + input_cost_per_audio_token_above_200k_tokens_priority=_model_info.get( + "input_cost_per_audio_token_above_200k_tokens_priority", None + ), + input_cost_per_audio_token_above_200k_tokens_batches=_model_info.get( + "input_cost_per_audio_token_above_200k_tokens_batches", None + ), input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), @@ -7291,6 +7487,9 @@ def get_model_info_helper( input_cost_per_token_above_200k_tokens_batches=_model_info.get( "input_cost_per_token_above_200k_tokens_batches" ), + input_cost_per_token_above_200k_tokens_flex=_model_info.get( + "input_cost_per_token_above_200k_tokens_flex", None + ), input_cost_per_token_above_272k_tokens_batches=_model_info.get( "input_cost_per_token_above_272k_tokens_batches" ), @@ -7298,6 +7497,9 @@ def get_model_info_helper( output_cost_per_token_above_200k_tokens_batches=_model_info.get( "output_cost_per_token_above_200k_tokens_batches" ), + output_cost_per_token_above_200k_tokens_flex=_model_info.get( + "output_cost_per_token_above_200k_tokens_flex", None + ), output_cost_per_token_above_272k_tokens_batches=_model_info.get( "output_cost_per_token_above_272k_tokens_batches" ), @@ -7312,6 +7514,7 @@ def get_model_info_helper( regional_processing_uplift_multiplier_us=_model_info.get( "regional_processing_uplift_multiplier_us", None ), + fedramp_uplift_multiplier=_model_info.get("fedramp_uplift_multiplier", None), regional_endpoint_uplift_multiplier=_model_info.get("regional_endpoint_uplift_multiplier", None), output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None), output_cost_per_character=_model_info.get("output_cost_per_character", None), @@ -7359,6 +7562,7 @@ def get_model_info_helper( output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None), output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None), + output_cost_per_image_token_priority=_model_info.get("output_cost_per_image_token_priority", None), output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None), output_vector_size=_model_info.get("output_vector_size", None), citation_cost_per_token=_model_info.get("citation_cost_per_token", None), @@ -7394,6 +7598,7 @@ def get_model_info_helper( supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), + thinking_on_by_default=_model_info.get("thinking_on_by_default", None), supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), @@ -8902,6 +9107,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]: return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {}))) +AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM" + + def get_valid_models( check_provider_endpoint: bool | None = None, custom_llm_provider: str | None = None, @@ -8957,7 +9165,7 @@ def get_valid_models( continue if provider == "azure": - valid_models.append("Azure-LLM") + valid_models.append(AZURE_MODEL_PLACEHOLDER) elif provider_config is not None and check_provider_endpoint and provider is not None: valid_models.extend( _get_valid_models_from_provider_api( @@ -10105,12 +10313,21 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): + if provider in ( + LlmProviders.EMPIRIOLABS, + LlmProviders.OPENAI, + LlmProviders.AZURE, + LlmProviders.AZURE_AI, + LlmProviders.EU_JEV, + LlmProviders.PARALLEL_AI, + ): configs: Final = { + LlmProviders.EMPIRIOLABS: litellm.EmpirioLabsDecisionsConfig, LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, + LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig, } return configs[provider]() return None @@ -10231,6 +10448,7 @@ class ProviderConfigManager: def get_provider_audio_transcription_config( model: str, provider: LlmProviders, + base_model: str | None = None, ) -> BaseAudioTranscriptionConfig | None: model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, @@ -10258,7 +10476,13 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): + if "gpt-transcribe" in (model, base_model): + from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, + ) + + return OpenAIGPTTranscribeAudioTranscriptionConfig() + elif "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() @@ -10558,12 +10782,14 @@ class ProviderConfigManager: from litellm.llms.ollama.common_utils import OllamaModelInfo return OllamaModelInfo() - elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.vllm.common_utils import ( - VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py - ) + elif LlmProviders.VLLM == provider: + from litellm.llms.vllm.common_utils import VLLMModelInfo return VLLMModelInfo() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + return HostedVLLMModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: @@ -10949,8 +11175,13 @@ class ProviderConfigManager: @staticmethod def get_provider_video_config( model: str | None, - provider: LlmProviders, + provider: LlmProviders | str, ) -> BaseVideoConfig | None: + if not isinstance(provider, LlmProviders): + custom_video_config: Final = ProviderConfigManager.get_custom_provider_video_config(provider) + if custom_video_config is not None or provider not in LlmProvidersSet: + return custom_video_config + return ProviderConfigManager.get_provider_video_config(model=model, provider=LlmProviders(provider)) if LlmProviders.OPENAI == provider: from litellm.llms.openai.videos.transformation import OpenAIVideoConfig @@ -11001,6 +11232,34 @@ class ProviderConfigManager: return VolcEngineVideoConfig() return None + @staticmethod + def get_custom_llm_handler(provider: str | None) -> CustomLLM | None: + """The `CustomLLM` registered for `provider` in `litellm.custom_provider_map`, if its handler is one.""" + from litellm.llms.custom_llm import CustomLLM + + return next( + ( + item["custom_handler"] + for item in litellm.custom_provider_map + if item["provider"] == provider and isinstance(item["custom_handler"], CustomLLM) + ), + None, + ) + + @staticmethod + def get_custom_provider_video_config(provider: str) -> BaseVideoConfig | None: + """The `BaseVideoConfig` registered for `provider` in `litellm.custom_provider_map`, if its handler is one.""" + from litellm.llms.base_llm.videos.transformation import BaseVideoConfig + + return next( + ( + item["custom_handler"] + for item in litellm.custom_provider_map + if item["provider"] == provider and isinstance(item["custom_handler"], BaseVideoConfig) + ), + None, + ) + @staticmethod def get_provider_container_config( provider: LlmProviders, diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 85e651b687c..66eb10e30db 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -26,6 +26,17 @@ from litellm.types.videos.utils import decode_video_id_with_provider from litellm.utils import ProviderConfigManager, client from litellm.videos.utils import VideoGenerationRequestUtils + +def _unsupported_video_call( + call_name: str, custom_llm_provider: str, model: str | None = None +) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"{call_name} is not supported for {custom_llm_provider}", + model=model or "", + llm_provider=custom_llm_provider, + ) + + #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() @@ -40,7 +51,7 @@ async def avideo_generation( size: str | None = None, user: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -170,7 +181,7 @@ def video_generation( size: str | None = None, user: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -208,11 +219,11 @@ def video_generation( # get provider config video_generation_provider_config: BaseVideoConfig | None = ProviderConfigManager.get_provider_video_config( model=model, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_generation_provider_config is None: - raise ValueError(f"video generation is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video generation", custom_llm_provider, model) local_vars.update(kwargs) # Get VideoGenerationOptionalRequestParams with only valid parameters @@ -329,11 +340,11 @@ def video_content( # get provider config video_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_provider_config is None: - raise ValueError(f"video support download is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video support download", custom_llm_provider) local_vars.update(kwargs) # For video content download, we don't need complex optional parameter handling @@ -456,7 +467,7 @@ async def avideo_remix( video_id: str, prompt: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -559,7 +570,7 @@ def video_remix( video_id: str, prompt: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -598,11 +609,11 @@ def video_remix( # get provider config video_remix_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_remix_provider_config is None: - raise ValueError(f"video remix is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video remix", custom_llm_provider) local_vars.update(kwargs) # For video remix, we need the video_id and prompt @@ -660,7 +671,7 @@ async def avideo_list( order: str | None = None, api_key: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -774,7 +785,7 @@ def video_list( limit: int | None = None, order: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -810,11 +821,11 @@ def video_list( # get provider config video_list_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_list_provider_config is None: - raise ValueError(f"video list is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video list", custom_llm_provider) local_vars.update(kwargs) # For video list, we need the query parameters @@ -871,7 +882,7 @@ def video_list( async def avideo_status( video_id: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -970,7 +981,7 @@ def video_status( def video_status( video_id: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -1031,11 +1042,11 @@ def video_status( # get provider config video_status_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_status_provider_config is None: - raise ValueError(f"video status is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video status", custom_llm_provider) local_vars.update(kwargs) # For video status, we need the video_id @@ -1088,7 +1099,7 @@ async def avideo_create_character( name: str, video: FileTypes, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1143,7 +1154,7 @@ def video_create_character( name: str, video: FileTypes, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1172,11 +1183,11 @@ def video_create_character( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video create character is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video create character", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"name": name} @@ -1218,7 +1229,7 @@ def video_create_character( async def avideo_get_character( character_id: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1268,7 +1279,7 @@ async def avideo_get_character( def video_get_character( character_id: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1297,11 +1308,11 @@ def video_get_character( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video get character is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video get character", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"character_id": character_id} @@ -1343,7 +1354,7 @@ async def avideo_edit( video_id: str, prompt: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1398,7 +1409,7 @@ def video_edit( video_id: str, prompt: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1430,11 +1441,11 @@ def video_edit( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video edit is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video edit", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"video_id": video_id, "prompt": prompt} @@ -1480,7 +1491,7 @@ async def avideo_extension( prompt: str, seconds: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1534,7 +1545,7 @@ def video_extension( prompt: str, seconds: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1564,11 +1575,11 @@ def video_extension( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video extension is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video extension", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 032b7d17423..76fd85d001b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,172 @@ { + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -40,6 +208,7 @@ }, "1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -189,6 +358,7 @@ }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md", "input_cost_per_token": 2e-06, "litellm_provider": "bedrock", "max_input_tokens": 256000, @@ -199,6 +369,7 @@ }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, @@ -368,6 +539,7 @@ }, "amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -376,6 +548,7 @@ }, "us.amazon.nova-canvas-v1:0": { "deprecation_date": "2026-09-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", @@ -384,6 +557,7 @@ }, "us.writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -396,6 +570,7 @@ }, "us.writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -408,6 +583,7 @@ }, "writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -420,6 +596,7 @@ }, "writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -800,6 +977,7 @@ }, "twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -814,6 +992,7 @@ }, "us.twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -828,6 +1007,7 @@ }, "eu.twelvelabs.marengo-embed-2-7-v1:0": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_query": 7e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, @@ -881,6 +1061,7 @@ }, "twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -890,6 +1071,7 @@ }, "us.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -899,6 +1081,7 @@ }, "eu.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -908,6 +1091,7 @@ }, "global.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", @@ -1011,7 +1195,8 @@ "supports_native_streaming": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": {"model":"anthropic.claude-haiku-4-5-20251001-v1:0","litellm_provider":"bedrock"} }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -1111,6 +1296,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -1939,6 +2125,7 @@ "anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1971,12 +2158,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2025,6 +2214,7 @@ "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -2057,12 +2247,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2111,6 +2303,7 @@ "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2143,12 +2336,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2197,6 +2392,7 @@ "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2229,12 +2425,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2278,6 +2476,7 @@ "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2310,12 +2509,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2364,6 +2565,7 @@ "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -2396,7 +2598,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2760,6 +2964,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2804,6 +3009,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2848,6 +3054,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2892,6 +3099,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2936,6 +3144,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -2980,6 +3189,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -3233,6 +3443,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -3276,6 +3488,8 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -3510,6 +3724,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "apac.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -3565,6 +3781,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "au.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -3598,6 +3816,7 @@ }, "azure/ada": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -3608,6 +3827,7 @@ "azure/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-06, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -3635,19 +3855,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/command-r-plus": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true - }, "azure_ai/claude-haiku-4-5": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-haiku-5-5", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -3671,6 +3882,8 @@ }, "azure_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-opus-5-5", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -3695,6 +3908,7 @@ }, "azure_ai/claude-opus-4-6": { "deprecation_date": "2027-02-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_legacy_thinking": true, "input_cost_per_token": 5e-06, @@ -3726,6 +3940,7 @@ }, "azure_ai/claude-opus-4-7": { "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3757,6 +3972,7 @@ }, "azure_ai/claude-fable-5": { "deprecation_date": "2027-12-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -3785,7 +4001,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3819,14 +4035,47 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-12-05" + "deprecation_date": "2027-12-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" + }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, "litellm_provider": "azure_ai", @@ -3893,6 +4142,7 @@ }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, @@ -3925,6 +4175,8 @@ }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-11-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/claude-sonnet-5-5", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3948,6 +4200,7 @@ }, "azure_ai/claude-sonnet-5": { "deprecation_date": "2027-06-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -3964,6 +4217,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -3980,6 +4234,7 @@ }, "azure_ai/claude-sonnet-4-6": { "deprecation_date": "2027-02-10", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, @@ -4040,6 +4295,7 @@ }, "azure_ai/gpt-oss-120b": { "deprecation_date": "2027-09-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "litellm_provider": "azure_ai", @@ -4295,6 +4551,7 @@ }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -4342,6 +4599,7 @@ "azure_ai/gpt-chat-latest": { "cache_read_input_token_cost": 5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -4376,6 +4634,7 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", @@ -4391,6 +4650,7 @@ "azure_ai/codex-mini": { "cache_read_input_token_cost": 3.75e-07, "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, @@ -4420,6 +4680,7 @@ }, "azure_ai/whisper": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_second": 0.0001, "litellm_provider": "azure_ai", "mode": "audio_transcription", @@ -4473,6 +4734,7 @@ }, "azure_ai/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -4519,6 +4781,8 @@ }, "azure_ai/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -4566,6 +4830,8 @@ }, "azure_ai/gpt-5.4-2026-03-05": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -4613,6 +4879,7 @@ }, "azure_ai/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -4658,6 +4925,7 @@ }, "azure_ai/gpt-5.4-pro-2026-03-05": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -4703,6 +4971,7 @@ }, "azure_ai/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -4744,6 +5013,7 @@ }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -4785,6 +5055,7 @@ }, "azure_ai/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -4826,6 +5097,7 @@ }, "azure_ai/gpt-5.4-nano-2026-03-17": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -4867,6 +5139,7 @@ }, "azure_ai/model_router": { "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", @@ -4878,6 +5151,7 @@ }, "azure_ai/model-router": { "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.4e-07, "output_cost_per_token": 0, "litellm_provider": "azure_ai", @@ -4889,6 +5163,8 @@ }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, @@ -4908,6 +5184,8 @@ }, "azure/eu/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -4927,6 +5205,8 @@ }, "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -4947,6 +5227,7 @@ "azure/eu/gpt-4o-mini": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -4970,6 +5251,7 @@ "azure/eu/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -5008,26 +5290,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/eu/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2.2e-05, - "cache_read_input_token_cost": 2.75e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.00011, - "input_cost_per_token": 5.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.00022, - "output_cost_per_token": 2.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/eu/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_audio_token_cost": 2.5e-06, "cache_read_input_token_cost": 2.75e-06, @@ -5059,6 +5321,8 @@ "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, @@ -5102,7 +5366,9 @@ "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_priority": 4.95e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, "input_cost_per_token_priority": 4.95e-07, @@ -5144,7 +5410,9 @@ "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -5192,6 +5460,8 @@ }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2-codex", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -5223,6 +5493,7 @@ }, "azure/eu/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -5255,6 +5526,8 @@ "azure/eu/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -5296,6 +5569,7 @@ "azure/eu/o1": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -5314,6 +5588,7 @@ "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -5348,6 +5623,8 @@ "azure/eu/o3-mini": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -5365,6 +5642,8 @@ "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -5382,6 +5661,7 @@ "azure/global-standard/gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", @@ -5401,6 +5681,7 @@ "azure/global-standard/gpt-4o-2024-11-20": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", @@ -5418,6 +5699,7 @@ }, "azure/global-standard/gpt-4o-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -5437,6 +5719,8 @@ }, "azure/global/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/global/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -5456,6 +5740,8 @@ }, "azure/global/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/global/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -5474,7 +5760,8 @@ "supports_vision": true }, "azure/global/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -5522,6 +5809,7 @@ }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5553,6 +5841,7 @@ }, "azure/global/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5611,29 +5900,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-16k": { - "deprecation_date": "2025-04-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 16385, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-16k-0613": { - "deprecation_date": "2025-04-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure", - "max_input_tokens": 16385, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-instruct": { "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", @@ -5652,18 +5918,6 @@ "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" }, - "azure/gpt-4": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 3e-05, - "litellm_provider": "azure", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-4-0125-preview": { "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -5675,18 +5929,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-4-0613": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 3e-05, - "litellm_provider": "azure", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-4-1106-preview": { "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -5698,28 +5940,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-4-32k": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 6e-05, - "litellm_provider": "azure", - "max_input_tokens": 32768, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, - "azure/gpt-4-32k-0613": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 6e-05, - "litellm_provider": "azure", - "max_input_tokens": 32768, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_tool_choice": true - }, "azure/gpt-4-turbo": { "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -5758,6 +5978,8 @@ }, "azure/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -5800,6 +6022,8 @@ }, "azure/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -5842,6 +6066,8 @@ }, "azure/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -5884,6 +6110,8 @@ }, "azure/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -5926,6 +6154,8 @@ }, "azure/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -5959,6 +6189,8 @@ }, "azure/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -5990,28 +6222,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-4.5-preview": { - "cache_read_input_token_cost": 3.75e-05, - "deprecation_date": "2025-07-14", - "input_cost_per_token": 7.5e-05, - "input_cost_per_token_batches": 3.75e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 0.00015, - "output_cost_per_token_batches": 7.5e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6030,7 +6244,9 @@ "supports_vision": true }, "azure/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "azure", @@ -6048,6 +6264,8 @@ }, "azure/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6067,6 +6285,8 @@ }, "azure/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -6086,6 +6306,7 @@ }, "azure/gpt-audio": { "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6119,6 +6340,7 @@ }, "azure/gpt-audio-2025-08-28": { "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6151,6 +6373,7 @@ }, "azure/gpt-audio-1.5": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6184,6 +6407,7 @@ }, "azure/gpt-audio-1.5-2026-02-23": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6218,6 +6442,7 @@ }, "azure/gpt-audio-mini": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "azure", @@ -6250,6 +6475,7 @@ }, "azure/gpt-audio-mini-2025-10-06": { "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "azure", @@ -6314,6 +6540,7 @@ }, "azure/gpt-4o-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -6339,6 +6566,7 @@ "azure/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", @@ -6415,6 +6643,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6424,6 +6653,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6449,6 +6681,7 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6458,6 +6691,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6482,6 +6718,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1-2026-07-07", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6491,6 +6729,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6516,6 +6757,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1-2026-07-07", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6525,6 +6768,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6547,8 +6793,10 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -6556,6 +6804,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6575,14 +6826,18 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6590,6 +6845,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6609,6 +6867,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6617,6 +6877,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6626,6 +6887,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6650,6 +6914,7 @@ "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -6659,6 +6924,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6680,6 +6948,7 @@ }, "azure/gpt-4o-mini-transcribe": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -6693,6 +6962,7 @@ }, "azure/gpt-4o-mini-tts": { "deprecation_date": "2027-06-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 6e-07, "litellm_provider": "azure", "mode": "audio_speech", @@ -6710,26 +6980,6 @@ "audio" ] }, - "azure/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2e-05, - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.0001, - "input_cost_per_token": 5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.0002, - "output_cost_per_token": 2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, "input_cost_per_audio_token": 4e-05, @@ -6758,6 +7008,7 @@ }, "azure/gpt-4o-transcribe": { "deprecation_date": "2026-10-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 6e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6772,6 +7023,7 @@ }, "azure/gpt-4o-transcribe-diarize": { "deprecation_date": "2027-04-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_audio_token": 6e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -6791,6 +7043,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -6807,7 +7062,9 @@ "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", @@ -6855,6 +7112,8 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-codex", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", @@ -6889,6 +7148,7 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", @@ -6921,6 +7181,8 @@ }, "azure/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -6967,6 +7229,8 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, @@ -7010,6 +7274,8 @@ "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.1-codex", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7039,7 +7305,9 @@ "supports_vision": true }, "azure/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -7085,7 +7353,9 @@ "azure/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-mini", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, @@ -7128,6 +7398,8 @@ }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_batches": 2.5e-09, "input_cost_per_token": 5e-08, @@ -7170,6 +7442,8 @@ "azure/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4-nano", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", @@ -7210,6 +7484,8 @@ }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-pro", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", @@ -7245,7 +7521,9 @@ "supports_vision": true }, "azure/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -7294,6 +7572,8 @@ "azure/gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -7318,6 +7598,8 @@ "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7339,6 +7621,8 @@ "azure/gpt-5.2-chat": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7359,6 +7643,8 @@ "azure/gpt-5.3-chat": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7379,6 +7665,8 @@ "azure/us/gpt-5.1-chat": { "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7400,6 +7688,8 @@ "azure/us/gpt-5.2-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7420,6 +7710,8 @@ "azure/us/gpt-5.3-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7440,6 +7732,8 @@ "azure/eu/gpt-5.1-chat": { "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7461,6 +7755,8 @@ "azure/eu/gpt-5.2-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7481,6 +7777,8 @@ "azure/eu/gpt-5.3-chat": { "cache_read_input_token_cost": 1.925e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-chat-latest", "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", "max_input_tokens": 111616, @@ -7498,38 +7796,10 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "deprecation_date": "2025-07-28", - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_prompt_caching": true - }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "deprecation_date": "2025-07-28", - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_prompt_caching": true - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.2-codex", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -7562,6 +7832,7 @@ "azure/gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7592,6 +7863,7 @@ }, "azure/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -7623,6 +7895,8 @@ }, "azure/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4", "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, @@ -7670,6 +7944,8 @@ "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.4", "input_cost_per_token": 1.75e-06, "input_cost_per_token_batches": 8.75e-07, "input_cost_per_token_priority": 3.5e-06, @@ -7714,6 +7990,8 @@ "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.3-codex", "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -7746,6 +8024,7 @@ "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.75e-06, "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", @@ -7859,6 +8138,8 @@ }, "azure/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, @@ -7916,6 +8197,8 @@ }, "azure/us/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.5", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, @@ -7965,6 +8248,8 @@ }, "azure/eu/gpt-5.4": { "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.5", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, @@ -8019,6 +8304,8 @@ "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, @@ -8074,6 +8361,8 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.5", "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, @@ -8123,6 +8412,8 @@ "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.5", "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, @@ -8169,6 +8460,8 @@ }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5-pro", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -8220,6 +8513,8 @@ "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.5-pro", "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, @@ -8332,6 +8627,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-sol", "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, @@ -8395,6 +8692,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-sol", "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, @@ -8458,6 +8757,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6.1-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, @@ -8521,6 +8822,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6.1-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, @@ -8584,6 +8887,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, @@ -8647,6 +8952,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, @@ -8710,11 +9017,7 @@ "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", @@ -8746,7 +9049,11 @@ "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "input_cost_per_token_above_272k_tokens_flex": 1e-05, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "input_cost_per_token_flex": 5e-06, + "output_cost_per_token_flex": 2.5e-05, + "cache_read_input_token_cost_flex": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-06 }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, @@ -8762,11 +9069,7 @@ "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", @@ -8798,7 +9101,11 @@ "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "input_cost_per_token_above_272k_tokens_flex": 1e-05, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, + "input_cost_per_token_flex": 5e-06, + "output_cost_per_token_flex": 2.5e-05, + "cache_read_input_token_cost_flex": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-06 }, "azure/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, @@ -8814,12 +9121,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, - "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8846,7 +9149,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_flex": 5e-09, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07 }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, @@ -8862,12 +9173,8 @@ "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, - "source": "https://azure.microsoft.com/en-us/blog/gpt-6-astra-sol-and-luna-for-production-agents-in-microsoft-foundry/", + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8894,7 +9201,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_flex": 5e-09, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07 }, "azure/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, @@ -8946,7 +9261,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6-sol-2026-09-22": { "cache_creation_input_token_cost": 2.5e-06, @@ -8998,7 +9321,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, @@ -9061,11 +9392,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", "supported_endpoints": [ "/v1/chat/completions", @@ -9093,7 +9420,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 5e-08, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, @@ -9109,11 +9444,7 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.014, - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014 - }, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/models", "supported_endpoints": [ "/v1/chat/completions", @@ -9141,7 +9472,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 5e-06, + "cache_read_input_token_cost_flex": 5e-08, + "cache_creation_input_token_cost_flex": 1.25e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, "azure/us/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, @@ -9431,6 +9770,7 @@ "azure/gpt-chat-latest": { "cache_read_input_token_cost": 5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -9563,6 +9903,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6-sol", "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, @@ -9618,6 +9960,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-07, "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6.1-terra", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "input_cost_per_token_above_272k_tokens_priority": 8.8e-06, @@ -9673,6 +10017,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-08, "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-6-luna", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "input_cost_per_token_above_272k_tokens_priority": 8.8e-07, @@ -9869,6 +10215,7 @@ "azure/us/gpt-chat-latest": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -9965,6 +10312,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6-sol", "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, @@ -10020,6 +10369,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-07, "cache_read_input_token_cost_priority": 4.4e-07, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6.1-terra", "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "input_cost_per_token_above_272k_tokens_priority": 8.8e-06, @@ -10075,6 +10426,8 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-08, "cache_read_input_token_cost_priority": 4.4e-08, "deprecation_date": "2028-01-11", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-6-luna", "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "input_cost_per_token_above_272k_tokens_priority": 8.8e-07, @@ -10122,6 +10475,8 @@ }, "azure/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, @@ -10182,6 +10537,8 @@ }, "azure/us/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, @@ -10236,6 +10593,8 @@ }, "azure/eu/gpt-5.5": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, @@ -10337,6 +10696,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_batches": 2.5e-06, @@ -10347,6 +10708,8 @@ }, "azure/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1.25e-06, @@ -10450,6 +10813,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, @@ -10458,6 +10823,8 @@ }, "azure/us/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -10555,6 +10922,8 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, @@ -10563,6 +10932,8 @@ }, "azure/eu/gpt-5.5-2026-04-24": { "deprecation_date": "2027-10-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -10699,6 +11070,7 @@ }, "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -10753,6 +11125,7 @@ "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -10801,6 +11174,7 @@ }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, @@ -10851,6 +11225,7 @@ "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -10899,6 +11274,8 @@ "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-1.5", "input_cost_per_image_token": 1e-05, "input_cost_per_token": 5e-06, "litellm_provider": "azure", @@ -10933,6 +11310,7 @@ }, "azure/high/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10943,6 +11321,7 @@ }, "azure/high/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10953,6 +11332,7 @@ }, "azure/high/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -10963,6 +11343,7 @@ }, "azure/low/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10973,6 +11354,7 @@ }, "azure/low/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10983,6 +11365,7 @@ }, "azure/low/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -10993,6 +11376,7 @@ }, "azure/medium/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11003,6 +11387,7 @@ }, "azure/medium/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11013,6 +11398,7 @@ }, "azure/medium/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11025,6 +11411,7 @@ "cache_read_input_image_token_cost": 2.5e-07, "cache_read_input_token_cost": 2e-07, "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_image_token": 2.5e-06, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -11039,6 +11426,8 @@ "azure/gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2026-12-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-2", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11056,6 +11445,8 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-12-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-image-2", "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", @@ -11071,6 +11462,7 @@ "azure/gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11088,6 +11480,7 @@ "azure/gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-09-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11105,6 +11498,7 @@ "azure/gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, "deprecation_date": "2027-09-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -11123,6 +11517,7 @@ "cache_read_input_image_token_cost": 2e-06, "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-10-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, "litellm_provider": "azure", @@ -11138,6 +11533,7 @@ }, "azure/low/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11148,6 +11544,7 @@ }, "azure/low/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11158,6 +11555,7 @@ }, "azure/low/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11168,6 +11566,7 @@ }, "azure/medium/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11178,6 +11577,7 @@ }, "azure/medium/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11188,6 +11588,7 @@ }, "azure/medium/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -11198,6 +11599,7 @@ }, "azure/high/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11208,6 +11610,7 @@ }, "azure/high/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11218,6 +11621,7 @@ }, "azure/high/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -11246,6 +11650,7 @@ }, "azure/o1": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -11263,6 +11668,7 @@ "azure/o1-2024-12-17": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", @@ -11315,6 +11721,7 @@ }, "azure/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -11352,6 +11759,7 @@ }, "azure/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -11390,6 +11798,8 @@ "azure/o3-deep-research": { "cache_read_input_token_cost": 2.5e-06, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -11426,6 +11836,8 @@ }, "azure/o3-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, @@ -11445,6 +11857,8 @@ "azure/o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", @@ -11461,6 +11875,8 @@ }, "azure/o3-pro": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -11496,6 +11912,8 @@ }, "azure/o3-pro-2025-06-10": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -11531,6 +11949,8 @@ }, "azure/o4-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, @@ -11569,6 +11989,8 @@ "azure/o4-mini-2025-04-16": { "cache_read_input_token_cost": 2.75e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", @@ -11620,6 +12042,7 @@ }, "azure/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11629,6 +12052,7 @@ }, "azure/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11638,6 +12062,7 @@ }, "azure/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, @@ -11683,6 +12108,8 @@ }, "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -11725,6 +12152,8 @@ }, "azure/us/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -11767,6 +12196,8 @@ }, "azure/us/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -11800,6 +12231,8 @@ }, "azure/us/gpt-4o-2024-08-06": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, @@ -11819,6 +12252,8 @@ }, "azure/us/gpt-4o": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -11838,6 +12273,8 @@ }, "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_creation_input_token_cost": 1.38e-06, "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, @@ -11858,6 +12295,7 @@ "azure/us/gpt-4o-mini": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -11881,6 +12319,7 @@ "azure/us/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", @@ -11919,26 +12358,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/us/gpt-4o-realtime-preview-2024-10-01": { - "cache_creation_input_audio_token_cost": 2.2e-05, - "cache_read_input_token_cost": 2.75e-06, - "deprecation_date": "2025-03-26", - "input_cost_per_audio_token": 0.00011, - "input_cost_per_token": 5.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 0.00022, - "output_cost_per_token": 2.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/us/gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_audio_token_cost": 2.5e-06, "cache_read_input_token_cost": 2.75e-06, @@ -11970,6 +12389,8 @@ "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, @@ -12013,7 +12434,9 @@ "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_priority": 4.95e-08, - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, "input_cost_per_token_priority": 4.95e-07, @@ -12057,6 +12480,8 @@ "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -12096,7 +12521,9 @@ "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { - "deprecation_date": "2027-05-15", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -12144,6 +12571,8 @@ }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2-codex", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -12175,6 +12604,7 @@ }, "azure/us/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -12207,6 +12637,7 @@ "azure/us/o1": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -12225,6 +12656,7 @@ "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -12258,6 +12690,7 @@ }, "azure/us/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -12295,6 +12728,7 @@ }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -12333,6 +12767,8 @@ "azure/us/o3-mini": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12350,6 +12786,8 @@ "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12367,6 +12805,8 @@ "azure/us/o4-mini": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12392,6 +12832,8 @@ "azure/us/o4-mini-2025-04-16": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-terra", "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", @@ -12423,6 +12865,8 @@ }, "azure_ai/Cohere-embed-v3-english": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, @@ -12434,6 +12878,8 @@ }, "azure_ai/Cohere-embed-v3-multilingual": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, @@ -12445,6 +12891,7 @@ }, "azure_ai/Codestral-2501": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 256000, @@ -12466,6 +12913,7 @@ }, "azure_ai/FLUX-1.1-pro": { "deprecation_date": "2027-01-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 5000, "mode": "image_generation", @@ -12477,6 +12925,7 @@ }, "azure_ai/FLUX.1-Kontext-pro": { "deprecation_date": "2027-01-05", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 5000, "mode": "image_generation", @@ -12488,6 +12937,7 @@ }, "azure_ai/flux.2-pro": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 32000, "mode": "image_generation", @@ -12499,6 +12949,7 @@ }, "azure_ai/FLUX.2-flex": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, @@ -12549,6 +13000,7 @@ }, "azure_ai/FW-GLM-5": { "deprecation_date": "2027-07-01", + "successor_model": "azure_ai/FW-GLM-5.2", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -12565,6 +13017,7 @@ }, "azure_ai/FW-GLM-5.1": { "deprecation_date": "2027-07-01", + "successor_model": "azure_ai/FW-GLM-5.2", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -12822,6 +13275,8 @@ "/v1/images/edits" ], "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/MAI-Image-2.6", "max_input_tokens": 32000 }, "azure_ai/MAI-Image-2.5-Flash": { @@ -12837,10 +13292,13 @@ "/v1/images/edits" ], "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/MAI-Image-2.6-Flash", "max_input_tokens": 32000 }, "azure_ai/MAI-Image-2.5-Pro": { "deprecation_date": "2026-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_image_token": 8e-06, "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", @@ -12880,6 +13338,7 @@ }, "azure_ai/Llama-3.3-70B-Instruct": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -12893,6 +13352,7 @@ }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -12907,6 +13367,7 @@ }, "azure_ai/Llama-4-Scout-17B-16E-Instruct": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 10000000, @@ -12919,149 +13380,10 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure_ai/Meta-Llama-3-70B-Instruct": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 1.1e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 8192, - "max_output_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/Meta-Llama-3.1-70B-Instruct": { - "deprecation_date": "2025-06-30", - "input_cost_per_token": 2.68e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.54e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/Phi-3-medium-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.8e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-medium-4k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.8e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-mini-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-mini-4k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-small-128k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3-small-8k-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-MoE-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.6e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6.4e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-mini-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": false - }, - "azure_ai/Phi-3.5-vision-instruct": { - "deprecation_date": "2025-08-30", - "input_cost_per_token": 1.3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 5.2e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Phi-4": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -13076,6 +13398,8 @@ }, "azure_ai/Phi-4-mini-instruct": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13088,6 +13412,8 @@ }, "azure_ai/Phi-4-multimodal-instruct": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -13103,6 +13429,8 @@ }, "azure_ai/Phi-4-mini-reasoning": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13115,6 +13443,8 @@ }, "azure_ai/Phi-4-reasoning": { "deprecation_date": "2027-01-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "microsoft--phi-4", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -13148,6 +13478,7 @@ }, "azure_ai/Cohere-parse-v5": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "mode": "ocr", "ocr_cost_per_page": 0.0015, @@ -13158,6 +13489,8 @@ }, "azure_ai/cohere-command-a": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "coherelabs-command-a-plus-05-2026-fp81", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13198,6 +13531,7 @@ }, "azure_ai/MAI-DS-R1": { "deprecation_date": "2026-02-27", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13209,30 +13543,10 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/cohere-rerank-v3-english": { - "deprecation_date": "2025-06-30", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" - }, - "azure_ai/cohere-rerank-v3-multilingual": { - "deprecation_date": "2025-06-30", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" - }, "azure_ai/cohere-rerank-v4.0-pro": { "deprecation_date": "2027-09-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "Cohere-rerank-v5-pro", "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -13244,6 +13558,8 @@ }, "azure_ai/cohere-rerank-v4.0-fast": { "deprecation_date": "2027-09-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "Cohere-rerank-v5-fast", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -13255,6 +13571,8 @@ }, "azure_ai/deepseek-v3.2": { "deprecation_date": "2027-11-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4.1-flash", "input_cost_per_token": 5.8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13271,6 +13589,8 @@ }, "azure_ai/deepseek-v3.2-speciale": { "deprecation_date": "2027-11-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-pro", "input_cost_per_token": 5.8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13285,20 +13605,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-v3": { - "deprecation_date": "2025-08-31", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13315,6 +13624,8 @@ }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/DeepSeek-V4-Flash-0731", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13332,6 +13643,7 @@ "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -13347,6 +13659,8 @@ }, "azure_ai/embed-v-4-0": { "deprecation_date": "2027-06-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/Cohere-Embed-V5-Fast", "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13366,6 +13680,8 @@ }, "azure_ai/grok-4": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4.3", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13424,6 +13740,7 @@ "azure_ai/grok-4-20-reasoning": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13443,6 +13760,7 @@ "azure_ai/grok-4-20-non-reasoning": { "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-06", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 1.25e-06, "litellm_provider": "azure_ai", "max_input_tokens": 262000, @@ -13460,6 +13778,7 @@ }, "azure_ai/grok-4-1-fast-non-reasoning": { "deprecation_date": "2027-05-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -13475,6 +13794,7 @@ }, "azure_ai/grok-4-1-fast-reasoning": { "deprecation_date": "2027-05-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -13495,6 +13815,7 @@ }, "azure_ai/grok-code-fast-1": { "deprecation_date": "2027-01-31", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 256000, @@ -13510,6 +13831,7 @@ }, "azure_ai/jais-30b-chat": { "deprecation_date": "2026-01-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "input_cost_per_token": 0.0032, "litellm_provider": "azure_ai", "max_input_tokens": 8192, @@ -13519,20 +13841,9 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", "supports_pdf_input": false }, - "azure_ai/jamba-instruct": { - "deprecation_date": "2025-03-01", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 70000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/kimi-k2.5": { "deprecation_date": "2027-01-26", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -13550,6 +13861,7 @@ }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -13574,6 +13886,8 @@ }, "azure_ai/ministral-3b": { "deprecation_date": "2026-12-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "ministral-ministral-4b-instruct", "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13585,32 +13899,6 @@ "supports_pdf_input": false, "supports_tool_choice": true }, - "azure_ai/mistral-large": { - "deprecation_date": "2025-04-15", - "input_cost_per_token": 4e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, - "azure_ai/mistral-large-2407": { - "deprecation_date": "2025-05-13", - "input_cost_per_token": 2e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/mistral-large-latest": { "input_cost_per_token": 2e-06, "litellm_provider": "azure_ai", @@ -13638,6 +13926,8 @@ }, "azure_ai/mistral-medium-2505": { "deprecation_date": "2026-12-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/mistral-medium-3-5", "input_cost_per_token": 4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13651,6 +13941,8 @@ }, "azure_ai/mistral-nemo": { "deprecation_date": "2026-01-30", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "successor_model": "azure_ai/mistral-small-2503", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -13661,21 +13953,10 @@ "supports_function_calling": true, "supports_pdf_input": false }, - "azure_ai/mistral-small": { - "deprecation_date": "2025-07-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_tool_choice": true - }, "azure_ai/mistral-small-2503": { "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "mistralai-mistral-small-4-119b-2603", "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -13689,6 +13970,8 @@ }, "babbage-002": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "litellm_provider": "text-completion-openai", @@ -13853,6 +14136,7 @@ }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -13897,6 +14181,7 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.3e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -13949,6 +14234,7 @@ }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -13982,6 +14268,7 @@ }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.18e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -13992,6 +14279,7 @@ }, "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14024,6 +14312,7 @@ }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14068,6 +14357,7 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.1e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -14160,6 +14450,7 @@ }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", @@ -14230,6 +14521,7 @@ }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14240,6 +14532,7 @@ }, "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14272,6 +14565,7 @@ }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14415,6 +14709,7 @@ }, "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14476,6 +14771,7 @@ }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14486,6 +14782,7 @@ }, "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14496,6 +14793,7 @@ }, "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14557,6 +14855,7 @@ }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14567,6 +14866,7 @@ }, "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14577,6 +14877,7 @@ }, "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4.7e-07, "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, @@ -14664,6 +14965,7 @@ }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -14685,6 +14987,7 @@ }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 5.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -14696,6 +14999,7 @@ }, "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", @@ -14766,6 +15070,7 @@ }, "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 4.45e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14776,6 +15081,7 @@ }, "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14808,6 +15114,7 @@ }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-07, "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, @@ -14852,6 +15159,7 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 7.3e-07, "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", @@ -14983,6 +15291,7 @@ }, "bedrock/us-east-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -14993,6 +15302,7 @@ }, "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15003,6 +15313,7 @@ }, "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15024,6 +15335,7 @@ }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15057,6 +15369,7 @@ }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15100,6 +15413,7 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15178,6 +15492,7 @@ }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15222,6 +15537,7 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15296,7 +15612,7 @@ "output_vector_size": 1536 }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0": { - "input_cost_per_token": 2e-07, + "input_cost_per_token": 3e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, "mode": "embedding", @@ -15332,6 +15648,8 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15362,6 +15680,7 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15381,7 +15700,8 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -15409,8 +15729,10 @@ "input_cost_per_image": 7.2e-05, "input_cost_per_video_per_second": 0.00084, "input_cost_per_audio_per_second": 0.000168, + "input_cost_per_token_batches": 8.1e-08, "output_cost_per_token": 0.0, "output_vector_size": 3072, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_embedding_image_input": true, "supports_image_input": true, "supports_video_input": true, @@ -15419,11 +15741,14 @@ "bedrock/us-gov-west-1/amazon.nova-lite-v1:0": { "cache_read_input_token_cost": 1.8e-08, "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -15434,11 +15759,14 @@ "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.05e-08, "input_cost_per_token": 4.2e-08, + "input_cost_per_token_batches": 2.1e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, + "output_cost_per_token_batches": 8.4e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -15446,12 +15774,21 @@ }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, + "cache_read_input_token_cost_flex": 1.2e-07, + "cache_read_input_token_cost_priority": 4.2e-07, "input_cost_per_token": 9.6e-07, + "input_cost_per_token_batches": 4.8e-07, + "input_cost_per_token_flex": 4.8e-07, + "input_cost_per_token_priority": 1.68e-06, "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, + "output_cost_per_token_batches": 1.92e-06, + "output_cost_per_token_flex": 1.92e-06, + "output_cost_per_token_priority": 6.72e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -15504,6 +15841,8 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15534,6 +15873,7 @@ "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15553,10 +15893,12 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15568,6 +15910,7 @@ }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15579,6 +15922,7 @@ }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15589,6 +15933,7 @@ }, "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -15678,6 +16023,7 @@ }, "bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15699,6 +16045,7 @@ }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -15732,6 +16079,7 @@ }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -15775,6 +16123,7 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", @@ -15947,17 +16296,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "cerebras/llama3.1-70b": { - "deprecation_date": "2025-01-17", - "input_cost_per_token": 6e-07, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true - }, "cerebras/llama3.1-8b": { "deprecation_date": "2026-05-27", "input_cost_per_token": 1e-07, @@ -16031,6 +16369,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 0.0001, "source": "https://developers.openai.com/api/docs/pricing" }, @@ -16096,6 +16436,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", + "successor_model": "claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -16129,6 +16471,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", + "successor_model": "claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -16177,6 +16521,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -16297,6 +16642,7 @@ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", "litellm_provider": "bedrock", "supports_tool_search": true, "max_input_tokens": 200000, @@ -16316,7 +16662,8 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "claude-opus-4-5-20251101": { "supports_web_search": true, @@ -16710,6 +17057,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -17508,12 +17856,15 @@ }, "computer-use-preview": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/responses" ], @@ -17537,12 +17888,15 @@ }, "computer-use-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/responses" ], @@ -17567,6 +17921,7 @@ "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, @@ -17588,6 +17943,7 @@ "deepseek-reasoner": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, @@ -21570,6 +21926,7 @@ "prompt_cache_min_tokens": 512, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -21618,6 +21975,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", + "successor_model": "databricks/databricks-claude-sonnet-5", "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", @@ -21727,6 +22085,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_function_calling": true, "supports_mid_conversation_system": true, @@ -21741,6 +22100,7 @@ "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-11-05", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21754,6 +22114,7 @@ "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "supported_modalities": [ "text" ], @@ -21767,14 +22128,19 @@ "supports_vision": false }, "databricks/system.ai.deepseek-v4-flash-0731": { + "deprecation_date": "2026-11-05", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", "litellm_provider": "databricks", "mode": "chat", + "source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "token_count_framing": 97 }, "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21787,6 +22153,7 @@ "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017", "supported_modalities": [ "text" ], @@ -21803,6 +22170,7 @@ "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.0002e-08, "deprecation_date": "2026-10-02", + "successor_model": "databricks/databricks-gemini-3-5-flash", "input_cost_per_token": 3.0001999999999996e-07, "input_dbu_cost_per_token": 4.285999999999999e-06, "litellm_provider": "databricks", @@ -22072,7 +22440,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22090,7 +22458,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22104,7 +22474,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22122,7 +22492,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22138,7 +22510,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22157,7 +22529,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22173,7 +22547,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22191,7 +22565,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22207,7 +22583,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22227,7 +22603,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -22841,6 +23219,7 @@ "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22852,6 +23231,7 @@ "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-glm-5-3", "supported_modalities": [ "text", "image" @@ -23076,6 +23456,8 @@ }, "davinci-002": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "text-completion-openai", @@ -23970,7 +24352,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24044,7 +24426,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24417,6 +24799,7 @@ "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -24469,6 +24852,7 @@ "deepseek/deepseek-reasoner": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-07-24", + "deprecation_source": "https://api-docs.deepseek.com/updates", "input_cost_per_token": 2.8e-07, "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", @@ -24522,6 +24906,7 @@ }, "deepseek.v3-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -25331,6 +25716,7 @@ "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", + "successor_model": "eu.anthropic.claude-haiku-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -25448,6 +25834,7 @@ "supports_vision": true, "prompt_cache_min_tokens": 1024, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "token_count_model": { "model": "anthropic.claude-opus-4-1-20250805-v1:0", "litellm_provider": "bedrock" @@ -25482,6 +25869,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "eu.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -25525,6 +25914,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "eu.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -25558,6 +25949,7 @@ }, "eu.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -25570,6 +25962,7 @@ }, "eu.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -28149,6 +28542,7 @@ "cache_read_input_token_cost": 1.4e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/accounts/fireworks/models/glm-5p3", "input_cost_per_token": 1.4e-06, "input_cost_per_token_priority": 1.75e-06, "litellm_provider": "fireworks_ai", @@ -28524,6 +28918,7 @@ "cache_read_input_token_cost": 1.4e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/glm-5p3", "input_cost_per_token": 1.4e-06, "input_cost_per_token_priority": 1.75e-06, "litellm_provider": "fireworks_ai", @@ -28907,6 +29302,8 @@ }, "ft:babbage-002": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.6e-06, "input_cost_per_token_batches": 8e-07, "litellm_provider": "text-completion-openai", @@ -28919,6 +29316,8 @@ }, "ft:davinci-002": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.2e-05, "input_cost_per_token_batches": 6e-06, "litellm_provider": "text-completion-openai", @@ -28931,6 +29330,8 @@ }, "ft:gpt-3.5-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -28938,6 +29339,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_system_messages": true, @@ -28945,45 +29347,57 @@ }, "ft:gpt-3.5-turbo-0125": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-3.5-turbo-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "ft:gpt-3.5-turbo", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-3.5-turbo-1106": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "fedramp_uplift_multiplier": 1.1, "supports_system_messages": true, "supports_tool_choice": true }, "ft:gpt-4-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", "supports_function_calling": true, "supports_system_messages": true, @@ -28993,6 +29407,8 @@ "cache_read_input_token_cost": 1.875e-06, "cache_read_input_token_cost_batches": 9e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 2.225e-06, "litellm_provider": "openai", @@ -29000,6 +29416,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.25e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29014,12 +29431,15 @@ "ft:gpt-4o-2024-11-20": { "cache_creation_input_token_cost": 1.875e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -29032,6 +29452,8 @@ "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "litellm_provider": "openai", @@ -29039,6 +29461,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29053,6 +29476,8 @@ "cache_read_input_token_cost": 7.5e-07, "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -29060,6 +29485,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29073,6 +29499,8 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 8e-07, "input_cost_per_token_batches": 4e-07, "litellm_provider": "openai", @@ -29080,6 +29508,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.6e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29093,6 +29522,8 @@ "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 2.5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "litellm_provider": "openai", @@ -29100,6 +29531,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29113,6 +29545,8 @@ "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 4e-06, "input_cost_per_token_batches": 2e-06, "litellm_provider": "openai", @@ -29120,6 +29554,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -29131,19 +29566,38 @@ }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, - "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-10-20", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 3e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, + "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":1500,"period":"day","shared_group":"gemini-2-flash"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29160,6 +29614,7 @@ ], "supports_audio_output": false, "supports_function_calling": true, + "supports_image_size": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29169,26 +29624,12 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_token_batches": 1.5e-07, - "input_cost_per_token_flex": 1.5e-07, - "input_cost_per_token_priority": 5.4e-07, - "input_cost_per_audio_token_priority": 1.8e-06, - "output_cost_per_token_batches": 1.25e-06, - "output_cost_per_token_flex": 1.25e-06, - "output_cost_per_token_priority": 4.5e-06, - "supports_image_size": false + "supports_web_search": true }, "gemini-2.5-flash-image": { "deprecation_date": "2027-03-15", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -29238,12 +29679,13 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -29258,6 +29700,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -29312,6 +29755,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -29352,6 +29796,7 @@ "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29364,6 +29809,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -29410,6 +29856,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -29458,6 +29905,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -29538,17 +29986,28 @@ }, "gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_date": "2027-05-07", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "deprecation_date": "2027-05-07", + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_above_200k_tokens": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 9e-07, + "input_cost_per_audio_token_batches": 2.5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, + "input_cost_per_audio_token_priority": 9e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, - "input_cost_per_audio_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -29558,7 +30017,10 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.5-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29576,6 +30038,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29587,19 +30050,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, @@ -29695,19 +30150,38 @@ }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, - "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-10-20", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_above_200k_tokens": 3e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-08, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-08, + "input_cost_per_audio_token_above_200k_tokens_priority": 5.4e-07, + "input_cost_per_audio_token_batches": 5e-08, + "input_cost_per_audio_token_flex": 5e-08, + "input_cost_per_audio_token_priority": 5.4e-07, "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_flex": 2e-07, + "output_cost_per_token_priority": 7.2e-07, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":1500,"period":"day","shared_group":"gemini-2-flash"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -29724,6 +30198,7 @@ ], "supports_audio_output": false, "supports_function_calling": true, + "supports_image_size": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -29733,23 +30208,7 @@ "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token_batches": 5e-08, - "input_cost_per_token_batches": 5e-08, - "input_cost_per_token_flex": 5e-08, - "input_cost_per_token_priority": 1.8e-07, - "input_cost_per_audio_token_priority": 5.4e-07, - "output_cost_per_token_batches": 2e-07, - "output_cost_per_token_flex": 2e-07, - "output_cost_per_token_priority": 7.2e-07, - "supports_image_size": false + "supports_web_search": true }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -29845,6 +30304,8 @@ }, "gemini-live-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -29978,6 +30439,8 @@ }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -30026,11 +30489,13 @@ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, + "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, @@ -30046,6 +30511,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30058,6 +30524,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30205,11 +30672,17 @@ }, "vertex_ai/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 1e-06, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -30224,7 +30697,7 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30258,6 +30731,8 @@ "vertex_ai/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, @@ -30369,6 +30844,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30431,6 +30908,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2027-01-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30462,7 +30941,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30592,6 +31071,7 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -30604,6 +31084,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, @@ -30842,6 +31323,8 @@ }, "gemini-embedding-001": { "deprecation_date": "2028-05-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-embedding-2", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -30866,6 +31349,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_embedding_image_input": true, "uses_embed_content": true }, "gemini-embedding-2": { @@ -30883,6 +31367,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30901,6 +31386,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30919,6 +31405,7 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supports_embedding_image_input": true, "supports_multimodal": true, "uses_embed_content": true }, @@ -30936,6 +31423,8 @@ }, "gemini/gemini-embedding-001": { "deprecation_date": "2028-05-14", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-embedding-2", "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, @@ -30964,12 +31453,15 @@ "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_embedding_image_input": true, "supports_multimodal": true, "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-3-pro-image-preview": { "deprecation_date": "2026-06-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3-pro-image", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -31018,6 +31510,8 @@ }, "gemini/gemini-3.1-flash-image-preview": { "deprecation_date": "2026-06-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-nano-banana-2.1", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -31066,6 +31560,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-05-25", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.1-flash-lite", "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -31123,6 +31619,8 @@ "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-embedding-2-preview": { + "deprecation_date": "2026-08-10", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, @@ -31138,7 +31636,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "successor_model": "gemini/gemini-embedding-2", "supports_audio_input": true, + "supports_embedding_image_input": true, "supports_multimodal": true, "supports_vision": true, "tpm": 10000000 @@ -31224,6 +31724,9 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.5-flash-lite", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -31283,6 +31786,8 @@ "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, "deprecation_date": "2027-03-15", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -31510,6 +32015,8 @@ }, "gemini/deep-research-pro-preview-12-2025": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/deep-research-preview-04-2026", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -31727,6 +32234,8 @@ }, "gemini/gemini-2.5-flash-preview-tts": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -31807,11 +32316,16 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "deprecation_date": "2026-07-28", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash", "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "gemini", @@ -31850,6 +32364,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.5-flash-lite", "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, @@ -31910,7 +32426,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -32315,7 +32831,9 @@ "supports_video_input": true, "supports_vision": true, "tpm": 800000, - "deprecation_date": "2026-10-22" + "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash" }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -32451,11 +32969,17 @@ }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_above_200k_tokens": 1e-06, + "input_cost_per_audio_token_above_200k_tokens_batches": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 1.8e-06, "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_audio_token_priority": 1.8e-06, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -32471,7 +32995,7 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32507,13 +33031,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32531,11 +33055,14 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, @@ -32647,6 +33174,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2026-11-19", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -32709,6 +33238,8 @@ "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, "deprecation_date": "2027-01-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "gemini-3.8-flash", "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -32739,7 +33270,7 @@ "output_cost_per_token_priority": 6.75e-06, "prompt_cache_min_tokens": 4096.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32827,6 +33358,8 @@ "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -33079,6 +33612,8 @@ }, "gemini/veo-3.1-fast-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33095,6 +33630,8 @@ }, "gemini/veo-3.1-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33110,6 +33647,8 @@ }, "gemini/veo-3.1-lite-generate-preview": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-omni-1.1-flash", "litellm_provider": "gemini", "max_input_tokens": 1024, "mode": "video_generation", @@ -33445,6 +33984,7 @@ }, "github_copilot/gpt-4.1": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -33457,6 +33997,7 @@ }, "github_copilot/gpt-4.1-2025-04-14": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, @@ -33558,6 +34099,7 @@ "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "deprecation_date": "2026-10-19", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "litellm_provider": "github_copilot", @@ -33600,6 +34142,7 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "deprecation_date": "2026-10-19", + "deprecation_source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "litellm_provider": "github_copilot", @@ -34453,6 +34996,7 @@ }, "google.gemma-3-12b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 9e-08, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, @@ -34474,6 +35018,7 @@ }, "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.3e-07, "input_cost_per_token_batches": 1.2e-07, "input_cost_per_token_flex": 1.2e-07, @@ -34495,6 +35040,7 @@ }, "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-08, "input_cost_per_token_batches": 2e-08, "input_cost_per_token_flex": 2e-08, @@ -34529,6 +35075,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "global.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -34564,6 +35112,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "global.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -34652,12 +35202,15 @@ }, "gpt-3.5-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 5e-07, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -34666,6 +35219,8 @@ }, "gpt-3.5-turbo-0125": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "openai", @@ -34673,6 +35228,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 7.5e-07, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34683,6 +35239,8 @@ }, "gpt-3.5-turbo-1106": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "openai", @@ -34690,6 +35248,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34700,18 +35259,22 @@ }, "gpt-3.5-turbo-16k": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_token": 3e-06, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, + "fedramp_uplift_multiplier": 1.1, "supports_prompt_caching": true, "supports_system_messages": true, "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { "deprecation_date": "2026-09-28", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -34730,12 +35293,15 @@ }, "gpt-4": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_prompt_caching": true, "supports_system_messages": true, @@ -34743,6 +35309,8 @@ }, "gpt-4-0613": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 3e-05, "input_cost_per_token_batches": 1.5e-05, "litellm_provider": "openai", @@ -34750,6 +35318,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34759,12 +35328,15 @@ }, "gpt-4-1106-preview": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -34773,12 +35345,15 @@ }, "gpt-4-turbo": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -34789,6 +35364,8 @@ }, "gpt-4-turbo-2024-04-09": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, "litellm_provider": "openai", @@ -34796,6 +35373,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -34810,6 +35388,8 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.5e-06, @@ -34818,6 +35398,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_priority": 1.4e-05, "search_context_cost_per_query": { @@ -34853,6 +35434,8 @@ "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-06, "input_cost_per_token_priority": 3.5e-06, "input_cost_per_token_batches": 1e-06, @@ -34861,6 +35444,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.4e-05, "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { @@ -34896,6 +35480,8 @@ "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_priority": 7e-07, @@ -34904,6 +35490,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8e-07, "output_cost_per_token_priority": 2.8e-06, "search_context_cost_per_query": { @@ -34939,6 +35526,8 @@ "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 4e-07, "input_cost_per_token_priority": 7e-07, "input_cost_per_token_batches": 2e-07, @@ -34947,6 +35536,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-06, "output_cost_per_token_batches": 8e-07, "search_context_cost_per_query": { @@ -34982,6 +35572,8 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_priority": 2e-07, @@ -34990,6 +35582,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 8e-07, "source": "https://developers.openai.com/api/docs/pricing", @@ -35019,6 +35612,8 @@ "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 5e-08, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 1e-07, "input_cost_per_token_priority": 2e-07, "input_cost_per_token_batches": 5e-08, @@ -35027,6 +35622,7 @@ "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 8e-07, "output_cost_per_token_batches": 2e-07, "source": "https://developers.openai.com/api/docs/pricing", @@ -35056,6 +35652,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 4.25e-06, @@ -35064,6 +35662,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.7e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35078,6 +35677,8 @@ }, "gpt-4o-2024-05-13": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_priority": 8.75e-06, @@ -35086,6 +35687,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 2.625e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35101,6 +35703,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_priority": 4.25e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35109,6 +35713,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.7e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -35125,6 +35730,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2.5e-06, "input_cost_per_token_priority": 4.25e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35133,6 +35740,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1.7e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -35147,6 +35755,8 @@ }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35164,6 +35774,8 @@ }, "gpt-4o-audio-preview-2025-06-03": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35181,6 +35793,8 @@ }, "gpt-audio": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35251,6 +35865,8 @@ }, "gpt-audio-2025-08-28": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -35288,6 +35904,8 @@ }, "gpt-audio-mini": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35325,6 +35943,8 @@ }, "gpt-audio-mini-2025-10-06": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35395,12 +36015,16 @@ "supports_tool_choice": true, "supports_vision": false, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-audio-1.5", "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.5e-07, @@ -35409,6 +36033,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1e-06, "supports_function_calling": true, @@ -35430,6 +36055,8 @@ "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, "deprecation_date": "2027-01-14", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -35438,6 +36065,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 1e-06, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { @@ -35457,6 +36085,7 @@ }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -35476,6 +36105,7 @@ "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -35500,6 +36130,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35518,6 +36149,8 @@ }, "gpt-4o-mini-search-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -35526,6 +36159,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35555,11 +36189,15 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-4o-mini-tts": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -35580,6 +36218,8 @@ }, "gpt-4o-mini-tts-2025-03-20": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -35607,6 +36247,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35625,6 +36266,8 @@ }, "gpt-4o-search-preview-2025-03-11": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, @@ -35633,6 +36276,7 @@ "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.025, @@ -35662,6 +36306,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_second": 0.0001, "source": "https://developers.openai.com/api/docs/pricing" }, @@ -35670,6 +36316,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -35678,6 +36326,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -35691,6 +36340,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -35699,6 +36350,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -35718,6 +36370,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -35753,6 +36406,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -35788,6 +36442,7 @@ "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -35814,6 +36469,8 @@ }, "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35826,6 +36483,8 @@ }, "low/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35838,6 +36497,8 @@ }, "low/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35850,6 +36511,8 @@ }, "medium/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -35862,6 +36525,8 @@ }, "medium/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -35874,6 +36539,8 @@ }, "medium/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -35886,6 +36553,8 @@ }, "high/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -35898,6 +36567,8 @@ }, "high/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -35910,6 +36581,8 @@ }, "high/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -35922,6 +36595,8 @@ }, "standard/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35934,6 +36609,8 @@ }, "standard/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35946,6 +36623,8 @@ }, "standard/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35958,6 +36637,8 @@ }, "1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -35970,6 +36651,8 @@ }, "1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -35982,6 +36665,8 @@ }, "1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36002,7 +36687,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "low/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36014,7 +36701,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "low/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36026,7 +36715,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.034, @@ -36038,7 +36729,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36050,7 +36743,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "medium/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36062,7 +36757,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.133, @@ -36074,7 +36771,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36086,7 +36785,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "high/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36098,7 +36799,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -36110,7 +36813,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36122,7 +36827,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "standard/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36134,7 +36841,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -36146,7 +36855,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36158,7 +36869,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36170,7 +36883,9 @@ ], "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-12-01" + "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst" }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, @@ -36178,6 +36893,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -36187,6 +36904,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, @@ -36233,12 +36951,15 @@ "gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", "supported_endpoints": [ "/v1/responses" @@ -36273,6 +36994,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2e-05, "reasoning_contexts": [ "auto", @@ -36318,17 +37040,22 @@ "source": "https://developers.openai.com/api/docs/pricing", "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol" }, "gpt-5.1-codex-mini": { "cache_read_input_token_cost": 2.5e-08, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", "supported_endpoints": [ "/v1/responses" @@ -36353,12 +37080,15 @@ "gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", "supported_endpoints": [ "/v1/responses" @@ -36384,12 +37114,15 @@ "gpt-5.1-codex": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", "supported_endpoints": [ "/v1/responses" @@ -36415,12 +37148,15 @@ "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -36449,6 +37185,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", @@ -36456,6 +37194,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2e-05, "reasoning_contexts": [ "auto", @@ -36514,6 +37253,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -36565,12 +37305,15 @@ "gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", "supported_endpoints": [ "/v1/responses" @@ -36596,12 +37339,15 @@ "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-08-10", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -36636,6 +37382,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -36692,6 +37439,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8.4e-05, "reasoning_contexts": [ "auto", @@ -36740,6 +37488,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 8.4e-05, "reasoning_contexts": [ "auto", @@ -36846,6 +37595,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -36939,6 +37689,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37040,6 +37791,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37132,6 +37884,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37222,6 +37975,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37314,6 +38068,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37405,6 +38160,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37449,10 +38205,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/chat/completions", @@ -37487,10 +38244,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/chat/completions", @@ -37525,10 +38283,11 @@ "input_cost_per_token": 1.25e-05, "input_cost_per_token_above_272k_tokens": 2.5e-05, "litellm_provider": "openai", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 0.0001125, "supported_endpoints": [ "/v1/responses" @@ -37566,6 +38325,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/chat/completions", @@ -37604,6 +38364,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/responses" @@ -37637,6 +38398,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, + "fedramp_uplift_multiplier": 1.1, "reasoning_effort_levels": [ "medium" ], @@ -37705,6 +38467,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37775,6 +38538,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37837,6 +38601,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37896,6 +38661,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -37963,6 +38729,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38028,6 +38795,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38085,6 +38853,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38145,6 +38914,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38207,6 +38977,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38269,6 +39040,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38328,6 +39100,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38361,13 +39134,17 @@ "source": "https://developers.openai.com/api/docs/pricing", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna" }, "gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-luna", "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_batches": 1e-07, @@ -38388,6 +39165,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38424,6 +39202,8 @@ }, "gpt-5-pro": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -38431,6 +39211,7 @@ "max_output_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-05, "reasoning_contexts": [ "auto", @@ -38473,6 +39254,8 @@ }, "gpt-5-pro-2025-10-06": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -38480,6 +39263,7 @@ "max_output_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 6e-05, "reasoning_contexts": [ "auto", @@ -38526,6 +39310,8 @@ "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -38535,6 +39321,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, @@ -38581,12 +39368,15 @@ "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.2-chat-latest", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -38616,12 +39406,15 @@ "gpt-5-chat-latest": { "cache_read_input_token_cost": 1.25e-07, "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -38654,6 +39447,7 @@ "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_priority": 2.8e-05, "reasoning_contexts": [ "auto", @@ -38693,17 +39487,22 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "deprecation_date": "2027-04-01" + "deprecation_date": "2027-04-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6-sol" }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, "deprecation_date": "2026-08-10", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.75e-06, "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", "supported_endpoints": [ "/v1/chat/completions", @@ -38732,6 +39531,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -38741,6 +39542,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, @@ -38790,6 +39592,8 @@ "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -38799,6 +39603,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 1e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, @@ -38847,6 +39652,8 @@ "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -38855,6 +39662,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "reasoning_contexts": [ @@ -38902,6 +39710,8 @@ "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -38910,6 +39720,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07, "reasoning_contexts": [ @@ -38957,6 +39768,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 1e-05, "input_cost_per_image_token_batches": 5e-06, "input_cost_per_token": 5e-06, @@ -38975,6 +39788,8 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 2.5e-06, "input_cost_per_image_token_batches": 1.25e-06, "input_cost_per_token": 2e-06, @@ -38982,6 +39797,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 8e-06, + "output_cost_per_image_token_batches": 4e-06, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -38994,6 +39810,8 @@ "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -39069,6 +39887,9 @@ "max_input_tokens": 128000, "max_output_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -39124,6 +39945,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39160,6 +39983,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39169,6 +39994,8 @@ "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, "input_cost_per_token": 6e-07, @@ -39204,6 +40031,8 @@ "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -39527,16 +40356,6 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-guard-3-8b": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://console.groq.com/docs/model/llama-guard-3-8b" - }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, "litellm_provider": "groq", @@ -39683,6 +40502,8 @@ }, "high/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -39694,6 +40515,8 @@ }, "high/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -39705,6 +40528,8 @@ }, "high/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -40082,6 +40907,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "jp.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -41244,6 +42071,8 @@ }, "low/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -41255,6 +42084,8 @@ }, "low/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -41266,6 +42097,8 @@ }, "low/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -41289,6 +42122,8 @@ }, "medium/1024-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41300,6 +42135,8 @@ }, "medium/1024-x-1536/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41311,6 +42148,8 @@ }, "medium/1536-x-1024/gpt-image-1": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -41322,6 +42161,8 @@ }, "low/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -41331,6 +42172,8 @@ }, "low/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -41340,6 +42183,8 @@ }, "low/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -41349,6 +42194,8 @@ }, "medium/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -41358,6 +42205,8 @@ }, "medium/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -41367,6 +42216,8 @@ }, "medium/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -41414,6 +42265,7 @@ }, "meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41426,6 +42278,7 @@ }, "meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock", @@ -41440,6 +42293,7 @@ }, "meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "bedrock", @@ -41454,6 +42308,7 @@ }, "meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41467,6 +42322,7 @@ }, "meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41479,6 +42335,7 @@ }, "meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41491,6 +42348,7 @@ }, "meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -41504,6 +42362,7 @@ }, "meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock_converse", @@ -41518,6 +42377,7 @@ }, "meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -41528,6 +42388,7 @@ }, "meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, @@ -41538,6 +42399,7 @@ }, "meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md", "input_cost_per_token": 2.4e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "bedrock_converse", @@ -41560,6 +42422,7 @@ }, "meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md", "input_cost_per_token": 1.7e-07, "input_cost_per_token_batches": 8.5e-08, "litellm_provider": "bedrock_converse", @@ -41902,6 +42765,7 @@ }, "minimax.minimax-m2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -41924,6 +42788,7 @@ }, "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -42144,6 +43009,7 @@ }, "mistral.devstral-2-123b": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_flex": 2e-07, @@ -42250,6 +43116,7 @@ }, "mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -42315,6 +43182,7 @@ }, "mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, @@ -42326,6 +43194,7 @@ }, "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4e-08, "input_cost_per_token_batches": 2e-08, "input_cost_per_token_flex": 2e-08, @@ -42345,6 +43214,7 @@ }, "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, @@ -42364,6 +43234,7 @@ }, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 4.12e-08, "input_cost_per_token_batches": 2.06e-08, "input_cost_per_token_flex": 2.06e-08, @@ -42383,6 +43254,7 @@ }, "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.03e-07, "input_cost_per_token_batches": 5.15e-08, "input_cost_per_token_flex": 5.15e-08, @@ -42402,6 +43274,7 @@ }, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -42421,6 +43294,7 @@ }, "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.2e-07, "input_cost_per_token_batches": 6e-08, "input_cost_per_token_flex": 6e-08, @@ -42440,6 +43314,7 @@ }, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -42459,6 +43334,7 @@ }, "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "input_cost_per_token_batches": 8e-08, "input_cost_per_token_flex": 8e-08, @@ -42851,6 +43727,7 @@ "mistral/zai-glm-5-2": { "cache_read_input_token_cost": 1.4e-07, "deprecation_date": "2026-10-31", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/models_overview", "input_cost_per_token": 1.4e-06, "litellm_provider": "mistral", "max_input_tokens": 1048576, @@ -43106,18 +43983,18 @@ "mode": "embedding" }, "mistral/mistral-large-4": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43127,18 +44004,18 @@ "supports_vision": true }, "mistral/mistral-large-4-0": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43426,6 +44303,7 @@ }, "moonshot.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_priority": 1.05e-06, @@ -43828,10 +44706,12 @@ "text", "image", "video" - ] + ], + "supports_embedding_image_input": true }, "multimodalembedding@001": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -43851,7 +44731,8 @@ "text", "image", "video" - ] + ], + "supports_embedding_image_input": true }, "nscale/Qwen/QwQ-32B": { "input_cost_per_token": 1.8e-07, @@ -44059,7 +44940,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44285,7 +45167,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44293,7 +45175,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44304,29 +45187,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44349,7 +45235,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44361,7 +45248,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44373,10 +45261,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44385,7 +45274,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44396,7 +45286,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44440,7 +45331,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44463,7 +45355,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44474,7 +45367,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44485,7 +45379,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44496,7 +45391,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44506,7 +45402,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44516,7 +45413,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44526,7 +45424,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44548,7 +45447,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44556,9 +45456,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44569,7 +45470,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44580,21 +45482,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44602,8 +45506,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -44639,6 +45544,7 @@ }, "nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, @@ -44660,6 +45566,7 @@ }, "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -44726,6 +45633,7 @@ "o1": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-10-23", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -44733,6 +45641,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -44748,6 +45657,8 @@ "o1-2024-12-17": { "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -44755,6 +45666,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -44769,6 +45681,8 @@ }, "o1-pro": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 0.00015, "input_cost_per_token_batches": 7.5e-05, "litellm_provider": "openai", @@ -44776,6 +45690,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 0.0003, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -44802,6 +45717,8 @@ }, "o1-pro-2025-03-19": { "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 0.00015, "input_cost_per_token_batches": 7.5e-05, "litellm_provider": "openai", @@ -44809,6 +45726,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 0.0003, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -44838,6 +45756,7 @@ "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -44847,6 +45766,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_flex": 4e-06, "output_cost_per_token_priority": 1.4e-05, @@ -44894,6 +45814,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", "supported_endpoints": [ "/v1/batch", @@ -44915,10 +45836,14 @@ "supports_tool_choice": false, "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-07-23" + "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol" }, "o3-deep-research-2025-06-26": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "openai", @@ -44926,6 +45851,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", "supported_endpoints": [ "/v1/batch", @@ -44953,6 +45879,8 @@ "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -44962,6 +45890,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_flex": 4e-06, "output_cost_per_token_priority": 1.4e-05, @@ -45004,6 +45933,8 @@ "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "openai", @@ -45011,6 +45942,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -45024,6 +45956,8 @@ "o3-mini-2025-01-31": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "openai", @@ -45031,6 +45965,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, @@ -45043,6 +45978,8 @@ }, "o3-pro": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -45050,6 +45987,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-05, "reasoning_contexts": [ "auto", @@ -45087,6 +46025,8 @@ }, "o3-pro-2025-06-10": { "deprecation_date": "2026-12-11", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -45094,6 +46034,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 4e-05, "reasoning_contexts": [ "auto", @@ -45134,6 +46075,8 @@ "cache_read_input_token_cost_flex": 1.38e-07, "cache_read_input_token_cost_priority": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "input_cost_per_token_flex": 5.5e-07, @@ -45143,6 +46086,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "output_cost_per_token_flex": 2.2e-06, "output_cost_per_token_priority": 8e-06, @@ -45170,6 +46114,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", "supported_endpoints": [ "/v1/batch", @@ -45191,10 +46136,14 @@ "supports_tool_choice": false, "supports_vision": true, "supports_pdf_input": true, - "deprecation_date": "2026-07-23" + "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol" }, "o4-mini-deep-research-2025-06-26": { "deprecation_date": "2026-07-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-sol", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openai", @@ -45202,6 +46151,7 @@ "max_output_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", "supported_endpoints": [ "/v1/batch", @@ -45229,6 +46179,8 @@ "cache_read_input_token_cost_flex": 1.38e-07, "cache_read_input_token_cost_priority": 5e-07, "deprecation_date": "2026-10-23", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.1e-06, "input_cost_per_token_batches": 5.5e-07, "input_cost_per_token_flex": 5.5e-07, @@ -45238,6 +46190,7 @@ "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "fedramp_uplift_multiplier": 1.1, "output_cost_per_token_batches": 2.2e-06, "output_cost_per_token_flex": 2.2e-06, "output_cost_per_token_priority": 8e-06, @@ -45527,6 +46480,7 @@ "mode": "embedding", "output_vector_size": 1024, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", + "supports_embedding_image_input": true, "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -46412,6 +47366,19 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/decision": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", @@ -47029,6 +47996,7 @@ }, "qwen.qwen3-coder-480b-a35b-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, @@ -47060,6 +48028,7 @@ }, "qwen.qwen3-coder-30b-a3b-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.625e-07, @@ -48639,6 +49608,7 @@ }, "text-embedding-004": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48650,6 +49620,7 @@ }, "text-embedding-005": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48720,6 +49691,7 @@ }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -48834,6 +49806,7 @@ }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "deprecation_date": "2026-10-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -48841,6 +49814,7 @@ "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/meta-models/Muse-Glimmer-30B", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -48900,6 +49874,7 @@ "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-11-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -48907,6 +49882,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/MiniMaxAI/MiniMax-M3.1-Flash", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -48983,6 +49959,7 @@ "together_ai/Qwen/Qwen3.8-2.4T-A95B": { "cache_read_input_token_cost": 2.5e-07, "deprecation_date": "2026-11-04", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, @@ -48990,6 +49967,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/Qwen/Qwen3.7-Max", "supports_prompt_caching": true }, "together_ai/arize-ai/qwen-2-1.5b-instruct": { @@ -49004,6 +49982,7 @@ "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, "deprecation_date": "2026-10-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -49011,6 +49990,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -49095,6 +50075,7 @@ "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -49102,6 +50083,7 @@ "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/meta-models/Muse-Spark-Open-816B-A42B", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -49159,6 +50141,8 @@ }, "tts-1": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_character": 1.5e-05, "litellm_provider": "openai", "mode": "audio_speech", @@ -49169,6 +50153,8 @@ }, "tts-1-hd": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_character": 3e-05, "litellm_provider": "openai", "mode": "audio_speech", @@ -49399,6 +50385,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, "deprecation_date": "2027-01-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -49437,6 +50424,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "us.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -49479,6 +50468,8 @@ "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "deprecation_date": "2027-04-08", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "successor_model": "us-gov.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, @@ -49519,6 +50510,7 @@ "prompt_cache_min_tokens": 1024, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -49594,8 +50586,9 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -49616,7 +50609,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -49715,6 +50710,7 @@ }, "us-gov.nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49730,6 +50726,7 @@ }, "us-gov.nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49985,6 +50982,8 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-10-14", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "us.anthropic.claude-sonnet-5", "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -50019,6 +51018,7 @@ }, "us.deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "input_cost_per_token": 1.35e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -50071,6 +51071,7 @@ }, "us.meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.4e-06, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50083,6 +51084,7 @@ }, "us.meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock", @@ -50097,6 +51099,7 @@ }, "us.meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "bedrock", @@ -50111,6 +51114,7 @@ }, "us.meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50124,6 +51128,7 @@ }, "us.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50136,6 +51141,7 @@ }, "us.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50148,6 +51154,7 @@ }, "us.meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -50161,6 +51168,7 @@ }, "us.meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7.2e-07, "input_cost_per_token_batches": 3.6e-07, "litellm_provider": "bedrock_converse", @@ -50175,6 +51183,7 @@ }, "us.meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-maverick-17b-instruct.md", "input_cost_per_token": 2.4e-07, "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "bedrock_converse", @@ -50197,6 +51206,7 @@ }, "us.meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-4-scout-17b-instruct.md", "input_cost_per_token": 1.7e-07, "input_cost_per_token_batches": 8.5e-08, "litellm_provider": "bedrock_converse", @@ -51457,6 +52467,7 @@ }, "vertex_ai/claude-haiku-4-5": { "deprecation_date": "2026-10-15", + "successor_model": "vertex_ai/claude-haiku-5-5", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, @@ -51576,32 +52587,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-opus": { - "deprecation_date": "2025-08-01", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-3-opus@20240229": { - "deprecation_date": "2025-08-01", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -51628,6 +52613,7 @@ }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", + "successor_model": "vertex_ai/claude-opus-5-5", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52008,6 +52994,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52046,6 +53033,7 @@ "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -52237,6 +53225,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "deprecation_date": "2026-09-29", + "successor_model": "vertex_ai/claude-sonnet-5-5", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -52294,6 +53283,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -52524,6 +53514,8 @@ }, "vertex_ai/gemini-2.5-flash-image": { "deprecation_date": "2027-03-15", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.1-flash-lite-image", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -52574,12 +53566,13 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -52594,6 +53587,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, @@ -52629,6 +53623,7 @@ "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, "output_cost_per_image_token_batches": 6e-05, + "output_cost_per_image_token_priority": 0.000216, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -52650,6 +53645,7 @@ "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -52662,6 +53658,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, @@ -52690,6 +53687,7 @@ "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, "output_cost_per_image_token_batches": 3e-05, + "output_cost_per_image_token_priority": 0.000108, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "supports_reasoning": false, @@ -52719,6 +53717,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, @@ -52799,17 +53798,28 @@ }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_date": "2027-05-07", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "deprecation_date": "2027-05-07", + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_above_200k_tokens": 5e-07, + "input_cost_per_audio_token_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_flex": 2.5e-07, + "input_cost_per_audio_token_above_200k_tokens_priority": 9e-07, + "input_cost_per_audio_token_batches": 2.5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, + "input_cost_per_audio_token_priority": 9e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, - "input_cost_per_audio_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -52820,7 +53830,10 @@ "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -52838,6 +53851,7 @@ "supports_audio_input": true, "supports_audio_output": false, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -52849,19 +53863,11 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, @@ -53277,6 +54283,7 @@ "vertex_ai/zai-org/glm-4.7-maas": { "cache_read_input_token_cost": 6e-08, "deprecation_date": "2026-10-21", + "successor_model": "vertex_ai/zai-org/glm-5.2-maas", "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -53295,6 +54302,7 @@ "vertex_ai/zai-org/glm-5-maas": { "cache_read_input_token_cost": 1e-07, "deprecation_date": "2026-10-21", + "successor_model": "vertex_ai/zai-org/glm-5.2-maas", "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -53714,6 +54722,7 @@ }, "vertex_ai/veo-3.1-generate-001": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -53729,6 +54738,7 @@ }, "vertex_ai/veo-3.1-fast-generate-001": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -53931,7 +54941,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53947,7 +54957,8 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "supports_embedding_image_input": true }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, @@ -54379,6 +55390,8 @@ "/v1/audio/transcriptions" ], "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "source": "https://developers.openai.com/api/docs/pricing" }, "xai/grok-4.20-multi-agent-beta-0309": { @@ -54642,6 +55655,8 @@ }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "zai.glm-5", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, @@ -54686,6 +55701,7 @@ }, "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -54999,6 +56015,7 @@ }, "openai/sora-2": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -55013,6 +56030,7 @@ }, "openai/sora-2-pro": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, @@ -55028,6 +56046,7 @@ "openai/sora-2-pro-high-res": { "litellm_provider": "openai", "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, "source": "https://platform.openai.com/docs/api-reference/videos", @@ -58694,6 +59713,8 @@ }, "gpt-4o-mini-tts-2025-12-15": { "deprecation_date": "2027-01-06", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "input_cost_per_token": 6e-07, "litellm_provider": "openai", "mode": "audio_speech", @@ -58714,6 +59735,8 @@ }, "gpt-4o-mini-transcribe-2025-03-20": { "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-4o-mini-transcribe-2025-12-15", "input_cost_per_audio_token": 1.25e-06, "input_cost_per_second": 5e-05, "input_cost_per_token": 1.25e-06, @@ -58729,6 +59752,8 @@ }, "gpt-4o-mini-transcribe-2025-12-15": { "deprecation_date": "2027-02-26", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-live-transcribe", "input_cost_per_audio_token": 1.25e-06, "input_cost_per_second": 5e-05, "input_cost_per_token": 1.25e-06, @@ -58750,6 +59775,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -58777,6 +59803,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -58828,6 +59855,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "deprecation_date": "2027-01-20", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-realtime-2.1-mini", "source": "https://developers.openai.com/api/docs/pricing" }, "gpt-realtime-whisper": { @@ -58849,6 +59878,7 @@ }, "sora-2": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, @@ -58863,6 +59893,7 @@ }, "sora-2-pro": { "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, @@ -58878,6 +59909,7 @@ "sora-2-pro-high-res": { "litellm_provider": "openai", "deprecation_date": "2026-09-24", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, "source": "https://developers.openai.com/api/docs/pricing", @@ -58894,6 +59926,8 @@ "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-image-2.5-sunburst", "input_cost_per_image_token": 8e-06, "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, @@ -58901,6 +59935,7 @@ "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3.2e-05, + "output_cost_per_image_token_batches": 1.6e-05, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -58909,48 +59944,6 @@ "/v1/images/edits" ] }, - "gemini-2.0-flash-exp-image-generation": { - "deprecation_date": "2025-11-14", - "input_cost_per_token": 0.0, - "litellm_provider": "gemini", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.039, - "output_cost_per_token": 0.0, - "source": "https://ai.google.dev/pricing", - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_vision": true - }, - "gemini/gemini-2.0-flash-exp-image-generation": { - "deprecation_date": "2025-11-14", - "input_cost_per_token": 0.0, - "litellm_provider": "gemini", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.039, - "output_cost_per_token": 0.0, - "source": "https://ai.google.dev/pricing", - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_vision": true, - "tpm": 250000, - "rpm": 10 - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -59003,6 +59996,8 @@ }, "gemini-2.5-flash-native-audio-preview-12-2025": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -59034,6 +60029,8 @@ }, "gemini-3.1-flash-live-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -59191,6 +60188,8 @@ }, "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -59225,6 +60224,8 @@ }, "gemini/gemini-3.1-flash-live-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -59264,6 +60265,8 @@ }, "gemini/gemini-3.1-flash-tts-preview": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", @@ -59333,6 +60336,8 @@ }, "gemini-2.5-flash-preview-tts": { "deprecation_date": "2026-11-17", + "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", + "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -59654,6 +60659,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -63392,6 +64398,10 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -63404,7 +64414,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63422,7 +64433,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63456,7 +64468,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -63547,6 +64560,7 @@ }, "claude-mythos-preview": { "deprecation_date": "2026-06-09", + "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -63736,7 +64750,8 @@ { "name": "claude-family-baseline", "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?", - "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.", + "fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"], + "description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.", "model_info": { "mode": "chat", "max_input_tokens": 200000, @@ -63779,6 +64794,14 @@ "thinking_always_on": true } }, + { + "name": "claude-thinking-on-by-default", + "pattern": "claude-[a-z]+-[5-9](?!\\d)", + "description": "Claude at major 5 or higher, in any id shape that contains claude--. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.", + "model_info": { + "thinking_on_by_default": true + } + }, { "name": "claude-mid-conversation-system", "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)", @@ -64223,6 +65246,7 @@ "fireworks_ai/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/glm-5p3-fast", "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -64492,6 +65516,7 @@ "fireworks_ai/accounts/fireworks/routers/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", + "successor_model": "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast", "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -64583,7 +65608,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64695,13 +65720,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -64804,13 +65829,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66235,9 +67260,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66616,8 +67641,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -66774,10 +67799,12 @@ }, "xai/grok-imagine-image-quality": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66791,10 +67818,12 @@ }, "xai/grok-imagine-image-quality-20260403": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66808,10 +67837,12 @@ }, "xai/grok-imagine-image-quality-latest": { "deprecation_date": "2026-11-02", + "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66825,10 +67856,12 @@ }, "xai/grok-imagine-image-pro": { "deprecation_date": "2026-05-15", + "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", "input_cost_per_image": 0.05, "litellm_provider": "xai", "mode": "image_generation", "source": "https://docs.x.ai/docs/models", + "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" ], @@ -66946,6 +67979,8 @@ }, "xai/grok-voice-transcribe-1.0": { "deprecation_date": "2026-10-02", + "deprecation_source": "https://docs.x.ai/docs/release-notes", + "successor_model": "xai/grok-voice-transcribe-2.0", "input_cost_per_second": 2.778e-05, "litellm_provider": "xai", "metadata": { @@ -67811,6 +68846,7 @@ }, "together_ai/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-08, "output_cost_per_token": 2.5e-07, "litellm_provider": "together_ai", @@ -67881,6 +68917,7 @@ "azure_ai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "deprecation_date": "2027-04-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -67916,6 +68953,7 @@ "supports_reasoning": true, "supports_tool_choice": true, "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/deepseek-v4-1-flash-is-coming-to-microsoft-foundry/4556431" }, "azure_ai/MAI-Image-2.6": { @@ -68024,6 +69062,8 @@ }, "azure_ai/Cohere-command-a-plus-05-2026": { "deprecation_date": "2026-10-16", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "coherelabs-command-a-plus-05-2026-fp81", "input_cost_per_token": 8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -68057,6 +69097,8 @@ }, "azure_ai/deepseek-r1": { "deprecation_date": "2026-08-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-pro", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, "litellm_provider": "azure_ai", @@ -68066,6 +69108,8 @@ }, "azure_ai/deepseek-v3-0324": { "deprecation_date": "2026-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-flash", "input_cost_per_token": 1.14e-06, "output_cost_per_token": 4.56e-06, "litellm_provider": "azure_ai", @@ -68075,6 +69119,8 @@ }, "azure_ai/deepseek-v3.1": { "deprecation_date": "2026-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/deepseek-v4-flash", "input_cost_per_token": 1.23e-06, "output_cost_per_token": 4.94e-06, "litellm_provider": "azure_ai", @@ -68084,6 +69130,8 @@ }, "azure_ai/grok-3": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", @@ -68093,6 +69141,8 @@ }, "azure_ai/grok-3-mini": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-reasoning", "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", @@ -68102,6 +69152,8 @@ }, "azure_ai/grok-4-fast-non-reasoning": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-non-reasoning", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -68111,6 +69163,8 @@ }, "azure_ai/grok-4-fast-reasoning": { "deprecation_date": "2026-05-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/grok-4-1-fast-reasoning", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -68120,12 +69174,18 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -68136,13 +69196,20 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -68151,13 +69218,20 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -68166,12 +69240,18 @@ }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -68183,12 +69263,18 @@ "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68202,12 +69288,18 @@ "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68233,6 +69325,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68308,6 +69401,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68328,7 +69422,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68410,12 +69507,18 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -68426,13 +69529,20 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -68441,13 +69551,20 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -68456,12 +69573,18 @@ }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -68473,12 +69596,18 @@ "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68492,12 +69621,18 @@ "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions" ], @@ -68523,6 +69658,7 @@ "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68588,6 +69724,7 @@ "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -68603,7 +69740,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68698,16 +69838,32 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 2.7e-06, - "input_cost_per_token_above_272k_tokens": 5.4e-06, - "cache_creation_input_token_cost": 3.375e-06, - "cache_creation_input_token_cost_above_272k_tokens": 6.75e-06, - "cache_read_input_token_cost": 2.7e-07, - "cache_read_input_token_cost_above_272k_tokens": 5.4e-07, - "output_cost_per_token": 1.62e-05, - "output_cost_per_token_above_272k_tokens": 2.43e-05, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.32e-05, + "cache_creation_input_token_cost_flex": 1.65e-06, + "cache_creation_input_token_cost_priority": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.056e-06, + "cache_read_input_token_cost_flex": 1.32e-07, + "cache_read_input_token_cost_priority": 5.28e-07, + "input_cost_per_token_above_272k_tokens_flex": 2.64e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.056e-05, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 5.28e-06, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.188e-05, + "output_cost_per_token_above_272k_tokens_priority": 4.752e-05, + "output_cost_per_token_flex": 7.92e-06, + "output_cost_per_token_priority": 3.168e-05, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_web_search": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { @@ -68735,16 +69891,32 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 2.7e-07, - "input_cost_per_token_above_272k_tokens": 5.4e-07, - "cache_creation_input_token_cost": 3.375e-07, - "cache_creation_input_token_cost_above_272k_tokens": 6.75e-07, - "cache_read_input_token_cost": 2.7e-08, - "cache_read_input_token_cost_above_272k_tokens": 5.4e-08, - "output_cost_per_token": 1.62e-06, - "output_cost_per_token_above_272k_tokens": 2.43e-06, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1.32e-06, + "cache_creation_input_token_cost_flex": 1.65e-07, + "cache_creation_input_token_cost_priority": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.056e-07, + "cache_read_input_token_cost_flex": 1.32e-08, + "cache_read_input_token_cost_priority": 5.28e-08, + "input_cost_per_token_above_272k_tokens_flex": 2.64e-07, + "input_cost_per_token_above_272k_tokens_priority": 1.056e-06, + "input_cost_per_token_flex": 1.32e-07, + "input_cost_per_token_priority": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.188e-06, + "output_cost_per_token_above_272k_tokens_priority": 4.752e-06, + "output_cost_per_token_flex": 7.92e-07, + "output_cost_per_token_priority": 3.168e-06, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supports_web_search": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { @@ -68771,9 +69943,9 @@ "supports_tool_choice": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "input_cost_per_token": 3.375e-06, - "cache_read_input_token_cost": 3.375e-07, - "output_cost_per_token": 2.025e-05, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05, "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144}, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supports_web_search": true @@ -68793,20 +69965,36 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "input_cost_per_token": 1.5e-06, "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 2.4e-07 + "cache_read_input_token_cost": 2.4e-07, + "cache_read_input_token_cost_batches": 1.2e-07, + "cache_read_input_token_cost_flex": 1.2e-07, + "cache_read_input_token_cost_priority": 4.2e-07, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.625e-06, + "output_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06 }, "bedrock_mantle/us-gov-west-1/xai.grok-4.6": { "use_openai_responses_path": true, "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, "mode": "chat", + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -68820,11 +70008,17 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": { "input_cost_per_token": 4.8e-08, + "input_cost_per_token_flex": 2.4e-08, + "input_cost_per_token_priority": 8.4e-08, "output_cost_per_token": 9.6e-08, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 4.8e-08, + "output_cost_per_token_flex": 4.8e-08, + "output_cost_per_token_priority": 1.68e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68838,11 +70032,18 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": { "input_cost_per_token": 1.56e-07, + "input_cost_per_token_batches": 7.8e-08, + "input_cost_per_token_flex": 7.8e-08, + "input_cost_per_token_priority": 2.73e-07, "output_cost_per_token": 4.8e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, "mode": "chat", + "output_cost_per_token_batches": 2.4e-07, + "output_cost_per_token_flex": 2.4e-07, + "output_cost_per_token_priority": 8.4e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68856,11 +70057,18 @@ }, "bedrock_mantle/us-gov-west-1/google.gemma-4-31b": { "input_cost_per_token": 1.68e-07, + "input_cost_per_token_batches": 8.4e-08, + "input_cost_per_token_flex": 8.4e-08, + "input_cost_per_token_priority": 2.94e-07, "output_cost_per_token": 4.8e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, "mode": "chat", + "output_cost_per_token_batches": 2.4e-07, + "output_cost_per_token_flex": 2.4e-07, + "output_cost_per_token_priority": 8.4e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "use_openai_responses_path": true, "supported_endpoints": [ "/v1/chat/completions", @@ -68875,12 +70083,18 @@ "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -68894,12 +70108,18 @@ "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69015,10 +70235,17 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, "mode": "chat", + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69033,12 +70260,18 @@ "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { "supports_vision": false, "input_cost_per_token": 8.4e-08, + "input_cost_per_token_batches": 4.2e-08, + "input_cost_per_token_flex": 4.2e-08, + "input_cost_per_token_priority": 1.47e-07, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69052,12 +70285,18 @@ "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { "supports_vision": false, "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -69070,6 +70309,7 @@ }, "bedrock_mantle/deepseek.v3.1": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 5.8e-07, "input_cost_per_token_batches": 2.9e-07, "input_cost_per_token_flex": 2.9e-07, @@ -69091,6 +70331,7 @@ }, "bedrock_mantle/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 6e-07, "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, @@ -69155,6 +70396,7 @@ }, "bedrock_mantle/qwen.qwen3-coder-30b-a3b-instruct": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -69176,6 +70418,7 @@ }, "bedrock_mantle/qwen.qwen3-coder-480b-a35b-instruct": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", "input_cost_per_token": 4.5e-07, "input_cost_per_token_batches": 2.25e-07, "input_cost_per_token_flex": 2.25e-07, @@ -69626,6 +70869,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69753,6 +71004,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -69878,12 +71137,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -69914,6 +71173,7 @@ "litellm_provider": "openai", "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_reasoning": true }, @@ -69923,6 +71183,7 @@ "litellm_provider": "openai", "mode": "chat", "output_cost_per_token": 7.5e-05, + "fedramp_uplift_multiplier": 1.1, "source": "https://developers.openai.com/api/docs/pricing", "supports_reasoning": true }, @@ -69990,6 +71251,7 @@ }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, @@ -70014,48 +71276,9 @@ "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70064,18 +71287,9 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { "deprecation_date": "2026-01-05", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70086,6 +71300,7 @@ }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { "deprecation_date": "2026-06-04", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70096,6 +71311,7 @@ }, "together_ai/Qwen/Qwen3-Coder-Next-FP8": { "deprecation_date": "2026-05-14", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70106,6 +71322,7 @@ }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70116,6 +71333,7 @@ }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70126,6 +71344,7 @@ }, "together_ai/Qwen/Qwen3-VL-32B-Instruct": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70136,6 +71355,7 @@ }, "together_ai/Qwen/Qwen3-VL-8B-Instruct": { "deprecation_date": "2026-04-16", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70147,6 +71367,7 @@ "together_ai/Qwen/Qwen3.5-397B-A17B": { "cache_read_input_token_cost": 3.5e-07, "deprecation_date": "2026-06-29", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70155,38 +71376,9 @@ "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-V3.1": { "deprecation_date": "2026-05-14", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70195,48 +71387,21 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" + "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/zai-org/GLM-5.3-Flash" }, "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { "deprecation_date": "2026-02-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -70245,28 +71410,9 @@ "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { "deprecation_date": "2026-02-25", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70277,6 +71423,7 @@ }, "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { "deprecation_date": "2026-03-06", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70285,18 +71432,9 @@ "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70307,6 +71445,7 @@ }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { "deprecation_date": "2026-04-16", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, @@ -70318,6 +71457,7 @@ "together_ai/moonshotai/Kimi-K2.6": { "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-08-19", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70329,6 +71469,7 @@ "together_ai/moonshotai/Kimi-K2.7-Code": { "cache_read_input_token_cost": 1.9e-07, "deprecation_date": "2026-08-27", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -70337,19 +71478,10 @@ "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-08-27", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, @@ -70360,16 +71492,19 @@ }, "together_ai/openai/gpt-oss-20b": { "deprecation_date": "2026-09-15", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" + "source": "https://api.together.ai/v1/models", + "successor_model": "together_ai/Qwen/Qwen3.5-9B" }, "together_ai/zai-org/GLM-4.5-Air-FP8": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -70380,6 +71515,7 @@ }, "together_ai/zai-org/GLM-4.7": { "deprecation_date": "2026-04-02", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70390,6 +71526,7 @@ }, "together_ai/zai-org/GLM-5": { "deprecation_date": "2026-06-22", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70401,6 +71538,7 @@ "together_ai/zai-org/GLM-5.1": { "cache_read_input_token_cost": 2.6e-07, "deprecation_date": "2026-07-10", + "deprecation_source": "https://docs.together.ai/docs/deprecations", "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, @@ -70411,6 +71549,7 @@ }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 4.13e-07, "input_cost_per_token": 1.65e-06, "litellm_provider": "azure", @@ -70427,6 +71566,8 @@ }, "azure/eu/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -70446,6 +71587,8 @@ }, "azure/eu/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -70465,6 +71608,8 @@ }, "azure/eu/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -70475,7 +71620,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "input_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "litellm_provider": "azure", @@ -70488,6 +71635,8 @@ }, "azure/eu/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -70508,6 +71657,8 @@ }, "azure/eu/gpt-5-codex": { "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.1-codex", "cache_read_input_token_cost": 1.38e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -70516,7 +71667,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-mini", "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, @@ -70537,6 +71690,8 @@ }, "azure/eu/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4-nano", "cache_read_input_token_cost": 5.5e-09, "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, @@ -70554,6 +71709,8 @@ }, "azure/eu/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.2-pro", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -70569,6 +71726,7 @@ }, "azure/eu/gpt-5.1-codex-max": { "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -70578,6 +71736,8 @@ }, "azure/eu/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.4", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, @@ -70598,6 +71758,8 @@ }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.3-codex", "cache_read_input_token_cost": 1.925e-07, "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", @@ -70621,6 +71783,7 @@ }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, @@ -70638,6 +71801,7 @@ }, "azure/eu/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, @@ -70658,6 +71822,7 @@ }, "azure/eu/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, @@ -70675,6 +71840,7 @@ }, "azure/eu/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, @@ -70848,6 +72014,7 @@ }, "azure/eu/o3": { "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -70866,6 +72033,7 @@ }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, @@ -70884,6 +72052,8 @@ }, "azure/eu/o3-deep-research": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-sol", "cache_read_input_token_cost": 2.75e-06, "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -70893,6 +72063,8 @@ }, "azure/eu/o4-mini": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "cache_read_input_token_cost": 3.03e-07, "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, @@ -70911,6 +72083,8 @@ }, "azure/eu/o4-mini-2025-04-16": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/eu/gpt-5.6-terra", "cache_read_input_token_cost": 3.03e-07, "input_cost_per_token": 1.21e-06, "input_cost_per_token_batches": 6.05e-07, @@ -70929,6 +72103,7 @@ }, "azure/eu/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "mode": "embedding", @@ -70936,6 +72111,7 @@ }, "azure/eu/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "mode": "embedding", @@ -70943,6 +72119,7 @@ }, "azure/eu/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "mode": "embedding", @@ -70992,6 +72169,7 @@ }, "azure/us/codex-mini": { "deprecation_date": "2026-11-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 4.13e-07, "input_cost_per_token": 1.65e-06, "litellm_provider": "azure", @@ -71008,6 +72186,8 @@ }, "azure/us/gpt-4.1": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, @@ -71027,6 +72207,8 @@ }, "azure/us/gpt-4.1-mini": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, @@ -71046,6 +72228,8 @@ }, "azure/us/gpt-4.1-nano": { "deprecation_date": "2027-04-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, "input_cost_per_token_batches": 5.5e-08, @@ -71056,7 +72240,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-4o-2024-05-13": { - "deprecation_date": "2026-12-09", + "deprecation_date": "2027-01-14", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "input_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "litellm_provider": "azure", @@ -71069,6 +72255,8 @@ }, "azure/us/gpt-5": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1", "cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, @@ -71089,6 +72277,8 @@ }, "azure/us/gpt-5-codex": { "deprecation_date": "2027-03-17", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.1-codex", "cache_read_input_token_cost": 1.38e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -71097,7 +72287,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5-mini": { - "deprecation_date": "2027-02-09", + "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-mini", "cache_read_input_token_cost": 2.75e-08, "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, @@ -71118,6 +72310,8 @@ }, "azure/us/gpt-5-nano": { "deprecation_date": "2027-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4-nano", "cache_read_input_token_cost": 5.5e-09, "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, @@ -71135,6 +72329,8 @@ }, "azure/us/gpt-5-pro": { "deprecation_date": "2027-04-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.2-pro", "input_cost_per_token": 1.65e-05, "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", @@ -71150,6 +72346,7 @@ }, "azure/us/gpt-5.1-codex-max": { "deprecation_date": "2027-05-18", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.375e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -71159,6 +72356,8 @@ }, "azure/us/gpt-5.2": { "deprecation_date": "2027-06-08", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.4", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, @@ -71179,6 +72378,8 @@ }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.3-codex", "cache_read_input_token_cost": 1.925e-07, "input_cost_per_token": 1.925e-06, "litellm_provider": "azure", @@ -71202,6 +72403,7 @@ }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 1.925e-07, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, @@ -71219,6 +72421,7 @@ }, "azure/us/gpt-5.4-mini": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, @@ -71239,6 +72442,7 @@ }, "azure/us/gpt-5.4-nano": { "deprecation_date": "2027-09-21", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, @@ -71256,6 +72460,7 @@ }, "azure/us/gpt-5.4-pro": { "deprecation_date": "2027-09-07", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, @@ -71285,6 +72490,8 @@ }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/us/gpt-5.6-sol", "cache_read_input_token_cost": 2.75e-06, "input_cost_per_token": 1.1e-05, "litellm_provider": "azure", @@ -71294,6 +72501,7 @@ }, "azure/us/text-embedding-3-large": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "mode": "embedding", @@ -71301,6 +72509,7 @@ }, "azure/us/text-embedding-3-small": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "mode": "embedding", @@ -71308,6 +72517,7 @@ }, "azure/us/text-embedding-ada-002": { "deprecation_date": "2028-02-09", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "mode": "embedding", @@ -71318,6 +72528,8 @@ "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "successor_model": "azure/gpt-realtime-2.1", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, "input_cost_per_token": 4e-06, @@ -71327,6 +72539,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71384,7 +72599,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71394,7 +72613,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71403,7 +72623,13 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", @@ -71421,7 +72647,11 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, @@ -72860,7 +74090,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, @@ -73719,6 +74950,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "deprecation_date": "2026-05-15", + "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", "input_cost_per_image_token": 1e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_above_200k_tokens": 2e-06, @@ -73729,6 +74961,7 @@ "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "source": "https://api.x.ai/v1/language-models", + "successor_model": "xai/grok-build-0.1", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -73818,6 +75051,8 @@ }, "vertex_ai/gemini-2.0-flash": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_character": 3.75e-08, @@ -73831,6 +75066,8 @@ }, "vertex_ai/gemini-2.0-flash-lite": { "deprecation_date": "2026-06-01", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/gemini-3.5-flash-lite", "input_cost_per_audio_token": 7.5e-08, "input_cost_per_audio_token_batches": 3.75e-08, "input_cost_per_character": 1.875e-08, @@ -73882,6 +75119,8 @@ }, "vertex_ai/veo-3.0-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -73897,6 +75136,8 @@ }, "vertex_ai/veo-3.0-fast-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-fast-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -73913,6 +75154,8 @@ }, "vertex_ai/veo-2.0-generate-001": { "deprecation_date": "2026-06-30", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "successor_model": "vertex_ai/veo-3.1-generate-001", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -74233,6 +75476,7 @@ "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -74260,12 +75504,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -74347,6 +75593,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74437,6 +75684,7 @@ }, "deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "input_cost_per_token": 1.35e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -74862,7 +76110,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -74910,6 +76158,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -75288,6 +76539,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -75512,6 +76764,8 @@ }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", + "successor_model": "azure_ai/kimi-k2.5", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -75613,6 +76867,7 @@ "azure/model-router": { "comment": "The flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it", "deprecation_date": "2027-05-20", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_token": 1.4e-07, "litellm_provider": "azure", "max_input_tokens": 200000, @@ -75624,6 +76879,7 @@ }, "azure/whisper": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", @@ -75685,6 +76941,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -75731,6 +76988,7 @@ "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, @@ -75840,6 +77098,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -75892,6 +77151,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -75936,6 +77196,7 @@ "thinking_always_on": false, "prompt_cache_min_tokens": 512, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -75983,6 +77244,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76019,6 +77281,7 @@ "max_output_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76075,6 +77338,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76121,6 +77385,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76161,6 +77426,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76202,6 +77468,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76248,6 +77515,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76299,6 +77567,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76345,6 +77614,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76377,6 +77647,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -76402,6 +77675,7 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76453,6 +77727,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -76503,6 +77778,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76547,6 +77823,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -76711,23 +77988,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -76757,23 +78030,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77073,6 +78342,7 @@ "litellm_provider": "xai", "mode": "realtime", "source": "https://docs.x.ai/developers/pricing", + "successor_model": "xai/grok-voice-3.0", "supported_endpoints": [ "/v1/realtime" ], @@ -77156,6 +78426,7 @@ "cache_creation_input_token_cost_above_1hr": 1.6e-05, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-10-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 8e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -77267,9 +78538,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -78311,6 +79583,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78525,6 +79798,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78600,6 +79874,8 @@ "cache_creation_input_token_cost_batches": 1.875e-06, "cache_read_input_token_cost_batches": 1.5e-07, "deprecation_date": "2026-11-30", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", + "successor_model": "bedrock/claude_platform/claude-sonnet-5-5", "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, @@ -78689,6 +79965,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -78812,6 +80089,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -80673,6 +81951,7 @@ "prompt_cache_min_tokens": 512.0, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, + "thinking_on_by_default": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, "supports_function_calling": true, @@ -80717,6 +81996,7 @@ "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, "deprecation_date": "2026-11-19", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "input_cost_per_token": 3.0002e-07, "input_dbu_cost_per_token": 4.286e-06, "litellm_provider": "databricks", @@ -80744,7 +82024,8 @@ "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token_priority": 2.1e-06, "cache_read_input_token_cost_priority": 5.25e-08, - "cache_creation_input_token_cost_priority": 5.25e-07 + "cache_creation_input_token_cost_priority": 5.25e-07, + "successor_model": "databricks/databricks-deepseek-v4-1-flash-1017" }, "databricks/databricks-deepseek-v4-1-flash-1017": { "cache_creation_input_token_cost": 3.0002e-07, @@ -81196,6 +82477,8 @@ "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, + "deprecation_date": "2026-10-30", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", "input_cost_per_token": 9.4997e-07, "input_dbu_cost_per_token": 1.3571e-05, "litellm_provider": "databricks", @@ -81208,6 +82491,7 @@ "output_cost_per_token": 4.00001e-06, "output_dbu_cost_per_token": 5.7143e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "successor_model": "databricks/databricks-kimi-k3", "supported_modalities": [ "text", "image" @@ -81294,6 +82578,23 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/deepseek-v4p1-flash-1017-ultrafast": { + "cache_read_input_token_cost": 4.8e-08, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 9.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing.md", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/deepseek-v4p1-flash-1017-spot": { "cache_read_input_token_cost": 4.2e-09, "input_cost_per_token": 2.1e-07, @@ -81388,17 +82689,17 @@ "supports_native_streaming": true }, "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_priority": 5.625e-08, - "input_cost_per_token": 2.25e-07, - "input_cost_per_token_priority": 2.8125e-07, + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 7.5e-07, - "output_cost_per_token_priority": 9.375e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://docs.fireworks.ai/serverless/pricing.md", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -81488,6 +82789,7 @@ ], "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, + "fedramp_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -81540,5 +82842,3339 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "empiriolabs/aplomb-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0.0, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages", + "/v1/decisions" + ], + "supports_vision": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/aplomb-1" + }, + "empiriolabs/deepseek-v4-1-flash-1017": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017" + }, + "empiriolabs/deepseek-v4-flash-0731": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 4.24e-07, + "output_cost_per_token": 1.272e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + }, + "empiriolabs/deepseek-v4-flash-0731:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 4.4e-07, + "output_cost_per_token": 1.32e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + }, + "empiriolabs/deepseek-v4-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 3.3e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro-0813": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.32e-06, + "output_cost_per_token": 3.96e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813" + }, + "empiriolabs/deepseek-v4-pro:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 4.8e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro:variant2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 3.301e-06, + "cache_read_input_token_cost": 1.38e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/deepseek-v4-pro:variant3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 4.8e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + }, + "empiriolabs/fugu-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 2.5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.007, + "search_context_size_medium": 0.007, + "search_context_size_high": 0.007 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-max" + }, + "empiriolabs/fugu-ultra-v1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-05, + "input_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens": 6.75e-05, + "cache_read_input_token_cost_above_272k_tokens": 3e-06, + "cache_read_input_token_cost": 1.5e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v1-0" + }, + "empiriolabs/fugu-ultra-v1-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost": 5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v1-1" + }, + "empiriolabs/fugu-ultra-v2-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "input_cost_per_token_above_272k_tokens": 1e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost": 5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/fugu-ultra-v2-0" + }, + "empiriolabs/gemma-4-26b-a4b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2.9e-07, + "cache_read_input_token_cost": 2.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/gemma-4-26b-a4b" + }, + "empiriolabs/glm-4-6v-flashx": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.6e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-4-6v-flashx" + }, + "empiriolabs/glm-4-7-flashx": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 202752, + "max_output_tokens": 131072, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 6.3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-4-7-flashx" + }, + "empiriolabs/glm-5-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 202745, + "input_cost_per_token": 8.25e-07, + "output_cost_per_token": 3.301e-06, + "input_cost_per_token_above_32k_tokens": 1.1e-06, + "output_cost_per_token_above_32k_tokens": 3.851e-06, + "cache_read_input_token_cost_above_32k_tokens": 2.2e-07, + "cache_read_input_token_cost": 1.65e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-1" + }, + "empiriolabs/glm-5-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-2:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 3.851e-06, + "cache_read_input_token_cost": 2.75e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-2:variant2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-2" + }, + "empiriolabs/glm-5-3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-3" + }, + "empiriolabs/glm-5-3-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 2.5e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/glm-5-3-flash" + }, + "empiriolabs/kimi-k2-6": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 16000, + "input_cost_per_token": 8.939e-07, + "output_cost_per_token": 3.7131e-06, + "cache_read_input_token_cost": 1.788e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-6" + }, + "empiriolabs/kimi-k2-7-code": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 9.5e-07, + "output_cost_per_token": 4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + }, + "empiriolabs/kimi-k2-7-code-highspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.9e-06, + "output_cost_per_token": 8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed" + }, + "empiriolabs/kimi-k2-7-code:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 8.939e-07, + "output_cost_per_token": 3.7131e-06, + "cache_read_input_token_cost": 1.788e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + }, + "empiriolabs/kimi-k3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/kimi-k3" + }, + "empiriolabs/mimo-v2-6-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-flash" + }, + "empiriolabs/mimo-v2-6-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 4.35e-07, + "output_cost_per_token": 8.7e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro" + }, + "empiriolabs/mimo-v2-6-pro-ultraspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 4.35e-06, + "output_cost_per_token": 8.7e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed" + }, + "empiriolabs/minimax-m2-7": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m2-7" + }, + "empiriolabs/minimax-m2-7-highspeed": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m2-7-highspeed" + }, + "empiriolabs/minimax-m3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.25e-07, + "output_cost_per_token": 9e-07, + "input_cost_per_token_above_512k_tokens": 4.5e-07, + "output_cost_per_token_above_512k_tokens": 1.8e-06, + "cache_read_input_token_cost_above_512k_tokens": 9e-08, + "cache_read_input_token_cost": 4.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3" + }, + "empiriolabs/minimax-m3-1-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 1.125e-07, + "output_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 1.125e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3-1-flash" + }, + "empiriolabs/minimax-m3:priority": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048458, + "max_output_tokens": 524288, + "input_cost_per_token": 3.375e-07, + "output_cost_per_token": 1.35e-06, + "input_cost_per_token_above_512k_tokens": 6.75e-07, + "output_cost_per_token_above_512k_tokens": 2.7e-06, + "cache_read_input_token_cost_above_512k_tokens": 1.35e-07, + "cache_read_input_token_cost": 6.75e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/minimax-m3" + }, + "empiriolabs/mistral-small-4": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.084, + "search_context_size_medium": 0.084, + "search_context_size_high": 0.14 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/mistral-small-4" + }, + "empiriolabs/muse-glimmer-30b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, + "cache_read_input_token_cost": 5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/muse-glimmer-30b" + }, + "empiriolabs/muse-spark-1-1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-1" + }, + "empiriolabs/muse-spark-1-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-2" + }, + "empiriolabs/muse-spark-1-3": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 4.25e-06, + "cache_read_input_token_cost": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/muse-spark-1-3" + }, + "empiriolabs/nova-lite-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 317379, + "max_output_tokens": 10000, + "input_cost_per_token": 6.9e-08, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 3.86e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-lite-1-0" + }, + "empiriolabs/nova-lite-2": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 3.16e-06, + "cache_read_input_token_cost": 2.128e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-lite-2" + }, + "empiriolabs/nova-micro-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 133003, + "max_output_tokens": 10000, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 1.6e-07, + "cache_read_input_token_cost": 2.24e-08, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://empiriolabs.ai/models/nova-micro-1-0" + }, + "empiriolabs/nova-pro-1-0": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 317319, + "max_output_tokens": 10000, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 9.6e-06, + "supports_system_messages": true, + "supports_vision": true, + "source": "https://empiriolabs.ai/models/nova-pro-1-0" + }, + "empiriolabs/perplexity-advanced-deep-research": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_output_tokens": 16000, + "input_cost_per_token": 1.2e-05, + "output_cost_per_token": 6e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.012, + "search_context_size_medium": 0.012, + "search_context_size_high": 0.012 + }, + "source": "https://empiriolabs.ai/models/perplexity-advanced-deep-research" + }, + "empiriolabs/perplexity-deep-research": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919327, + "input_cost_per_token": 4.8e-06, + "output_cost_per_token": 1.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-deep-research" + }, + "empiriolabs/perplexity-pro-search": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_output_tokens": 32768, + "input_cost_per_token": 7.8e-06, + "output_cost_per_token": 3.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-pro-search" + }, + "empiriolabs/perplexity-sonar": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919327, + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 2.4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar" + }, + "empiriolabs/perplexity-sonar-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919328, + "max_output_tokens": 32768, + "input_cost_per_token": 7.2e-06, + "output_cost_per_token": 3.6e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar-pro" + }, + "empiriolabs/perplexity-sonar-reasoning-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 919330, + "max_output_tokens": 131072, + "input_cost_per_token": 4.8e-06, + "output_cost_per_token": 1.9e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "source": "https://empiriolabs.ai/models/perplexity-sonar-reasoning-pro" + }, + "empiriolabs/qwen3-5-122b-a10b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 1.15e-07, + "output_cost_per_token": 9.17e-07, + "input_cost_per_token_above_128k_tokens": 2.87e-07, + "output_cost_per_token_above_128k_tokens": 2.294e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b" + }, + "empiriolabs/qwen3-5-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 8.6e-08, + "output_cost_per_token": 6.88e-07, + "input_cost_per_token_above_128k_tokens": 2.58e-07, + "output_cost_per_token_above_128k_tokens": 2.064e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-27b" + }, + "empiriolabs/qwen3-5-35b-a3b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 5.7e-08, + "output_cost_per_token": 4.59e-07, + "input_cost_per_token_above_128k_tokens": 2.29e-07, + "output_cost_per_token_above_128k_tokens": 1.835e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b" + }, + "empiriolabs/qwen3-5-397b-a17b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 258048, + "max_output_tokens": 64000, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 1.032e-06, + "input_cost_per_token_above_128k_tokens": 4.3e-07, + "output_cost_per_token_above_128k_tokens": 2.58e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b" + }, + "empiriolabs/qwen3-5-4b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 7e-08, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-4b" + }, + "empiriolabs/qwen3-5-9b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.3e-07, + "cache_read_input_token_cost": 4.5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-9b" + }, + "empiriolabs/qwen3-5-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 995904, + "max_output_tokens": 32768, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 3.68e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-flash" + }, + "empiriolabs/qwen3-5-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 995904, + "max_output_tokens": 65536, + "input_cost_per_token": 2.9e-08, + "output_cost_per_token": 2.87e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-flash" + }, + "empiriolabs/qwen3-5-omni-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 4.4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash" + }, + "empiriolabs/qwen3-5-omni-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 2.8e-06, + "output_cost_per_token": 1.66e-05, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus" + }, + "empiriolabs/qwen3-5-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 2.21e-06, + "input_cost_per_token_above_256k_tokens": 1.08e-06, + "output_cost_per_token_above_256k_tokens": 6.62e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-plus" + }, + "empiriolabs/qwen3-5-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.15e-07, + "output_cost_per_token": 6.88e-07, + "input_cost_per_token_above_128k_tokens": 2.87e-07, + "output_cost_per_token_above_128k_tokens": 1.72e-06, + "input_cost_per_token_above_256k_tokens": 5.73e-07, + "output_cost_per_token_above_256k_tokens": 3.44e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-5-plus" + }, + "empiriolabs/qwen3-6-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229376, + "max_output_tokens": 64000, + "input_cost_per_token": 4.12564e-07, + "output_cost_per_token": 2.475384e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-27b" + }, + "empiriolabs/qwen3-6-35b-a3b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 4.2e-07, + "cache_read_input_token_cost": 3.5e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b" + }, + "empiriolabs/qwen3-6-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.5e-06, + "input_cost_per_token_above_256k_tokens": 1e-06, + "output_cost_per_token_above_256k_tokens": 4e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-flash" + }, + "empiriolabs/qwen3-6-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.65e-07, + "output_cost_per_token": 9.9e-07, + "input_cost_per_token_above_256k_tokens": 6.6e-07, + "output_cost_per_token_above_256k_tokens": 3.961e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-flash" + }, + "empiriolabs/qwen3-6-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "input_cost_per_token_above_256k_tokens": 2e-06, + "output_cost_per_token_above_256k_tokens": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.026, + "search_context_size_medium": 0.026, + "search_context_size_high": 0.026 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-plus" + }, + "empiriolabs/qwen3-6-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.76e-07, + "output_cost_per_token": 1.651e-06, + "input_cost_per_token_above_256k_tokens": 1.101e-06, + "output_cost_per_token_above_256k_tokens": 6.602e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-6-plus" + }, + "empiriolabs/qwen3-7-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 1.3e-07, + "input_cost_per_token_above_32k_tokens": 1e-07, + "output_cost_per_token_above_32k_tokens": 4e-07, + "cache_read_input_token_cost_above_32k_tokens": 2e-08, + "input_cost_per_token_above_256k_tokens": 2e-07, + "output_cost_per_token_above_256k_tokens": 8e-07, + "cache_read_input_token_cost_above_256k_tokens": 4e-08, + "cache_read_input_token_cost": 6e-09, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.03, + "search_context_size_medium": 0.03, + "search_context_size_high": 0.03 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-flash" + }, + "empiriolabs/qwen3-7-flash:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.8e-08, + "output_cost_per_token": 1.1e-07, + "input_cost_per_token_above_32k_tokens": 8.3e-08, + "output_cost_per_token_above_32k_tokens": 3.3e-07, + "cache_read_input_token_cost_above_32k_tokens": 1.7e-08, + "input_cost_per_token_above_256k_tokens": 1.65e-07, + "output_cost_per_token_above_256k_tokens": 6.6e-07, + "cache_read_input_token_cost_above_256k_tokens": 3.3e-08, + "cache_read_input_token_cost": 6e-09, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-flash" + }, + "empiriolabs/qwen3-7-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-max" + }, + "empiriolabs/qwen3-7-max:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-max" + }, + "empiriolabs/qwen3-7-plus": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "input_cost_per_token_above_256k_tokens": 1.2e-06, + "output_cost_per_token_above_256k_tokens": 4.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.03, + "search_context_size_medium": 0.03, + "search_context_size_high": 0.03 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-plus" + }, + "empiriolabs/qwen3-7-plus:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 65536, + "input_cost_per_token": 2.76e-07, + "output_cost_per_token": 1.101e-06, + "input_cost_per_token_above_256k_tokens": 8.26e-07, + "output_cost_per_token_above_256k_tokens": 3.301e-06, + "cache_read_input_token_cost_above_256k_tokens": 1.66e-07, + "cache_read_input_token_cost": 5.6e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-7-plus" + }, + "empiriolabs/qwen3-8-27b": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 1.7e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 8e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-27b" + }, + "empiriolabs/qwen3-8-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 1.6e-07, + "output_cost_per_token": 4.7e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-flash" + }, + "empiriolabs/qwen3-8-max": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max" + }, + "empiriolabs/qwen3-8-max-0902": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max-0902" + }, + "empiriolabs/qwen3-8-max:variant1": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 983616, + "max_output_tokens": 131072, + "input_cost_per_token": 1.65e-06, + "output_cost_per_token": 4.951e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/qwen3-8-max" + }, + "empiriolabs/qwen3-8-omni-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1006592, + "max_output_tokens": 131072, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 9.4e-07, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.02, + "search_context_size_medium": 0.02, + "search_context_size_high": 0.02 + }, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash" + }, + "empiriolabs/seed-2-0-code": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 2.4e-06, + "input_cost_per_token_above_128k_tokens": 8e-07, + "output_cost_per_token_above_128k_tokens": 4.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "source": "https://empiriolabs.ai/models/seed-2-0-code" + }, + "empiriolabs/seed-2-0-lite": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 3.1e-07, + "output_cost_per_token": 2.5e-06, + "input_cost_per_token_above_128k_tokens": 6.2e-07, + "output_cost_per_token_above_128k_tokens": 5e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-lite" + }, + "empiriolabs/seed-2-0-mini": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229156, + "max_output_tokens": 128000, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 5e-07, + "input_cost_per_token_above_128k_tokens": 2.4e-07, + "output_cost_per_token_above_128k_tokens": 1e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-mini" + }, + "empiriolabs/seed-2-0-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229096, + "max_output_tokens": 128000, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 3.79e-06, + "input_cost_per_token_above_128k_tokens": 1.26e-06, + "output_cost_per_token_above_128k_tokens": 7.58e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-0-pro" + }, + "empiriolabs/seed-2-1-turbo": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 229093, + "max_output_tokens": 65536, + "input_cost_per_token": 6.3e-07, + "output_cost_per_token": 3.13e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/seed-2-1-turbo" + }, + "empiriolabs/skylark-embedding-vision": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 8000, + "input_cost_per_token": 2.5e-07, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/skylark-embedding-vision" + }, + "empiriolabs/step-3-5-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/step-3-5-flash" + }, + "empiriolabs/step-3-5-flash-2603": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, + "cache_read_input_token_cost": 2e-08, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/step-3-5-flash-2603" + }, + "empiriolabs/step-3-7-flash": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.15e-06, + "cache_read_input_token_cost": 4e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/step-3-7-flash" + }, + "empiriolabs/step-5-preview": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1024000, + "max_output_tokens": 131072, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 5e-08, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "source": "https://empiriolabs.ai/models/step-5-preview" + }, + "empiriolabs/stepaudio-2-5-chat": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 3e-07, + "supports_system_messages": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.013, + "search_context_size_medium": 0.013, + "search_context_size_high": 0.013 + }, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/stepaudio-2-5-chat" + }, + "empiriolabs/stepaudio-3-chat": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 3e-07, + "supports_system_messages": true, + "supports_reasoning": true, + "supports_prompt_caching": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_audio_input": true, + "source": "https://empiriolabs.ai/models/stepaudio-3-chat" + }, + "empiriolabs/text-embedding-v4": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 8192, + "input_cost_per_token": 7e-08, + "source": "https://empiriolabs.ai/models/text-embedding-v4" + }, + "empiriolabs/tongyi-embedding-vision-flash": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 1024, + "input_cost_per_token": 9e-08, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/tongyi-embedding-vision-flash" + }, + "empiriolabs/tongyi-embedding-vision-plus": { + "litellm_provider": "empiriolabs", + "mode": "embedding", + "max_input_tokens": 1024, + "input_cost_per_token": 9e-08, + "supports_embedding_image_input": true, + "source": "https://empiriolabs.ai/models/tongyi-embedding-vision-plus" + }, + "wally/glm-5.3-flash": { + "cache_read_input_token_cost": 7.6e-08, + "input_cost_per_token": 2.72e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048567, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048569, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/deepseek-v4.1-flash-fast": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "wally/glm-5.3-fast": { + "cache_read_input_token_cost": 3.9e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4-pro-fast": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 1.91e-06, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.83e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/minimax-m3": { + "cache_read_input_token_cost": 6.6e-08, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "wally", + "max_input_tokens": 524287, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "source": "https://www.runwally.com/pricing", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash-1017": { + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3.75e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://inference.runanywhere.ai/v1/models", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wally/deepseek-v4.1-flash-f2": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "wally", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "max" + ], + "source": "https://inference.runanywhere.ai/v1/models", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "together_ai/moonshotai/Kimi-K3.1-Pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "input_cost_per_token": 3.6e-06, + "input_cost_per_token_batches": 1.8e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "output_cost_per_token_batches": 9e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { + "cache_creation_input_token_cost": 4.5e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "input_cost_per_token": 3.6e-06, + "input_cost_per_token_batches": 1.8e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "output_cost_per_token_batches": 9e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { + "cache_creation_input_token_cost": 4.5e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.8e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.76e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "input_cost_per_token_batches": 1.2e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4.2e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "output_cost_per_token_batches": 3.6e-07, + "output_cost_per_token_flex": 3.6e-07, + "output_cost_per_token_priority": 1.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 5.04e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { + "input_cost_per_token": 7.2e-08, + "input_cost_per_token_batches": 3.6e-08, + "input_cost_per_token_flex": 3.6e-08, + "input_cost_per_token_priority": 1.32e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.76e-07, + "output_cost_per_token_batches": 1.44e-07, + "output_cost_per_token_flex": 1.44e-07, + "output_cost_per_token_priority": 4.8e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": false + }, + "bedrock_mantle/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.15e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "output_cost_per_token_batches": 3.9e-07, + "output_cost_per_token_flex": 3.9e-07, + "output_cost_per_token_priority": 1.365e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "gmi/MiniMaxAI/MiniMax-M2": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M2.7": { + "cache_read_input_token_cost": 4.2e-08, + "input_cost_per_token": 2.1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 8.4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 4.8e-08, + "input_cost_per_token": 2.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 9.6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.125e-08, + "input_cost_per_token": 1.125e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-FP8": { + "input_cost_per_token": 1.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.09e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8": { + "cache_read_input_token_cost": 1.75e-08, + "input_cost_per_token": 8.75e-08, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8": { + "input_cost_per_token": 3.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262141, + "mode": "chat", + "output_cost_per_token": 4.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-30B-A3B": { + "input_cost_per_token": 8e-08, + "litellm_provider": "gmi", + "max_input_tokens": 40946, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-32B-FP8": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 9e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262128, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Next-80B-A3B-Instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-122B-A10B": { + "input_cost_per_token": 4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-27B": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-35B-A3B": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.5-397B-A17B": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-35B-A3B": { + "input_cost_per_token": 2.48e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.485e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-Plus": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.6-Plus-2026-04-02": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.7-Max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-27B": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 4.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Flash": { + "cache_read_input_token_cost": 1.6e-08, + "input_cost_per_token": 1.6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Max": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/Qwen/Qwen3.8-Max-0902": { + "cache_read_input_token_cost": 2.06e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.951e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/XiaomiMiMo/MiMo-V2.5": { + "cache_read_input_token_cost": 2.38e-09, + "input_cost_per_token": 1.19e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 2.38e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/XiaomiMiMo/MiMo-V2.5-Pro": { + "cache_read_input_token_cost": 2.52e-09, + "input_cost_per_token": 3.045e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 6.09e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-Prover-V2-671B": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.18e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.18e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-0528": { + "cache_read_input_token_cost": 2.3e-07, + "input_cost_per_token": 5.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.29e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Llama-8B": { + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3.9e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 9e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.1-Terminus": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.2-Exp": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 4.1e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V3.2-Speciale": { + "input_cost_per_token": 2.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Flash": { + "cache_read_input_token_cost": 1.82e-08, + "input_cost_per_token": 9.1e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 1.82e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 9.1e-09, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 8.58e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 3.52e-08, + "input_cost_per_token": 1.056e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 3.168e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/deepseek-ai/deepseek-v4-flash-vision-exp": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.1-flash-lite-preview": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.1-pro-preview": { + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.5-flash": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 9e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.7-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-3.8-flash": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.75e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemini-4-argon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1.3e-07, + "litellm_provider": "gmi", + "max_input_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + "input_cost_per_token": 8e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 5.7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.29e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.44e-07, + "input_cost_per_token": 8.55e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k2.7-code-highspeed": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/moonshotai/kimi-k3.1-pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/nvidia/nemotron-3-ultra-550b-a55b": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.4-pro": { + "input_cost_per_token": 3e-05, + "litellm_provider": "gmi", + "max_input_tokens": 409600, + "mode": "chat", + "output_cost_per_token": 0.00018, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.5": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-luna": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-sol": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-5.6-terra": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-astra": { + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-luna": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6-sol": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-6.1-sol": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1050000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/openai/gpt-oss-120b": { + "input_cost_per_token": 5e-08, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/stepfun-ai/Step-3.7-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/stepfun-ai/Step-5-Preview": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.7e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/HunyuanOCR": { + "input_cost_per_token": 5e-07, + "litellm_provider": "gmi", + "max_input_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/Hy3": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 5.8e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/hy3-preview": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 1.8e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/tencent/hy4-preview": { + "cache_read_input_token_cost": 4.2e-08, + "input_cost_per_token": 8.34e-07, + "litellm_provider": "gmi", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/x-ai/grok-4.5": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/xai/grok4.6": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 2000000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131076, + "mode": "chat", + "output_cost_per_token": 1.1e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.5-FP8": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.6": { + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 204800, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-4.7-Flash": { + "input_cost_per_token": 7e-08, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5-FP8": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 1.92e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.1-FP8": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 202752, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.2-FP8": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.3e-07, + "input_cost_per_token": 7e-07, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "gmi/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3p1-pro": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_priority": 3e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 1.5e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3.1-Pro": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1000000, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "gpt-image-2.6-sunburst": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "gpt-image-2.6-flare": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 8e-06, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 3e-05, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ], + "supports_pdf_input": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-west-1/moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 4.95e-06, + "cache_creation_input_token_cost_flex": 2.475e-06, + "cache_creation_input_token_cost_priority": 8.6625e-06, + "cache_read_input_token_cost": 3.96e-07, + "cache_read_input_token_cost_flex": 1.98e-07, + "cache_read_input_token_cost_priority": 6.93e-07, + "input_cost_per_token": 3.96e-06, + "input_cost_per_token_batches": 1.98e-06, + "input_cost_per_token_flex": 1.98e-06, + "input_cost_per_token_priority": 6.93e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 868928, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-05, + "output_cost_per_token_batches": 9.9e-06, + "output_cost_per_token_flex": 9.9e-06, + "output_cost_per_token_priority": 3.465e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_cache_breakpoint": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_regex_lookaround": false, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 64.0, + "token_count_tokenizer": "kimi_k3" + }, + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json", + "successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-east-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { + "deprecation_date": "2026-09-10", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md", + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 195904, + "max_output_tokens": 4096, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", + "successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0", + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.7": { + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "bedrock_mantle/xai.grok-4.7": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_flex": 2.75e-07, + "cache_read_input_token_cost_priority": 9.625e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "output_cost_per_token_batches": 3.3e-06, + "output_cost_per_token_flex": 3.3e-06, + "output_cost_per_token_priority": 1.155e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "use_openai_responses_path": true + }, + "together_ai/ricursive-ai/Ripple-65B-Preview": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.together.ai/v1/models", + "supports_prompt_caching": true } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 700a35edb06..00fee0a0d5a 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3158,7 +3158,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } @@ -5250,13 +5250,13 @@ "generateContent": false } }, - "empiriolabs": { - "display_name": "EmpirioLabs (`empiriolabs`)", - "url": "https://docs.litellm.ai/docs/providers/empiriolabs", + "wally": { + "display_name": "Wally (`wally`)", + "url": "https://docs.litellm.ai/docs/providers/wally", "endpoints": { - "chat_completions": false, - "responses": false, - "text_completion": false, + "chat_completions": true, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, "image_edits": false, @@ -5269,11 +5269,37 @@ "realtime": false, "video_generations": false, "batches": false, - "messages": false, - "interactions": false, + "messages": true, + "interactions": true, "systemone": false, - "a2a": false, - "generateContent": false + "a2a": true, + "generateContent": true + } + }, + "empiriolabs": { + "display_name": "EmpirioLabs AI (`empiriolabs`)", + "url": "https://docs.litellm.ai/docs/providers/empiriolabs", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "openinfer": { diff --git a/pyproject.toml b/pyproject.toml index 7a653b28f81..2cc2d0f70a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,7 +155,8 @@ stt-vertex-chirp = [ stt-nvidia-riva = [ # NVIDIA Riva STT provider (gRPC). These are imported lazily inside the # provider handler so litellm core remains usable without them. - "nvidia-riva-client>=2.15.0", + "grpcio>=1.67.1,<2.0", + "protobuf>=5.26.1,<7.0", "soundfile>=0.12.1", "audioread>=3.0.1", "numpy>=1.26.0", @@ -241,6 +242,7 @@ dev = [ "pytest-timeout==2.4.0", "vcrpy==8.2.1", "pytest-recording==0.13.4", + "nvidia-riva-client==2.16.0", ] e2e-dev = [ "playwright==1.61.0", @@ -266,6 +268,7 @@ proxy-dev = [ "azure-identity==1.25.2", "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", + "httptools>=0.7.1,<0.8", ] ci = [ "psutil==7.2.2", @@ -274,6 +277,9 @@ ci = [ # protobuf, Pillow is a compiled C extension). "tenacity==8.5.0", "google-generativeai==0.8.6", + # The Riva tests serve NVIDIA's own message definitions, which checks + # LiteLLM's hand-written copy of them on every run. + "nvidia-riva-client==2.16.0", "Pillow==12.3.0; python_version == '3.10'", "Pillow==13.0.0; python_version >= '3.11'", # Azure batch E2E tests still import psycopg2 directly. diff --git a/schema.prisma b/schema.prisma index 179b741fe7c..78f16d9f1fd 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1615,9 +1615,18 @@ model LiteLLM_ClaudeCodePluginTable { updated_at DateTime? @default(now()) @updatedAt created_by String? + uploaded_archive LiteLLM_ClaudeCodePluginUpload? + @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginUpload { + plugin_name String @id + content Bytes + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_name], references: [name], onDelete: Cascade, onUpdate: Cascade) +} + + // User/team-scoped memory store with a GLOBAL unique key. // `value` is a string (typically markdown/text meant for LLM context); // `metadata` is an optional JSON envelope for structured tags without schema changes. @@ -2062,74 +2071,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/scripts/check_mcp_sdk_install.py b/scripts/check_mcp_sdk_install.py index f2c2891bfd8..f5261bc22f8 100644 --- a/scripts/check_mcp_sdk_install.py +++ b/scripts/check_mcp_sdk_install.py @@ -13,6 +13,7 @@ IMPORTED_MODULES: Final[tuple[str, ...]] = ( "litellm.proxy.mcp_server.server", "litellm.proxy.mcp_server.mcp_server_manager", "litellm.proxy.mcp_server.rest_endpoints", + "litellm.proxy.proxy_server", ) diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 26bb034bfa6..219ef912952 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -18,6 +18,7 @@ from collections.abc import Iterator, Mapping from typing import Final, NamedTuple GATED_METHODS: Final = frozenset({"post"}) +SCHEMA_MEDIA_TYPES: Final = ("application/json", "application/sdp", "multipart/form-data") class Violation(NamedTuple): @@ -30,18 +31,21 @@ def _as_mapping(node: object) -> Mapping[str, object]: return node if isinstance(node, Mapping) else {} -def _json_schema(container: object) -> object: +def _content_schema(container: object) -> object: content: Final = _as_mapping(_as_mapping(container).get("content")) - return _as_mapping(content.get("application/json")).get("schema") + return next( + (schema for media_type in SCHEMA_MEDIA_TYPES if (schema := _as_mapping(content.get(media_type)).get("schema"))), + None, + ) def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: - if not _json_schema(operation.get("requestBody")): + if not _content_schema(operation.get("requestBody")): yield "OA001" description: Final = operation.get("description") if not isinstance(description, str) or not description.strip(): yield "OA002" - if not _json_schema(_as_mapping(operation.get("responses")).get("200")): + if not _content_schema(_as_mapping(operation.get("responses")).get("200")): yield "OA003" diff --git a/scripts/check_schema_compatibility.py b/scripts/check_schema_compatibility.py index 898c65dff2c..c56986a38c8 100644 --- a/scripts/check_schema_compatibility.py +++ b/scripts/check_schema_compatibility.py @@ -13,6 +13,7 @@ from pydantic import TypeAdapter Schema = Mapping[str, Mapping[str, str]] MODELS: Final = re.compile(r"^[ \t]*model\s+(\w+)\s*\{(.*?)^[ \t]*\}", re.MULTILINE | re.DOTALL) +DROPPED_UNUSED_MODELS: Final = frozenset({"LiteLLM_AccountingDelivery", "LiteLLM_AccountingEvent"}) def parse_model_fields(schema: str) -> Schema: @@ -27,18 +28,18 @@ def _field_changes(model: str, field: str, old_type: str, new_fields: Mapping[st if field not in new_fields: yield f"Breaking: Field {model}.{field} was removed" return - new_type: Final = new_fields[field] - if old_type == new_type: - return - if "?" in old_type and "?" not in new_type: + old_base: Final = old_type.split()[0] + new_base: Final = new_fields[field].split()[0] + if old_base.endswith("?") and not new_base.endswith("?"): yield f"Breaking: Field {model}.{field} changed from optional to required" - if not old_type.startswith(new_type.split("?")[0]): - yield f"Breaking: Field {model}.{field} changed type from {old_type} to {new_type}" + if old_base.rstrip("?") != new_base.rstrip("?"): + yield f"Breaking: Field {model}.{field} changed type from {old_base} to {new_base}" def _model_changes(model: str, old_fields: Mapping[str, str], new_schema: Schema) -> Iterator[str]: if model not in new_schema: - yield f"Breaking: Model {model} was removed" + if model not in DROPPED_UNUSED_MODELS: + yield f"Breaking: Model {model} was removed" return for field, old_type in old_fields.items(): yield from _field_changes(model, field, old_type, new_schema[model]) @@ -52,14 +53,20 @@ def check_breaking_changes(old_schema: Schema, new_schema: Schema) -> tuple[str, return tuple(changes()) -def main() -> int: - parser: Final = argparse.ArgumentParser() - parser.add_argument("--base-ref", required=True) - options: Final = parser.parse_args() - base_ref: Final = TypeAdapter(str).validate_python(options.base_ref) - base: Final = subprocess.run( +def _merge_base(base_ref: str) -> str: + return subprocess.run( ("git", "merge-base", "HEAD", base_ref), capture_output=True, text=True, check=True ).stdout.strip() + + +def main() -> int: + parser: Final = argparse.ArgumentParser() + source: Final = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--base-ref", help="compare against the merge base of HEAD and this ref (needs history)") + source.add_argument("--base-commit", help="compare against this commit as is (works in a shallow clone)") + options: Final = TypeAdapter(dict[str, str | None]).validate_python(vars(parser.parse_args())) + base_commit: Final = options["base_commit"] + base: Final = base_commit if base_commit else _merge_base(options["base_ref"] or "") previous: Final = subprocess.run( ("git", "show", f"{base}:schema.prisma"), capture_output=True, text=True, check=True ).stdout diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index bedb6f4e98b..6f0450c0091 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -212,33 +212,23 @@ orthogonal to the OTLP variables above; their credentials still go in ### Enterprise billing metrics -License-gated request metering is opt-in and gated entirely on -`billing_metrics_endpoint`. Empty (default) and no billing env is added to -the container, so existing deployments are unchanged. Set it and both -gateway and backend export billable-request counts over OTLP/HTTP, -authenticating to the collector with the mTLS client certificate issued for -your deployment. - -The proxy accepts the certificate, key, and CA bundle as either a file path -or literal PEM content. This stack takes the PEM, writes each one to its own -Secrets Manager entry, grants the task-execution role -`secretsmanager:GetSecretValue` on them, and injects them as -`LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and `_CA_CERT` when -set), so no volume mount is needed on Fargate. +License-gated request metering uses Cognito bearer authentication. Set the +endpoint, client ID, and token URL as plain values, and provide the client +secret through the sensitive Terraform variable. The client secret is stored +in Secrets Manager and injected into both gateway and backend tasks ```hcl billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics" +billing_metrics_client_id = "cognito-client-id" +billing_metrics_token_url = "https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token" ``` ```bash -export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)" -export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)" +export TF_VAR_billing_metrics_client_secret="cognito-client-secret" ``` -`billing_metrics_ca_cert_pem` is only for private or test collectors whose -CA is not in the system trust store; leave it empty against -`telemetry.litellm.ai`. Metering requires an enterprise license, so pair -this with `litellm_license`. To tune the export cadence, set +The client ID and client secret are required when the endpoint is set +Metering also requires an enterprise license. To tune the export cadence, set `LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` / `backend_extra_env` diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 412e6c423f5..dec7f599ced 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -79,32 +79,19 @@ locals { { name = "OTEL_HEADERS", valueFrom = var.otel_headers_secret_arn }, ] : [] - # Enterprise request metering, gated on billing_metrics_endpoint. The - # endpoint rides in as a plain env var; the mTLS material is stored in - # Secrets Manager (secrets.tf) and injected as PEM-valued env vars, which - # the proxy accepts in place of file paths. Each PEM is wired only when the - # operator supplied it, so an empty ca_cert_pem falls back to the system - # trust store. - billing_metrics_enabled = var.billing_metrics_endpoint != "" - billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != "" - billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != "" - billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != "" + # Enterprise request metering, gated on billing_metrics_endpoint. + billing_metrics_enabled = var.billing_metrics_endpoint != "" + billing_metrics_client_secret_enabled = local.billing_metrics_enabled && var.billing_metrics_client_secret != "" billing_metrics_env = local.billing_metrics_enabled ? [ { name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint }, + { name = "LITELLM_BILLING_METRICS_CLIENT_ID", value = var.billing_metrics_client_id }, + { name = "LITELLM_BILLING_METRICS_TOKEN_URL", value = var.billing_metrics_token_url }, ] : [] - billing_metrics_secrets = concat( - local.billing_metrics_client_cert_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CLIENT_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_client_cert[0].arn }, - ] : [], - local.billing_metrics_client_key_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CLIENT_KEY", valueFrom = aws_secretsmanager_secret.billing_metrics_client_key[0].arn }, - ] : [], - local.billing_metrics_ca_cert_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CA_CERT", valueFrom = aws_secretsmanager_secret.billing_metrics_ca_cert[0].arn }, - ] : [], - ) + billing_metrics_secrets = local.billing_metrics_client_secret_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_SECRET", valueFrom = aws_secretsmanager_secret.billing_metrics_client_secret[0].arn }, + ] : [] managed_db_env = var.create_database ? [ { name = "IAM_TOKEN_DB_AUTH", value = "true" }, @@ -338,27 +325,12 @@ locals { # ---------- Gateway ---------- resource "aws_ecs_task_definition" "gateway" { - # Metering needs a client certificate AND its key. Each secret is created only - # when its own PEM is supplied, so an endpoint set with a missing key would - # otherwise apply cleanly and leave the proxy logging "missing config" and - # never exporting. ca_cert_pem stays optional: empty means fall back to the - # system trust store. - # - # The guard lives here, on an unconditional resource, rather than on the cert - # secret: that secret is count-gated on the cert itself, so it has zero - # instances in exactly the case this must catch. Adding count or for_each to - # this resource would silently stop the guard from evaluating. - # - # endpoint cert key -> result - # "" any any -> metering off, no secrets created - # set set set -> metering on - # set any-missing -> plan fails here lifecycle { precondition { condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" + var.billing_metrics_client_id != "" && var.billing_metrics_client_secret != "" ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." + error_message = "billing_metrics_client_id and billing_metrics_client_secret are required when billing_metrics_endpoint is set." } precondition { @@ -465,9 +437,7 @@ resource "aws_ecs_service" "gateway" { aws_secretsmanager_secret_version.license, aws_secretsmanager_secret_version.database_url, aws_secretsmanager_secret_version.redis_url, - aws_secretsmanager_secret_version.billing_metrics_client_cert, - aws_secretsmanager_secret_version.billing_metrics_client_key, - aws_secretsmanager_secret_version.billing_metrics_ca_cert, + aws_secretsmanager_secret_version.billing_metrics_client_secret, ] tags = local.tags @@ -481,9 +451,9 @@ resource "aws_ecs_task_definition" "backend" { lifecycle { precondition { condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" + var.billing_metrics_client_id != "" && var.billing_metrics_client_secret != "" ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." + error_message = "billing_metrics_client_id and billing_metrics_client_secret are required when billing_metrics_endpoint is set." } } @@ -566,9 +536,7 @@ resource "aws_ecs_service" "backend" { aws_secretsmanager_secret_version.ui_password, aws_secretsmanager_secret_version.database_url, aws_secretsmanager_secret_version.redis_url, - aws_secretsmanager_secret_version.billing_metrics_client_cert, - aws_secretsmanager_secret_version.billing_metrics_client_key, - aws_secretsmanager_secret_version.billing_metrics_ca_cert, + aws_secretsmanager_secret_version.billing_metrics_client_secret, ] tags = local.tags diff --git a/terraform/litellm/aws/iam.tf b/terraform/litellm/aws/iam.tf index 3c55f07b02a..a653bd82d31 100644 --- a/terraform/litellm/aws/iam.tf +++ b/terraform/litellm/aws/iam.tf @@ -53,9 +53,7 @@ data "aws_iam_policy_document" "secrets_access" { [aws_secretsmanager_secret.master_key.arn], aws_secretsmanager_secret.license[*].arn, aws_secretsmanager_secret.ui_password[*].arn, - aws_secretsmanager_secret.billing_metrics_client_cert[*].arn, - aws_secretsmanager_secret.billing_metrics_client_key[*].arn, - aws_secretsmanager_secret.billing_metrics_ca_cert[*].arn, + aws_secretsmanager_secret.billing_metrics_client_secret[*].arn, aws_secretsmanager_secret.database_url[*].arn, aws_secretsmanager_secret.redis_url[*].arn, local.extra_secret_arns, diff --git a/terraform/litellm/aws/secrets.tf b/terraform/litellm/aws/secrets.tf index 921bae4d827..6821b8b43dc 100644 --- a/terraform/litellm/aws/secrets.tf +++ b/terraform/litellm/aws/secrets.tf @@ -75,59 +75,21 @@ resource "aws_secretsmanager_secret_version" "ui_password" { secret_string = var.ui_password } -# Billing-metrics mTLS material — only created when metering is enabled -# (billing_metrics_endpoint non-empty) and the operator supplied the PEM. -# The task-execution role gets GetSecretValue via iam.tf, and gateway + -# backend pick the env vars up through shared_secrets in ecs.tf. -resource "aws_secretsmanager_secret" "billing_metrics_client_cert" { - count = local.billing_metrics_client_cert_enabled ? 1 : 0 +resource "aws_secretsmanager_secret" "billing_metrics_client_secret" { + count = local.billing_metrics_client_secret_enabled ? 1 : 0 - name = "${local.name}-billing-metrics-client-cert" - description = "LITELLM_BILLING_METRICS_CLIENT_CERT for gateway + backend." + name = "${local.name}-billing-metrics-client-secret" + description = "LITELLM_BILLING_METRICS_CLIENT_SECRET for gateway + backend." recovery_window_in_days = 0 tags = local.tags } -resource "aws_secretsmanager_secret_version" "billing_metrics_client_cert" { - count = local.billing_metrics_client_cert_enabled ? 1 : 0 +resource "aws_secretsmanager_secret_version" "billing_metrics_client_secret" { + count = local.billing_metrics_client_secret_enabled ? 1 : 0 - secret_id = aws_secretsmanager_secret.billing_metrics_client_cert[0].id - secret_string = var.billing_metrics_client_cert_pem -} - -resource "aws_secretsmanager_secret" "billing_metrics_client_key" { - count = local.billing_metrics_client_key_enabled ? 1 : 0 - - name = "${local.name}-billing-metrics-client-key" - description = "LITELLM_BILLING_METRICS_CLIENT_KEY for gateway + backend." - recovery_window_in_days = 0 - - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "billing_metrics_client_key" { - count = local.billing_metrics_client_key_enabled ? 1 : 0 - - secret_id = aws_secretsmanager_secret.billing_metrics_client_key[0].id - secret_string = var.billing_metrics_client_key_pem -} - -resource "aws_secretsmanager_secret" "billing_metrics_ca_cert" { - count = local.billing_metrics_ca_cert_enabled ? 1 : 0 - - name = "${local.name}-billing-metrics-ca-cert" - description = "LITELLM_BILLING_METRICS_CA_CERT for gateway + backend." - recovery_window_in_days = 0 - - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "billing_metrics_ca_cert" { - count = local.billing_metrics_ca_cert_enabled ? 1 : 0 - - secret_id = aws_secretsmanager_secret.billing_metrics_ca_cert[0].id - secret_string = var.billing_metrics_ca_cert_pem + secret_id = aws_secretsmanager_secret.billing_metrics_client_secret[0].id + secret_string = var.billing_metrics_client_secret } resource "aws_secretsmanager_secret" "db_master_password" { diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index 9b7a279a907..be7656c2d48 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -764,13 +764,7 @@ variable "otel_headers_secret_arn" { # ---------- Enterprise billing metrics ---------- # -# License-gated request metering. Opt-in and gated entirely on -# billing_metrics_endpoint: leave it empty (the default) and nothing -# metering-related lands in the container env. Set it and gateway + backend -# export billable-request counts over OTLP/HTTP, authenticating to the -# collector with an mTLS client cert. The proxy accepts the cert, key, and CA -# as either a file path or literal PEM content, so on Fargate they are -# injected straight from Secrets Manager as env vars and no volume is needed. +# License-gated request metering. The client secret is stored in Secrets Manager. variable "billing_metrics_endpoint" { description = <<-EOT @@ -784,44 +778,34 @@ variable "billing_metrics_endpoint" { default = "" } -variable "billing_metrics_client_cert_pem" { +variable "billing_metrics_client_id" { description = <<-EOT - PEM content of the mTLS client certificate issued for this deployment. - When billing_metrics_endpoint is set, the stack stores this in a - `-litellm--billing-metrics-client-cert` Secrets Manager - entry, grants the task-execution role GetSecretValue on it, and exposes - it to gateway + backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required - whenever metering is enabled. + Cognito OAuth2 client ID exposed to the gateway and backend as + LITELLM_BILLING_METRICS_CLIENT_ID. Required when billing_metrics_endpoint + is set. + EOT + type = string + default = "" +} + +variable "billing_metrics_client_secret" { + description = <<-EOT + Cognito OAuth2 client secret stored in Secrets Manager and exposed to + the gateway and backend as LITELLM_BILLING_METRICS_CLIENT_SECRET. + Required when billing_metrics_endpoint is set. EOT type = string default = "" sensitive = true } -variable "billing_metrics_client_key_pem" { +variable "billing_metrics_token_url" { description = <<-EOT - PEM content of the private key matching - billing_metrics_client_cert_pem. Stored in a - `-litellm--billing-metrics-client-key` Secrets Manager - entry and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required - whenever metering is enabled. + Cognito OAuth2 client-credentials token endpoint exposed to gateway and + backend as LITELLM_BILLING_METRICS_TOKEN_URL. EOT type = string - default = "" - sensitive = true -} - -variable "billing_metrics_ca_cert_pem" { - description = <<-EOT - PEM content of the CA bundle used to verify the metering collector. - Only needed for private or test collectors whose CA is not in the - system trust store; telemetry.litellm.ai is publicly trusted, so leave - this empty for production. When set, it is exposed as - LITELLM_BILLING_METRICS_CA_CERT. - EOT - type = string - default = "" - sensitive = true + default = "https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token" } # ---------- Collector sidecar ---------- diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 3227dcac179..9e21ba14543 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -210,33 +210,23 @@ Behavior matches the AWS stack 1:1; the only naming differences are ### Enterprise billing metrics -License-gated request metering is opt-in and gated entirely on -`billing_metrics_endpoint`. Empty (default) and no billing env is added to -the container, so existing deployments are unchanged. Set it and both -gateway and backend export billable-request counts over OTLP/HTTP, -authenticating to the collector with the mTLS client certificate issued for -your deployment. - -The proxy accepts the certificate, key, and CA bundle as either a file path -or literal PEM content. This stack takes the PEM, writes each one to its own -Secret Manager entry, grants the runtime service account -`roles/secretmanager.secretAccessor` on them, and injects them as Cloud Run -secret env vars `LITELLM_BILLING_METRICS_CLIENT_CERT` / `_CLIENT_KEY` (and -`_CA_CERT` when set), so no volume mount is needed. +License-gated request metering uses Cognito bearer authentication. Set the +endpoint, client ID, and token URL as plain values, and provide the client +secret through the sensitive Terraform variable. The client secret is stored +in Secret Manager and injected into both gateway and backend services ```hcl billing_metrics_endpoint = "https://telemetry.litellm.ai/v1/metrics" +billing_metrics_client_id = "cognito-client-id" +billing_metrics_token_url = "https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token" ``` ```bash -export TF_VAR_billing_metrics_client_cert_pem="$(cat client.crt)" -export TF_VAR_billing_metrics_client_key_pem="$(cat client.key)" +export TF_VAR_billing_metrics_client_secret="cognito-client-secret" ``` -`billing_metrics_ca_cert_pem` is only for private or test collectors whose -CA is not in the system trust store; leave it empty against -`telemetry.litellm.ai`. Metering requires an enterprise license, so pair -this with `litellm_license`. To tune the export cadence, set +The client ID and client secret are required when the endpoint is set +Metering also requires an enterprise license. To tune the export cadence, set `LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` / `backend_extra_env` diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index d0b32a367d6..07bf68b3a62 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -62,32 +62,19 @@ locals { { name = "OTEL_HEADERS", secret = var.otel_headers_secret, version = "latest" }, ] : [] - # Enterprise request metering, gated on billing_metrics_endpoint. The - # endpoint rides in as a plain env var; the mTLS material lives in Secret - # Manager (secrets.tf) and is injected as PEM-valued env vars, which the - # proxy accepts in place of file paths. Each PEM is wired only when the - # operator supplied it, so an empty ca_cert_pem falls back to the system - # trust store. - billing_metrics_enabled = var.billing_metrics_endpoint != "" - billing_metrics_client_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_client_cert_pem != "" - billing_metrics_client_key_enabled = local.billing_metrics_enabled && var.billing_metrics_client_key_pem != "" - billing_metrics_ca_cert_enabled = local.billing_metrics_enabled && var.billing_metrics_ca_cert_pem != "" + # Enterprise request metering, gated on billing_metrics_endpoint. + billing_metrics_enabled = var.billing_metrics_endpoint != "" + billing_metrics_client_secret_enabled = local.billing_metrics_enabled && var.billing_metrics_client_secret != "" billing_metrics_env_kv = local.billing_metrics_enabled ? [ { name = "LITELLM_BILLING_METRICS_ENDPOINT", value = var.billing_metrics_endpoint }, + { name = "LITELLM_BILLING_METRICS_CLIENT_ID", value = var.billing_metrics_client_id }, + { name = "LITELLM_BILLING_METRICS_TOKEN_URL", value = var.billing_metrics_token_url }, ] : [] - billing_metrics_env_secrets = concat( - local.billing_metrics_client_cert_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CLIENT_CERT", secret = google_secret_manager_secret.billing_metrics_client_cert[0].id, version = "latest" }, - ] : [], - local.billing_metrics_client_key_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CLIENT_KEY", secret = google_secret_manager_secret.billing_metrics_client_key[0].id, version = "latest" }, - ] : [], - local.billing_metrics_ca_cert_enabled ? [ - { name = "LITELLM_BILLING_METRICS_CA_CERT", secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id, version = "latest" }, - ] : [], - ) + billing_metrics_env_secrets = local.billing_metrics_client_secret_enabled ? [ + { name = "LITELLM_BILLING_METRICS_CLIENT_SECRET", secret = google_secret_manager_secret.billing_metrics_client_secret[0].id, version = "latest" }, + ] : [] # Cloud Run v2 secret env vars use value_source.secret_key_ref pointing at a # secret resource ID. Shared between gateway and backend (the migrations @@ -481,9 +468,7 @@ resource "google_cloud_run_v2_service" "gateway" { google_secret_manager_secret_iam_member.license, google_secret_manager_secret_iam_member.extras, google_secret_manager_secret_iam_member.otel_headers, - google_secret_manager_secret_iam_member.billing_metrics_client_cert, - google_secret_manager_secret_iam_member.billing_metrics_client_key, - google_secret_manager_secret_iam_member.billing_metrics_ca_cert, + google_secret_manager_secret_iam_member.billing_metrics_client_secret, google_secret_manager_secret_iam_member.metrics_run_monitoring, google_project_iam_member.runtime_metric_writer, google_storage_bucket_iam_member.proxy_config_runtime, @@ -603,9 +588,7 @@ resource "google_cloud_run_v2_service" "backend" { google_secret_manager_secret_iam_member.ui_password, google_secret_manager_secret_iam_member.extras, google_secret_manager_secret_iam_member.otel_headers, - google_secret_manager_secret_iam_member.billing_metrics_client_cert, - google_secret_manager_secret_iam_member.billing_metrics_client_key, - google_secret_manager_secret_iam_member.billing_metrics_ca_cert, + google_secret_manager_secret_iam_member.billing_metrics_client_secret, google_storage_bucket_iam_member.proxy_config_runtime, google_sql_user.app, terraform_data.migration, diff --git a/terraform/litellm/gcp/iam.tf b/terraform/litellm/gcp/iam.tf index 5c2bc38bc12..1e0a5fe9626 100644 --- a/terraform/litellm/gcp/iam.tf +++ b/terraform/litellm/gcp/iam.tf @@ -10,9 +10,9 @@ resource "google_service_account" "runtime" { lifecycle { precondition { condition = !var.create_runtime || var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" + var.billing_metrics_client_id != "" && var.billing_metrics_client_secret != "" ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set and create_runtime is true." + error_message = "billing_metrics_client_id and billing_metrics_client_secret are required when billing_metrics_endpoint is set and create_runtime is true." } } } @@ -91,28 +91,10 @@ resource "google_secret_manager_secret_iam_member" "otel_headers" { member = "serviceAccount:${google_service_account.runtime.email}" } -# Billing-metrics mTLS accessors — only created when request metering is -# enabled and the matching PEM was supplied. -resource "google_secret_manager_secret_iam_member" "billing_metrics_client_cert" { - count = local.billing_metrics_client_cert_enabled ? 1 : 0 +resource "google_secret_manager_secret_iam_member" "billing_metrics_client_secret" { + count = local.billing_metrics_client_secret_enabled ? 1 : 0 - secret_id = google_secret_manager_secret.billing_metrics_client_cert[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${google_service_account.runtime.email}" -} - -resource "google_secret_manager_secret_iam_member" "billing_metrics_client_key" { - count = local.billing_metrics_client_key_enabled ? 1 : 0 - - secret_id = google_secret_manager_secret.billing_metrics_client_key[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${google_service_account.runtime.email}" -} - -resource "google_secret_manager_secret_iam_member" "billing_metrics_ca_cert" { - count = local.billing_metrics_ca_cert_enabled ? 1 : 0 - - secret_id = google_secret_manager_secret.billing_metrics_ca_cert[0].id + secret_id = google_secret_manager_secret.billing_metrics_client_secret[0].id role = "roles/secretmanager.secretAccessor" member = "serviceAccount:${google_service_account.runtime.email}" } diff --git a/terraform/litellm/gcp/secrets.tf b/terraform/litellm/gcp/secrets.tf index b3b656721ab..f3636b6e8e8 100644 --- a/terraform/litellm/gcp/secrets.tf +++ b/terraform/litellm/gcp/secrets.tf @@ -64,59 +64,21 @@ resource "google_secret_manager_secret_version" "ui_password" { secret_data = var.ui_password } -# Billing-metrics mTLS material — only created when metering is enabled -# (billing_metrics_endpoint non-empty) and the operator supplied the PEM. -# The runtime SA gets accessor permission via iam.tf, and gateway + backend -# pick the env vars up through billing_metrics_env_secrets in cloudrun.tf. -resource "google_secret_manager_secret" "billing_metrics_client_cert" { - count = local.billing_metrics_client_cert_enabled ? 1 : 0 +resource "google_secret_manager_secret" "billing_metrics_client_secret" { + count = local.billing_metrics_client_secret_enabled ? 1 : 0 - secret_id = "${local.name}-billing-metrics-client-cert" + secret_id = "${local.name}-billing-metrics-client-secret" labels = local.labels replication { auto {} } } -resource "google_secret_manager_secret_version" "billing_metrics_client_cert" { - count = local.billing_metrics_client_cert_enabled ? 1 : 0 +resource "google_secret_manager_secret_version" "billing_metrics_client_secret" { + count = local.billing_metrics_client_secret_enabled ? 1 : 0 - secret = google_secret_manager_secret.billing_metrics_client_cert[0].id - secret_data = var.billing_metrics_client_cert_pem -} - -resource "google_secret_manager_secret" "billing_metrics_client_key" { - count = local.billing_metrics_client_key_enabled ? 1 : 0 - - secret_id = "${local.name}-billing-metrics-client-key" - labels = local.labels - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "billing_metrics_client_key" { - count = local.billing_metrics_client_key_enabled ? 1 : 0 - - secret = google_secret_manager_secret.billing_metrics_client_key[0].id - secret_data = var.billing_metrics_client_key_pem -} - -resource "google_secret_manager_secret" "billing_metrics_ca_cert" { - count = local.billing_metrics_ca_cert_enabled ? 1 : 0 - - secret_id = "${local.name}-billing-metrics-ca-cert" - labels = local.labels - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "billing_metrics_ca_cert" { - count = local.billing_metrics_ca_cert_enabled ? 1 : 0 - - secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id - secret_data = var.billing_metrics_ca_cert_pem + secret = google_secret_manager_secret.billing_metrics_client_secret[0].id + secret_data = var.billing_metrics_client_secret } resource "google_secret_manager_secret" "metrics_run_monitoring" { diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 6161e893e02..e40ee36fa18 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -602,14 +602,7 @@ variable "gateway_metrics_collector_image" { # ---------- Enterprise billing metrics ---------- # -# License-gated request metering. Opt-in and gated entirely on -# billing_metrics_endpoint: leave it empty (the default) and nothing -# metering-related is added to the container env. Set it and gateway + -# backend export billable-request counts over OTLP/HTTP, authenticating to -# the collector with an mTLS client cert. The proxy accepts the cert, key, -# and CA as either a file path or literal PEM content, so on Cloud Run they -# are injected straight from Secret Manager as env vars and no volume is -# needed. +# License-gated request metering. The client secret is stored in Secret Manager. variable "billing_metrics_endpoint" { description = <<-EOT @@ -623,44 +616,34 @@ variable "billing_metrics_endpoint" { default = "" } -variable "billing_metrics_client_cert_pem" { +variable "billing_metrics_client_id" { description = <<-EOT - PEM content of the mTLS client certificate issued for this deployment. - When billing_metrics_endpoint is set, the stack stores this in a - `-litellm--billing-metrics-client-cert` Secret Manager - entry, grants the runtime SA accessor on it, and exposes it to gateway + - backend as LITELLM_BILLING_METRICS_CLIENT_CERT. Required whenever - metering is enabled. + Cognito OAuth2 client ID exposed to the gateway and backend as + LITELLM_BILLING_METRICS_CLIENT_ID. Required when billing_metrics_endpoint + is set. + EOT + type = string + default = "" +} + +variable "billing_metrics_client_secret" { + description = <<-EOT + Cognito OAuth2 client secret stored in Secret Manager and exposed to + the gateway and backend as LITELLM_BILLING_METRICS_CLIENT_SECRET. + Required when billing_metrics_endpoint is set. EOT type = string default = "" sensitive = true } -variable "billing_metrics_client_key_pem" { +variable "billing_metrics_token_url" { description = <<-EOT - PEM content of the private key matching - billing_metrics_client_cert_pem. Stored in a - `-litellm--billing-metrics-client-key` Secret Manager entry - and exposed as LITELLM_BILLING_METRICS_CLIENT_KEY. Required whenever - metering is enabled. + Cognito OAuth2 client-credentials token endpoint exposed to gateway and + backend as LITELLM_BILLING_METRICS_TOKEN_URL. EOT type = string - default = "" - sensitive = true -} - -variable "billing_metrics_ca_cert_pem" { - description = <<-EOT - PEM content of the CA bundle used to verify the metering collector. - Only needed for private or test collectors whose CA is not in the - system trust store; telemetry.litellm.ai is publicly trusted, so leave - this empty for production. When set, it is exposed as - LITELLM_BILLING_METRICS_CA_CERT. - EOT - type = string - default = "" - sensitive = true + default = "https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token" } # ---------- Collector sidecar ---------- diff --git a/tests/_support/mcp_count_tool_upstream.py b/tests/_support/mcp_count_tool_upstream.py new file mode 100644 index 00000000000..f1aa61c7dba --- /dev/null +++ b/tests/_support/mcp_count_tool_upstream.py @@ -0,0 +1,42 @@ +import json +import queue +from collections.abc import Callable, Mapping +from typing import Final + +import httpx2 + +COUNT_OUTPUT_SCHEMA: Final = {"type": "object", "properties": {"n": {"type": "integer"}}, "required": ["n"]} + + +def count_tool_upstream( + structured: Mapping[str, object], methods_seen: "queue.SimpleQueue[str]" +) -> Callable[[httpx2.Request], httpx2.Response]: + """An MCP upstream with one tool, `count`, whose declared output schema wants an integer `n`.""" + + def respond(request: httpx2.Request) -> httpx2.Response: + if request.method != "POST": + return httpx2.Response(200) + payload: Final = json.loads(request.content) + methods_seen.put(payload["method"]) + if "id" not in payload: + return httpx2.Response(202) + results: Final[Mapping[str, Mapping[str, object]]] = { + "initialize": { + "protocolVersion": payload["params"].get("protocolVersion"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": "count", "version": "1"}, + }, + "tools/list": { + "tools": [{"name": "count", "inputSchema": {"type": "object"}, "outputSchema": COUNT_OUTPUT_SCHEMA}] + }, + "tools/call": {"content": [{"type": "text", "text": "ok"}], "structuredContent": structured}, + } + return httpx2.Response( + 200, json={"jsonrpc": "2.0", "id": payload["id"], "result": results.get(payload["method"], {})} + ) + + return respond + + +def methods_received(methods_seen: "queue.SimpleQueue[str]") -> tuple[str, ...]: + return tuple(methods_seen.get_nowait() for _ in range(methods_seen.qsize())) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a84a40c67c1..f0275c34c06 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -87,6 +87,9 @@ IGNORE_FUNCTIONS = [ "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. "_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update. + "_json", # router_rust index views: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap. + "to_wire", # router_rust model info: bounded by the nesting depth of a stored deployment or cost-map entry (finite JSON trees: pydantic refuses a cycle when it dumps a deployment). + "from_wire", # router_rust model info: walks only what router-core merged from to_wire's output, so it never goes deeper than to_wire did. ] diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 1b1ee484afc..42ee6178bf0 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -138,6 +138,7 @@ ignored_function_names = [ "_resolve_pre_call_check_input_tokens", # Tested through the token-count failure pre-call check tests in test_router.py "_params_neither_dropped_nor_allowed", # Tested through the allowed-or-dropped param pre-call check tests in test_router.py "_deployment_rejects_params", # Tested through the unsupported-param pre-call check tests in test_router.py + "_cache_flags_left_to_the_catalog", # Tested through the wildcard prompt cache flag tests in test_anthropic_cache_control_hook.py "_deployment_lacks_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_pre_call_check_requested_chat_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_deployment_supports_param", # Tested through the deployment-capability pre-call check tests in test_router.py @@ -175,6 +176,15 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "job_cost_of_poll", # Read by job_to_charge_once and charged_video_id, tested through the guardrail-priced poll tests in test_batch_utils.py + "charged_video_id", # Read by job_to_charge_once and get_spend_logs_id, tested through both in test_batch_utils.py and test_spend_tracking_utils.py + "_requested_video_id", # Read by charged_video_id, tested through the polled-id video test in test_batch_utils.py + "_guardrail_cost_of_poll", # Read by job_cost_of_poll and repeat_poll_cost, tested through the guardrail-priced poll tests + "repeat_poll_cost", # Read by job_to_charge_once and _charge_request_cost, tested through every limiter's guardrail poll test + "pays_only_its_guardrail", # Read by get_spend_logs_id, tested through its guardrail-only poll test in test_spend_tracking_utils.py + "charged_poll_cost", # Read by get_spend_logs_id, tested through the finished job guardrail poll tests in test_db_spend_update_writer.py + "_polled_batch", # Read by defers_batch_cost and _charged_batch, tested through the redacted batch retrieve tests + "_charged_batch", # Read by job_to_charge_once, tested through the batch charge identity tests in test_batch_utils.py "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py diff --git a/tests/code_coverage_tests/router_litellm_globals_check.py b/tests/code_coverage_tests/router_litellm_globals_check.py index ff9fac0e3dc..172a3754000 100644 --- a/tests/code_coverage_tests/router_litellm_globals_check.py +++ b/tests/code_coverage_tests/router_litellm_globals_check.py @@ -21,6 +21,7 @@ CHECKED = ( "litellm/router_core.py", "litellm/router_deprecated.py", "litellm/router_python_core.py", + "litellm/router_rust/core.py", ) diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py index a8e0d6fa0d7..1c41526101a 100644 --- a/tests/code_coverage_tests/router_surface_check.py +++ b/tests/code_coverage_tests/router_surface_check.py @@ -11,7 +11,8 @@ modules listed there. The lists hold what is used today and may only shrink: also fails, so the entry is removed in the same change. "Outside" means every non-test file under litellm/ and enterprise/ except -litellm/router.py and the router's own modules. Uses from the router family +litellm/router.py and the router's own modules (including every +module under litellm/router_rust/, the Rust core's host-side glue). Uses from the router family (litellm/router_utils, litellm/router_strategy) are listed separately, because those modules become private parts of the router core in a later step. @@ -42,9 +43,12 @@ ROUTER_FILES = { "litellm/router_core.py", "litellm/router_python_core.py", "litellm/router_python_host.py", + "litellm/router_rust/core.py", "litellm/router_deprecated.py", "litellm/router_host.py", } +# The Rust core's host-side glue modules are part of the router. +ROUTER_FILE_PREFIXES = ("litellm/router_rust/",) ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"] ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"} @@ -128,7 +132,7 @@ def scan() -> Dict[str, Dict[str, Set[str]]]: uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}} imports: Dict[str, Set[str]] = {} for rel, path in _python_files(): - if rel in ROUTER_FILES or _is_test(rel): + if rel in ROUTER_FILES or rel.startswith(ROUTER_FILE_PREFIXES) or _is_test(rel): continue with open(path, encoding="utf-8") as f: src = f.read() diff --git a/tests/code_coverage_tests/test_provider_cache.py b/tests/code_coverage_tests/test_provider_cache.py index fc39f2c1e6c..81ad7913559 100644 --- a/tests/code_coverage_tests/test_provider_cache.py +++ b/tests/code_coverage_tests/test_provider_cache.py @@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi assert len(provider.hits) == (2 if truncated else 1) -def test_store_outage_preserves_provider_success(provider: Provider) -> None: +@pytest.mark.parametrize("cluster_mode", ("0", "1")) +def test_store_outage_preserves_provider_success( + provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str, +) -> None: + monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode) with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) port: Final = probe.getsockname()[1] @@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None assert isinstance(hit, CacheHit) and hit.payload == b"new" +def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None: + other: Final = replace(store, namespace=store.namespace + "-other") + first_lease: Final = store.lookup("shared-key") + assert isinstance(first_lease, CaptureLease) + assert store.publish("shared-key", first_lease, b"first namespace") + other_lease: Final = other.lookup("shared-key") + assert isinstance(other_lease, CaptureLease) + assert other.publish("shared-key", other_lease, b"other namespace") + first_hit: Final = store.lookup("shared-key") + other_hit: Final = other.lookup("shared-key") + assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace" + assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace" + + def test_identity_preserves_values_and_never_contains_credentials() -> None: variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'') url: Final = "https://example.invalid/v1/chat/completions" diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 7843c1a4091..b4ca8bd3d50 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -32,25 +32,25 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup -- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown +- `llm_translation/` - Azure realtime translation is explicitly skipped pending investigation of intermittent missing transcripts and remains unqualified. OpenAI translation and other Azure realtime cases remain enabled. LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation - `embeddings/` - the `/embeddings` endpoint across providers -- `batches/` - the `/batches` endpoint (placeholder until the first test lands) +- `batches/` - the `/batches` endpoint. GovCloud batch coverage is explicitly skipped until its infrastructure is provisioned. Other missing credentials and provider errors remain failures - `realtime/` - realtime websocket sessions, including the pipecat audio path - `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`) - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface, not Playwright) - `a2a/` - the A2A (agent-to-agent) surface: admin registration via `/v1/agents`, proxy-fronted card discovery at `/.well-known/agent-card.json`, and JSON-RPC `message/send` invocation, driving agents backed by the litellm completion bridge (a real provider) and asserting protocol-version normalization (0.3 vs 1.0) -- `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below) +- `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below). The semantic tool filter test turns on a proxy-wide setting, so it is marked `proxy_wide_config` and runs on the redeployed managed-files stack after the main run, never next to it - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) -- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, run by hand since its weekly workflow never had working Bedrock credentials), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares, or a `Ramp` that steps the rate up through the run, built with `ramped()`, whose plan carries each request's send second and whose run reports `sustained_rps`, the last step the pod held, scored in `perf/setup/ramp.py`, with every step's latency, completed rate, non-200s, CPU per request, Redis latency and Postgres connections cut from the samplers' time-stamped `Series` (`perf/setup/series.py`) to the step's window and emitted as `perf_ramp_step__` JUnit properties for the dashboard) and `Limit`s (one metric at one percentile under one ceiling, or over one floor for `sustained_rps`). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` +- `router/` - routing and reliability behavior (fallbacks, cooldowns) +- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, run by hand since its weekly workflow never had working Bedrock credentials), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The memory tests (`test_reliability_memory_e2e.py`, run by the perf lane on its own fresh stack: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches, the check for the v1.100.0 retry-breadcrumb leak). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares, or a `Ramp` that steps the rate up through the run, built with `ramped()`, whose plan carries each request's send second and whose run reports `sustained_rps`, the last step the pod held, scored in `perf/setup/ramp.py`, with every step's latency, completed rate, non-200s, CPU per request, Redis latency and Postgres connections cut from the samplers' time-stamped `Series` (`perf/setup/series.py`) to the step's window and emitted as `perf_ramp_step__` JUnit properties for the dashboard) and `Limit`s (one metric at one percentile under one ceiling, or over one floor for `sustained_rps`). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. The Bedrock Runtime setup is explicitly skipped until stage IAM permits `bedrock:InvokeModel` on its default project. The other setups still run, and skipped Runtime cases do not qualify that route. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only @@ -100,7 +100,7 @@ Each suite provides its own `client` fixture (see `llm_translation/passthrough_c Request and response bodies are typed pydantic models in `models.py`. Only the fields a test reads are modelled, and nothing passes raw dicts. Outcomes come back as a `Result[R]` tagged union (`Success`, `NetworkError`, `UnauthorizedError`, `RateLimitedError`, `ValidationError`, `UnknownApiError`). Handle them with `match`, or call `unwrap(...)` when a non-success should fail the test. The harness hard-fails and never skips: a test marked `e2e` fails when no proxy answers its liveness probe, and once a request reaches the proxy any wrong behavior is likewise a hard failure, so a missing proxy turns the run red instead of being mistaken for a pass -Mark live tests with `@pytest.mark.e2e` (on the class or the module). Coverage of the harness itself lives outside the suite in `tests/e2e_harness/` (see its `AGENTS.md`) and runs without a proxy, so nothing under `tests/e2e/` is a markerless test. Add `@pytest.mark.quiet_stack` to a test that measures the proxy itself (RSS, latency): the shared stack lock in `stack_lock.py` then runs it while no other test on the host is hitting the stack, marked or not, so the reading depends only on the test's own traffic. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +Mark live tests with `@pytest.mark.e2e` (on the class or the module). Coverage of the harness itself lives outside the suite in `tests/e2e_harness/` (see its `AGENTS.md`) and runs without a proxy, so nothing under `tests/e2e/` is a markerless test. A test that measures the proxy itself (RSS, latency) cannot share the stack with the rest of the suite, so it goes in `perf/`, marked `perf`. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache ## Record and replay fixtures @@ -329,3 +329,7 @@ other... - the `codex/` suite is the one exception: a single setup that a LiteLLM or provider bug breaks may carry `pytest.mark.xfail(strict=True, reason=...)` with a plain one-sentence reason on that setup's `pytest.param`, so it keeps running and goes red once the bug is fixed. Strict only, and never on a flaky setup (one that passes some runs and fails others) - a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod + +## Provider response cache + +The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 1ede9c89b97..ac6f9cd08e4 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -11,8 +11,9 @@ cost write-back via a cross-run marker baton (design below). Only supported cells are tested. The capability table in `capabilities.py` holds one row per supported (provider, scenario) pair, so there are no skipped cells in the -parametrized run. The batches suite never skips: missing provider creds or upstream -failures are hard test failures (see `tests/e2e/AGENTS.md`). +parametrized run. Missing provider creds or upstream failures are hard test failures +(see `tests/e2e/AGENTS.md`). GovCloud is explicitly skipped until its infrastructure +is provisioned and does not count as qualified coverage. | Provider | create | retrieve | cancel | list | content download | file backing | |-----------|--------|----------|--------|------|------------------|--------------| diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index b7a52da5902..eb83db73199 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1210,6 +1210,7 @@ def _govcloud_params() -> LiteLLMParamsBody: ) +@pytest.mark.skip(reason="GovCloud E2E infrastructure is not provisioned") class TestBedrockBatchGovCloud: """Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1). diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py index b2442d492cf..14201ef6846 100644 --- a/tests/e2e/claude_code/conftest.py +++ b/tests/e2e/claude_code/conftest.py @@ -597,14 +597,21 @@ def _build_control_plane_client(proxy_config: ProxyConfig): ) -def _register_deployment(proxy, deployment: CompatDeployment) -> str: - """Register one deployment and return its proxy-assigned model_id - once it is servable on the data plane. The aliases are shared by every - cell and, under xdist, by every worker, so no call to them belongs to - one test and none is cached: the matrix exists to reach real providers.""" - return proxy.create_model( - deployment.model_name, - deployment.litellm_params, +def _register_deployments(proxy, deployments: tuple[CompatDeployment, ...]): + """The aliases are shared by every cell and, under xdist, by every worker, + so no call to them belongs to one test and none is cached: the matrix + exists to reach real providers.""" + from models import ModelInfoBody, ModelNewBody + + return proxy.register_models( + tuple( + ModelNewBody( + model_name=deployment.model_name, + litellm_params=deployment.litellm_params, + model_info=ModelInfoBody(), + ) + for deployment in deployments + ), provider_live=True, ) @@ -632,18 +639,14 @@ def _compat_models_registered() -> Any: yield return + from proxy_client import ModelNotRegistered, ModelRegistered from requests import RequestException proxy = _build_control_plane_client(proxy_config) - registered_ids: list[str] = [] - failures: list[tuple[str, str]] = [] + registrations = _register_deployments(proxy, load_all_deployments()) + registered_ids = tuple(r.model_id for r in registrations if isinstance(r, ModelRegistered)) + failures = tuple((r.model_name, r.reason) for r in registrations if isinstance(r, ModelNotRegistered)) try: - for deployment in load_all_deployments(): - try: - model_id = _register_deployment(proxy, deployment) - registered_ids.append(model_id) - except (AssertionError, RequestException) as exc: - failures.append((deployment.model_name, str(exc))) if failures: summary = "\n".join( f" - {name}: {reason}" for name, reason in failures diff --git a/tests/e2e/codex/setups.py b/tests/e2e/codex/setups.py index 4716706c51c..f86397686f3 100644 --- a/tests/e2e/codex/setups.py +++ b/tests/e2e/codex/setups.py @@ -13,7 +13,7 @@ from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from models import LiteLLMParamsBody GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol" -GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol" +GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/global.openai.gpt-6.1-sol" GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol" GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol" CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5" @@ -60,7 +60,10 @@ SETUPS: Final = ( RateBucket.BEDROCK_CONVERSE, ), id="bedrock_runtime", - marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME), + marks=( + _drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME), + pytest.mark.skip(reason="Stage IAM lacks bedrock:InvokeModel on the default Bedrock Runtime project"), + ), ), pytest.param( Deployment( diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 4c7c57ca18f..8fb96bef2f9 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -35,6 +35,7 @@ from e2e_config import ( PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, + PROXY_WIDE_CONFIG_OPT_IN_ENV, SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, SAGEMAKER_NOVA_OPT_IN_ENV, SECRET_MANAGER_OPT_IN_ENV, @@ -54,7 +55,6 @@ from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client -from stack_lock import stack_lock _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() @@ -77,6 +77,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "secret_manager": SECRET_MANAGER_OPT_IN_ENV, "sagemaker_nova": SAGEMAKER_NOVA_OPT_IN_ENV, "sagemaker_nova2_lite": SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, + "proxy_wide_config": PROXY_WIDE_CONFIG_OPT_IN_ENV, } ) @@ -162,11 +163,6 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_PERF_TESTS is set", ) - config.addinivalue_line( - "markers", - "quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; " - "every other test waits for it to finish", - ) config.addinivalue_line( "markers", "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " @@ -200,6 +196,11 @@ def pytest_configure(config: pytest.Config) -> None: "sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; " "deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it", ) + config.addinivalue_line( + "markers", + "proxy_wide_config: turns on a proxy-wide setting through /config/update, so it runs on a stack no other suite " + "is driving; deselected unless E2E_PROXY_WIDE_CONFIG_STACK is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: @@ -269,7 +270,7 @@ def pytest_collection_finish(session: pytest.Session) -> None: ahead of xdist's own collection-finish report, and the controller schedules no test until every worker has reported, so this is the idle footprint of a stack that just passed its readiness gate. The fixture hands the capture to the - idle-budget test in router/test_reliability_memory_e2e.py.""" + idle-budget test in perf/test_reliability_memory_e2e.py.""" if session.config.getoption("collectonly") or not any(_uses_idle_rss(item) for item in session.items): return session.config.stash[_IDLE_RSS] = read_rss_everywhere(build_proxy_client(), timeout=IDLE_RSS_READ_TIMEOUT_SECONDS) @@ -299,12 +300,6 @@ def _proxy_fail_reason() -> str | None: return None -@pytest.hookimpl(wrapper=True) -def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None) -> Generator[None, object, object]: - with stack_lock(exclusive=item.get_closest_marker("quiet_stack") is not None): - return (yield) - - @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 854958fc39e..457f03406f1 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -20,8 +20,8 @@ - {id: guardrail.ibm_guardrails.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Output policy validation"} - {id: guardrail.semantic_guard.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/semantic_guard", rationale: "Semantic policy compliance"} - {id: guardrail.block_code_execution.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/block_code_execution", rationale: "Code-injection prevention"} -- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"} -- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"} +- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"} +- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"} - {id: guardrail.microsoft_purview.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/microsoft_purview/purview_dlp.py", rationale: "DLP sensitive-data disclosure"} - {id: guardrail.headroom.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/headroom/headroom.py", rationale: "Anomaly detection threshold"} - {id: guardrail.generic_guardrail_api.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py", rationale: "Vendor-agnostic custom API"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 367135eae26..d99bcff46e0 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -77,6 +77,10 @@ - {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"} - {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"} - {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"} +- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"} +- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"} +- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"} +- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} @@ -94,6 +98,8 @@ - {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} - {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} - {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"} +- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"} +- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"} - {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} - {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} - {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index cbaf46130d6..49997aef162 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -44,8 +44,6 @@ - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} - {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} -- {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"} -- {id: quota_management.spend_tracking.margin_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: margin_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider margin is added to the real request's spend row"} - {id: quota_management.spend_tracking.guardrail_cost.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: guardrail_cost, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/guardrail_cost.py + proxy/spend_tracking/spend_tracking_utils.py", rationale: "A priced Bedrock guardrail charge reaches the persisted spend row"} - {id: quota_management.spend_tracking.key_rollup.matches_sum_of_logs, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_rollup, assertions: [matches_sum_of_logs], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "A key's rolled-up spend equals the sum of its log rows"} - {id: quota_management.spend_tracking.concurrent_burst.loses_no_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: concurrent_burst, assertions: [loses_no_spend], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "Concurrent calls all land as spend; no row lost to write contention"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d2ba2c325df..7d9bc380bd0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -93,6 +93,8 @@ LlmCapability = Literal[ "tool_search_history", "tool_use", "upstream_stream_failure", + "transcription", + "translation", "vision", "web_search", "web_search_server_tool", @@ -162,14 +164,7 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, + LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell, Field(discriminator="module"), ] diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 724aa2362e7..6316d22d3fa 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -210,6 +210,7 @@ OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER" SAGEMAKER_NOVA_OPT_IN_ENV: Final = "SAGEMAKER_NOVA_ENDPOINT" SAGEMAKER_NOVA2_LITE_OPT_IN_ENV: Final = "SAGEMAKER_NOVA2_LITE_ENDPOINT" +PROXY_WIDE_CONFIG_OPT_IN_ENV: Final = "E2E_PROXY_WIDE_CONFIG_STACK" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) @@ -234,7 +235,6 @@ MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) MEMORY_RSS_CEILING_MB = float(os.environ.get("E2E_MEMORY_RSS_CEILING_MB", "200")) MEMORY_IDLE_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_IDLE_RSS_BUDGET_MB", "768")) -MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) def ws_base_url() -> str: diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index cf517c7a4fe..9db95822734 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -38,7 +38,7 @@ from models import ( from proxy_client import ProxyClient from pydantic import BaseModel, Field -GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash" +GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash" GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"] BlockedWordAction = Literal["BLOCK", "MASK"] diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py index b9593feb2ed..0eb1bf79daa 100644 --- a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py @@ -128,7 +128,7 @@ class TestOpenAIModerationGuardrail: ) allowed = unwrap( - client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=64) + client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=512) ) assert allowed.content or allowed.choices, ( "the same moderation guardrail must let a benign /v1/messages prompt through, but " diff --git a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py index d5a6a9d9866..87545299169 100644 --- a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py @@ -19,10 +19,9 @@ verdict comes from the proxy itself. from __future__ import annotations -from typing import Final +from typing import Final, Literal import pytest - from e2e_config import unique_marker from e2e_http import StreamingResponse, UnknownApiError from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta @@ -31,14 +30,29 @@ from guardrails_client import ( ToolPermissionParamsBody, ToolPermissionRuleBody, poll_until_blocked, + poll_until_blocked_stream, poll_until_guardrail_applied, ) from lifecycle import ResourceManager -from models import ChatResponse, ChatTool, ChatToolFunction +from models import ( + AnthropicCustomTool, + AnthropicMessagesBody, + AnthropicMessagesResponse, + AnthropicToolChoice, + ChatMessage, + ChatResponse, + ChatTool, + ChatToolFunction, + JsonSchemaProperty, + ResponsesFunctionTool, + ResponsesToolBody, + ResponsesToolOutput, + ToolInputSchema, +) -pytestmark = pytest.mark.e2e +pytestmark: Final = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL: Final = "gemini-2.5-flash" #: The one tool the guardrail permits, and one it does not. Both are declared by #: the caller in the request body; the guardrail reads them there. @@ -63,11 +77,15 @@ DENIED_TOOL: Final = ChatTool( TOOL_PROMPT: Final = "What is the weather in Paris right now?" +type FlatToolEndpoint = Literal["messages", "responses"] +ALLOWED_TOOL_SCHEMA: Final = ToolInputSchema(properties={"city": JsonSchemaProperty(type="string")}, required=["city"]) +DENIED_TOOL_SCHEMA: Final = ToolInputSchema() + def _register_tool_permission(client: GuardrailsClient, resources: ResourceManager, *, name: str) -> None: """Allow-list exactly one tool: everything else falls to `default_action=deny` and, with `on_disallowed_action=block`, is rejected outright.""" - guardrail_id = client.register( + guardrail_id: Final = client.register( name, ToolPermissionParamsBody( mode="pre_call", @@ -100,6 +118,57 @@ def _tool_call_names(response: ChatResponse) -> tuple[str, ...]: ) +def _send_flat_tool( + client: GuardrailsClient, + key: str, + endpoint: FlatToolEndpoint, + *, + guardrail: str, + tool: ChatTool, + schema: ToolInputSchema, + require_tool: bool = False, +) -> StreamingResponse: + name: Final = tool.function.name + match endpoint: + case "messages": + return client.proxy.transport.send( + "/v1/messages", + headers=client.proxy.transport.bearer(key), + json=AnthropicMessagesBody( + model=MODEL, + messages=[ChatMessage(role="user", content=TOOL_PROMPT)], + max_tokens=128, + tools=[ + AnthropicCustomTool(name=name, description=tool.function.description or "", input_schema=schema) + ], + tool_choice=AnthropicToolChoice(type="any") if require_tool else None, + guardrails=[guardrail], + ), + ) + case "responses": + return client.proxy.transport.send( + "/v1/responses", + headers=client.proxy.transport.bearer(key), + json=ResponsesToolBody( + model=MODEL, + input=TOOL_PROMPT, + tools=[ResponsesFunctionTool(name=name, description=tool.function.description, parameters=schema)], + tool_choice="required" if require_tool else None, + guardrails=[guardrail], + ), + ) + + +def _flat_tool_call_names(endpoint: FlatToolEndpoint, body: str) -> tuple[str, ...]: + match endpoint: + case "messages": + content: Final = AnthropicMessagesResponse.model_validate_json(body).content or [] + return tuple(block.name for block in content if block.type == "tool_use" and block.name) + case "responses": + output: Final = ResponsesToolOutput.model_validate_json(body).output + return tuple(item.name for item in output if item.type == "function_call" and item.name) + + class TestToolPermissionPreCall: @pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["chat_completions"]) @meta( @@ -118,10 +187,10 @@ class TestToolPermissionPreCall: rejected with a 400 that names the denied tool. An unauthorized tool that merely reaches the model is the whole failure mode this guardrail exists to prevent, so a 200 here is a hard failure.""" - name = f"e2e-toolperm-block-{unique_marker()}" + name: Final = f"e2e-toolperm-block-{unique_marker()}" _register_tool_permission(client, resources, name=name) - result = poll_until_blocked( + result: Final = poll_until_blocked( lambda: client.chat( scoped_key, MODEL, @@ -161,10 +230,10 @@ class TestToolPermissionPreCall: and the model calls it. Without the header check a guardrail that never attached would pass this test for the wrong reason, so the 200 alone is not the contract.""" - name = f"e2e-toolperm-allow-{unique_marker()}" + name: Final = f"e2e-toolperm-allow-{unique_marker()}" _register_tool_permission(client, resources, name=name) - outcome = poll_until_guardrail_applied( + outcome: Final = poll_until_guardrail_applied( lambda: client.chat_raw( scoped_key, MODEL, @@ -178,13 +247,117 @@ class TestToolPermissionPreCall: ) assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}" - applied = _applied_guardrails(outcome) + applied: Final = _applied_guardrails(outcome) assert name in applied, ( "the allowed call must carry x-litellm-applied-guardrails naming the guardrail; " f"without it the 200 only proves the guardrail never ran. Got {applied!r}" ) - called = _tool_call_names(ChatResponse.model_validate_json(outcome.body)) + called: Final = _tool_call_names(ChatResponse.model_validate_json(outcome.body)) + assert called == (ALLOWED_TOOL.function.name,), ( + f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}" + ) + + @pytest.mark.parametrize( + "endpoint", + [ + pytest.param( + "messages", + marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["messages"]), + ), + pytest.param( + "responses", + marks=pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["responses"]), + ), + ], + ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) + def test_pre_call_blocks_flat_tool_outside_the_allow_list( + self, + client: GuardrailsClient, + resources: ResourceManager, + scoped_key: str, + endpoint: FlatToolEndpoint, + ) -> None: + name: Final = f"e2e-toolperm-block-{endpoint}-{unique_marker()}" + _register_tool_permission(client, resources, name=name) + + outcome: Final = poll_until_blocked_stream( + lambda: _send_flat_tool( + client, scoped_key, endpoint, guardrail=name, tool=DENIED_TOOL, schema=DENIED_TOOL_SCHEMA + ) + ) + + assert outcome.status_code == 400, ( + f"tool_permission let a tool outside the allow-list through /{endpoint}; " + f"got {outcome.status_code}: {outcome.body[:400]}" + ) + assert DENIED_TOOL.function.name in outcome.body, ( + f"the block must name the denied tool so the caller can fix the request; got: {outcome.body[:400]}" + ) + assert "Violated guardrail policy" in outcome.body, ( + f"the block must be a guardrail verdict, got: {outcome.body[:400]}" + ) + + @pytest.mark.parametrize( + "endpoint", + [ + pytest.param( + "messages", + marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["messages"]), + ), + pytest.param( + "responses", + marks=pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["responses"]), + ), + ], + ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) + def test_pre_call_allows_permitted_flat_tool( + self, + client: GuardrailsClient, + resources: ResourceManager, + scoped_key: str, + endpoint: FlatToolEndpoint, + ) -> None: + name: Final = f"e2e-toolperm-allow-{endpoint}-{unique_marker()}" + _register_tool_permission(client, resources, name=name) + + outcome: Final = poll_until_guardrail_applied( + lambda: _send_flat_tool( + client, + scoped_key, + endpoint, + guardrail=name, + tool=ALLOWED_TOOL, + schema=ALLOWED_TOOL_SCHEMA, + require_tool=True, + ), + name, + ) + + assert outcome.ok, f"the permitted tool must be served, got {outcome.status_code}: {outcome.body[:400]}" + applied: Final = _applied_guardrails(outcome) + assert name in applied, ( + f"the allowed call must carry x-litellm-applied-guardrails naming the guardrail; got {applied!r}" + ) + called: Final = _flat_tool_call_names(endpoint, outcome.body) assert called == (ALLOWED_TOOL.function.name,), ( f"the served call must carry one tool call for the permitted tool, got {called!r}: {outcome.body[:400]}" ) diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md index c85471da90d..1694024f390 100644 --- a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -19,6 +19,15 @@ a matching `function_call` output item, the test sends a `function_call_output` back, and the follow-up response incorporates the result (the temperature 72 appears). That raw-websocket tool path is the source of truth for tool calling. +The audio surfaces provision their own OpenAI and Azure deployments. +`TestRealtimeTranslation` streams the spoken weather question into +`/v1/realtime/translations` with French as the target and asserts translated +audio, a French transcript, and a billed spend row. `TestRealtimeLiveTranscription` +streams it into `/v1/realtime?intent=transcription` on gpt-live-transcribe and +asserts the deltas add up to the completed transcript and the session is billed. +`test_azure_ga_text_turn_is_answered_and_billed` guards a plain text turn over +the Azure GA deployment, spend row included + `test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure, and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are @@ -48,8 +57,9 @@ Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy b kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by uncommenting their entry. -Every provider is provisioned and asserted; the suite never skips a provider. Per -`tests/e2e/AGENTS.md` there is no sanctioned skip: the whole-suite proxy-liveness +Azure realtime translation is explicitly skipped pending investigation of intermittent +missing transcripts and does not count as qualified coverage. OpenAI translation and +other Azure realtime cases remain enabled. Per `tests/e2e/AGENTS.md`, the whole-suite proxy-liveness probe hard-fails when no proxy answers, and a provider whose credentials or upstream realtime model are missing on the gateway is likewise a hard failure, not a skip. Give the gateway each provider's credentials to turn its tests green. diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 6bd5d61d721..7e7f9f424a2 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -29,9 +29,13 @@ from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) +REALTIME_ROUTE: Final = "/v1/realtime" +TRANSLATIONS_ROUTE: Final = "/v1/realtime/translations" -def realtime_ws_url(model: str) -> str: - return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}" + +def realtime_ws_url(model: str, *, route: str = REALTIME_ROUTE, intent: str | None = None) -> str: + query: Final = {"model": model} if intent is None else {"model": model, "intent": intent} + return f"{ws_base_url()}{route}?{urlencode(query)}" @dataclass(frozen=True, slots=True) @@ -183,6 +187,75 @@ class InputAudioBufferAppend(BaseModel): audio: str +class InputAudioBufferCommit(BaseModel): + type: str = "input_audio_buffer.commit" + + +class SessionInputAudioBufferAppend(BaseModel): + type: str = "session.input_audio_buffer.append" + audio: str + + +AudioAppend = InputAudioBufferAppend | SessionInputAudioBufferAppend + + +class TranslationOutput(BaseModel): + language: str + + +class TranslationAudio(BaseModel): + output: TranslationOutput + + +class TranslationSessionConfig(BaseModel): + audio: TranslationAudio + + +class TranslationSessionUpdate(BaseModel): + type: str = "session.update" + session: TranslationSessionConfig + + +class PcmFormat(BaseModel): + type: str = "audio/pcm" + rate: int = 24000 + + +class InputTranscription(BaseModel): + model: str + + +class TranscriptionAudioInput(BaseModel): + format: PcmFormat = PcmFormat() + transcription: InputTranscription + + +class TranscriptionAudio(BaseModel): + input: TranscriptionAudioInput + + +class TranscriptionSessionConfig(BaseModel): + type: str = "transcription" + audio: TranscriptionAudio + + +class TranscriptionSessionUpdate(BaseModel): + type: str = "session.update" + session: TranscriptionSessionConfig + + +RealtimeClientEvent = ( + SessionUpdate + | TranslationSessionUpdate + | TranscriptionSessionUpdate + | ConversationItemCreate + | ResponseCreate + | InputAudioBufferAppend + | InputAudioBufferCommit + | SessionInputAudioBufferAppend +) + + def user_message(text: str) -> ConversationItemCreate: return ConversationItemCreate( item=MessageItem(content=[InputTextContent(text=text)]) @@ -320,7 +393,7 @@ class RealtimeSession: connection: Connection @step("Send the realtime event {event.type} over the websocket") - def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate | InputAudioBufferAppend) -> None: + def send(self, event: RealtimeClientEvent) -> None: self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True)) @step("Wait for a {stop_type} event on the realtime websocket") @@ -350,9 +423,9 @@ class RealtimeSession: @step("Stream the spoken question as mic chunks, then silence until the turn goes idle") def stream_and_collect( self, - chunks: Sequence[InputAudioBufferAppend], + chunks: Sequence[AudioAppend], *, - tail: InputAudioBufferAppend, + tail: AudioAppend, interval: float, idle: float, timeout: float, @@ -363,9 +436,9 @@ class RealtimeSession: def _live_mic_events( self, - chunks: Sequence[InputAudioBufferAppend], + chunks: Sequence[AudioAppend], *, - tail: InputAudioBufferAppend, + tail: AudioAppend, interval: float, idle: float, timeout: float, @@ -408,13 +481,19 @@ class RealtimeClient: ) return model_name, model_id - @step("Open a /v1/realtime websocket session to {model}") + @step("Open a {route} websocket session to {model}") @contextmanager def connect( - self, *, key: str, model: str, timeout: float = 15.0 + self, + *, + key: str, + model: str, + route: str = REALTIME_ROUTE, + intent: str | None = None, + timeout: float = 15.0, ) -> Generator[RealtimeSession, None, None]: with connect( - realtime_ws_url(model), + realtime_ws_url(model, route=route, intent=intent), additional_headers={"Authorization": f"Bearer {key}"}, open_timeout=timeout, ) as connection: diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index 3b7ba3a2430..7dfb8671202 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -4,7 +4,10 @@ Each test opens a websocket through the proxy, speaks the OpenAI GA realtime event schema, and asserts the proxy normalizes the provider's stream into that schema: the session lifecycle, the canonical response event sequence with a reconstructed transcript and usage, and a full tool-call round-trip (call -> -tool result -> a follow-up response that uses the result). +tool result -> a follow-up response that uses the result). The audio tests +stream the spoken weather question into /v1/realtime/translations and into a +/v1/realtime?intent=transcription session, and assert the translated or +transcribed output and the session's spend row. One GA-speaking client validates every provider; only the model alias changes. Every provider is provisioned at session start, so a missing realtime alias is a @@ -12,19 +15,28 @@ hard failure, not a skip; once configured, a protocol failure is likewise a hard failure. See REALTIME_COVERAGE_MATRIX.md. """ +import base64 +import wave +from pathlib import Path +from types import MappingProxyType from typing import Final import pytest from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import LiteLLMParamsBody, SpendLogRow from pydantic import BaseModel from realtime_client import ( PROVIDERS, + TRANSLATIONS_ROUTE, ConversationItemCreate, + DeltaEvent, FunctionCallArgumentsDone, FunctionCallOutputItem, FunctionTool, + InputAudioBufferAppend, + InputAudioBufferCommit, + InputTranscription, JsonSchema, JsonSchemaProperty, RealtimeClient, @@ -35,8 +47,18 @@ from realtime_client import ( ResponseDone, ServerEnvelope, SessionConfig, + SessionInputAudioBufferAppend, SessionUpdate, + TranscriptionAudio, + TranscriptionAudioInput, + TranscriptionSessionConfig, + TranscriptionSessionUpdate, + TranslationAudio, + TranslationOutput, + TranslationSessionConfig, + TranslationSessionUpdate, as_text, + events_of_type, parse_last, realtime_model, transcript, @@ -264,3 +286,215 @@ def test_upstream_handshake_refusal_is_an_error_event_and_policy_close( assert closed.value.rcvd is not None assert closed.value.rcvd.code == 1008, closed.value + + +WEATHER_WAV: Final = Path(__file__).parent / "fixtures" / "weather_question_24k.wav" +PCM_RATE: Final = 24000 +CHUNK_SECONDS: Final = 0.1 +CHUNK_BYTES: Final = int(PCM_RATE * CHUNK_SECONDS) * 2 +SILENCE_B64: Final = base64.b64encode(bytes(CHUNK_BYTES)).decode() +REGISTRY_ROUTES: Final = MappingProxyType({"openai": "openai", "azure": "azure_openai"}) + + +def weather_pcm_chunks() -> tuple[str, ...]: + with wave.open(str(WEATHER_WAV), "rb") as wav: + assert (wav.getnchannels(), wav.getsampwidth(), wav.getframerate()) == (1, 2, PCM_RATE), ( + "fixture must be mono PCM16 at 24 kHz" + ) + pcm: Final = wav.readframes(wav.getnframes()) + return tuple(base64.b64encode(pcm[i : i + CHUNK_BYTES]).decode() for i in range(0, len(pcm), CHUNK_BYTES)) + + +def has_billed_row(rows: list[SpendLogRow]) -> bool: + return any((row.spend or 0) > 0 for row in rows) + + +def audio_marks(provider: RealtimeProvider, capability: str) -> tuple[pytest.MarkDecorator, ...]: + return ( + pytest.mark.covers(f"llm.realtime.{REGISTRY_ROUTES[provider.id]}.{capability}.stream.works"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(provider.id),), + models=(provider.litellm_params.model,), + capabilities=(Capability.AUDIO_INPUT,), + mode=Mode.WEBSOCKET, + ) + ), + ) + + +TRANSLATION_PROVIDERS: Final = ( + RealtimeProvider( + "openai", + "openai-translate", + LiteLLMParamsBody(model="openai/gpt-realtime-translate", api_key="os.environ/OPENAI_API_KEY"), + ), + RealtimeProvider( + "azure", + "azure-translate", + LiteLLMParamsBody( + model="azure/gpt-realtime-translate", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + ), + ), +) + +LIVE_TRANSCRIPTION_PROVIDERS: Final = ( + RealtimeProvider( + "openai", + "openai-live-transcribe", + LiteLLMParamsBody(model="openai/gpt-live-transcribe", api_key="os.environ/OPENAI_API_KEY"), + ), + RealtimeProvider( + "azure", + "azure-live-transcribe", + LiteLLMParamsBody( + model="azure/gpt-live-transcribe", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + ), + ), +) + +TRANSLATION_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=( + *audio_marks(p, "translation"), + pytest.mark.skipif( + p.id == "azure", + reason="Azure realtime translation intermittently returns no transcript; qualification deferred", + ), + ), + ) + for p in TRANSLATION_PROVIDERS +) +LIVE_TRANSCRIPTION_PARAMS: Final = tuple( + pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS +) +AZURE_GA_PROVIDER: Final = next(p for p in PROVIDERS if p.id == "azure") + + +class TranscriptionCompleted(BaseModel): + type: str + transcript: str + + +class TestRealtimeTranslation: + @pytest.mark.parametrize("provider", TRANSLATION_PARAMS) + def test_spoken_english_comes_back_as_french_audio_and_transcript_and_is_billed( + self, + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, + ) -> None: + model, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model, route=TRANSLATIONS_ROUTE) as session: + session.send( + TranslationSessionUpdate( + session=TranslationSessionConfig(audio=TranslationAudio(output=TranslationOutput(language="fr"))) + ) + ) + session.collect_until("session.updated", timeout=30) + events: Final = session.stream_and_collect( + tuple(SessionInputAudioBufferAppend(audio=chunk) for chunk in weather_pcm_chunks()), + tail=SessionInputAudioBufferAppend(audio=SILENCE_B64), + interval=CHUNK_SECONDS, + idle=8, + timeout=60, + ) + + types: Final = tuple(e.type for e in events) + audio_bytes: Final = sum( + len(base64.b64decode(DeltaEvent.model_validate_json(e.payload).delta)) + for e in events_of_type(events, "session.output_audio.delta") + ) + assert audio_bytes > 0, f"translation returned no translated audio; types={types}" + translated: Final = "".join( + DeltaEvent.model_validate_json(e.payload).delta + for e in events_of_type(events, "session.output_transcript.delta") + ).lower() + assert translated.strip(), f"translation returned no translated text; types={types}" + assert "weather" not in translated, f"the spoken weather question came back untranslated: {translated!r}" + + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the translation session was not billed: {rows}" + + +class TestRealtimeLiveTranscription: + @pytest.mark.parametrize("provider", LIVE_TRANSCRIPTION_PARAMS) + def test_spoken_question_streams_transcript_deltas_that_match_the_completed_transcript( + self, + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, + ) -> None: + model, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model, intent="transcription") as session: + session.send( + TranscriptionSessionUpdate( + session=TranscriptionSessionConfig( + audio=TranscriptionAudio( + input=TranscriptionAudioInput(transcription=InputTranscription(model=model)) + ) + ) + ) + ) + session.collect_until("session.updated", timeout=30) + for chunk in weather_pcm_chunks(): + session.send(InputAudioBufferAppend(audio=chunk)) + session.send(InputAudioBufferCommit()) + events: Final = session.collect_until("conversation.item.input_audio_transcription.completed", timeout=60) + + completed: Final = TranscriptionCompleted.model_validate_json(events[-1].payload) + assert "weather" in completed.transcript.lower(), ( + f"transcript of a spoken weather question does not mention weather: {completed.transcript!r}" + ) + streamed: Final = "".join( + DeltaEvent.model_validate_json(e.payload).delta + for e in events_of_type(events, "conversation.item.input_audio_transcription.delta") + ) + assert streamed.strip() == completed.transcript.strip(), ( + f"streamed deltas {streamed!r} do not add up to the completed transcript {completed.transcript!r}" + ) + + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the live transcription session was not billed: {rows}" + + +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.AZURE,), + models=(AZURE_GA_PROVIDER.litellm_params.model,), + mode=Mode.WEBSOCKET, + ) +) +def test_azure_ga_text_turn_is_answered_and_billed( + client: RealtimeClient, resources: ResourceManager, scoped_key: str +) -> None: + model, model_id = client.provision(AZURE_GA_PROVIDER) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model) as session: + session.collect_until("session.created", timeout=20) + session.send(SessionUpdate(session=SessionConfig(instructions="Reply with the single word hello."))) + session.collect_until("session.updated", timeout=20) + session.send(user_message("Say the single word hello.")) + session.send(ResponseCreate()) + events: Final = session.collect_until("response.done", timeout=60) + + assert "hello" in transcript(events).lower(), f"Azure GA realtime did not answer the turn: {transcript(events)!r}" + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the Azure GA realtime turn was not billed: {rows}" diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index cd1ee8fb03c..43c276dfdf0 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -8,10 +8,13 @@ the shared multipart transport, since the SDK refuses to send them. A model-less request comes back as one of two 400s depending on whether any wildcard deployment happens to be registered on the shared proxy, so the assertion accepts either phrasing and holds both to naming the model as the problem. +Streamed gpt-transcribe (stream=true) on OpenAI and Azure must send transcript +deltas, a done event, and data: [DONE], and bill the call. """ from __future__ import annotations +from dataclasses import dataclass from pathlib import Path from typing import Final @@ -20,10 +23,10 @@ from e2e_config import unique_marker from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import UnknownApiError, unwrap from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import LiteLLMParamsBody, SpendLogRow from proxy_client import ProxyClient from pydantic import BaseModel, Field -from sdk_clients import SdkClients +from sdk_clients import SdkClients, response_header pytestmark = pytest.mark.e2e @@ -221,3 +224,102 @@ class TestWhisperTranscriptionFormats: assert all(word.start <= word.end for word in transcript.words), ( f"word timings out of order: {transcript.words}" ) + + +STREAM_DONE: Final = "[DONE]" + + +@dataclass(frozen=True, slots=True) +class _StreamingTranscriber: + id: str + registry_route: str + provider: Provider + litellm_params: LiteLLMParamsBody + + +STREAMING_TRANSCRIBERS: Final = ( + _StreamingTranscriber( + "openai", + "openai", + Provider.OPENAI, + LiteLLMParamsBody(model="openai/gpt-transcribe", api_key="os.environ/OPENAI_API_KEY"), + ), + _StreamingTranscriber( + "azure", + "azure_openai", + Provider.AZURE, + LiteLLMParamsBody( + model="azure/gpt-transcribe", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version="v1", + ), + ), +) + +STREAMING_PARAMS: Final = tuple( + pytest.param( + t, + id=t.id, + marks=( + pytest.mark.covers(f"llm.audio_transcriptions.{t.registry_route}.transcription.stream.works"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(t.provider,), + models=(t.litellm_params.model,), + mode=Mode.STREAM, + ) + ), + ), + ) + for t in STREAMING_TRANSCRIBERS +) + + +class _TranscriptStreamEvent(BaseModel): + type: str + delta: str = "" + text: str = "" + + +def _has_billed_row(rows: list[SpendLogRow]) -> bool: + return any((row.spend or 0) > 0 for row in rows) + + +class TestStreamedAudioTranscriptions: + @pytest.mark.parametrize("transcriber", STREAMING_PARAMS) + def test_stream_sends_transcript_deltas_then_done_and_bills_the_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, transcriber: _StreamingTranscriber + ) -> None: + model: Final = f"e2e-transcribe-stream-{transcriber.id}-{unique_marker()}" + model_id: Final = proxy.create_model(model, transcriber.litellm_params) + resources.defer(lambda: proxy.delete_model(model_id)) + client: Final = sdk.openai(resources.key()) + + with client.audio.transcriptions.with_streaming_response.create( + model=model, file=(WEATHER_WAV.name, WEATHER_WAV.read_bytes(), "audio/wav"), stream=True + ) as response: + content_type: Final = response_header(response.headers, "content-type") + call_id: Final = response_header(response.headers, "x-litellm-call-id") + payloads: Final = tuple( + line.removeprefix("data: ") for line in response.iter_lines() if line.startswith("data: ") + ) + + assert "text/event-stream" in (content_type or ""), f"stream=true did not stream: {content_type!r}" + assert payloads and payloads[-1] == STREAM_DONE, f"the stream did not end with data: [DONE]: {payloads[-3:]}" + assert payloads.count(STREAM_DONE) == 1, f"data: [DONE] was sent more than once: {payloads}" + events: Final = tuple(_TranscriptStreamEvent.model_validate_json(payload) for payload in payloads[:-1]) + deltas: Final = tuple(e.delta for e in events if e.type == "transcript.text.delta") + assert len(deltas) > 1, f"the transcript arrived in {len(deltas)} delta events, not as a stream: {events}" + done: Final = events[-1] + assert done.type == "transcript.text.done", f"the last event before [DONE] is not the done event: {done}" + assert "weather" in done.text.lower(), f"transcript of a spoken weather question lost the word: {done.text!r}" + assert "".join(deltas).strip() == done.text.strip(), ( + f"the deltas {deltas} do not add up to the final transcript {done.text!r}" + ) + + assert call_id, "the streamed response carries no x-litellm-call-id to find its spend log" + rows: Final = proxy.poll_logs_for_request_id(call_id, predicate=_has_billed_row) + assert _has_billed_row(rows), f"the streamed transcription was not billed: {rows}" diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 1e0a8cbb412..4967f18818a 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" -GEMINI_BACKEND = "gemini/gemini-2.5-flash" +GEMINI_BACKEND = "gemini/gemini-3.8-flash" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" @@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = { OPENAI_CHAT_MODEL: Final = "gpt-5.5" ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5" -GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash" +GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash" CHAT_MODELS: Final = ( pytest.param( diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 6d8cd24648e..63b8b60867a 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is self-contained and never inherits pricing another suite or a stale config left on the shared proxy. custom-priced-flash sets input/output rates deliberately far above the canonical gemini price; the isolation sibling shares the same -gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked +gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked independently: - billing: a real call's logged cost breakdown charges input and output tokens at @@ -37,10 +37,8 @@ from models import ( pytestmark = pytest.mark.e2e -BACKEND_MODEL = "gemini/gemini-2.5-flash" +BACKEND_MODEL = "gemini/gemini-3.8-flash" GEMINI_API_KEY = "os.environ/GEMINI_API_KEY" -# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) -# so an override that is ignored or under-applied bills at the base rate and fails. CUSTOM_INPUT_RATE = 5e-05 CUSTOM_OUTPUT_RATE = 1e-04 @@ -78,7 +76,7 @@ def _provision( input_cost_per_token: float | None, output_cost_per_token: float | None, ) -> str: - """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and + """Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and return its model name. With the cost fields set the deployment carries a custom pricing override; with them None it is a plain sibling on the same backend. The marker keeps the name unique so concurrent runs on the shared proxy never @@ -251,7 +249,7 @@ class TestCustomPricing: assert sibling_entry is not None, f"{sibling} absent from /model/info" # custom-priced-flash overrides pricing; the sibling shares the same - # gemini/gemini-2.5-flash backend but sets no override, so it must keep its + # gemini/gemini-3.8-flash backend but sets no override, so it must keep its # own price. Equal rates mean the override leaked into the shared cost map. assert ( sibling_entry.model_info.input_cost_per_token diff --git a/tests/e2e/llm_translation/test_google_native_e2e.py b/tests/e2e/llm_translation/test_google_native_e2e.py index b18683bd4ad..65a341dbc2a 100644 --- a/tests/e2e/llm_translation/test_google_native_e2e.py +++ b/tests/e2e/llm_translation/test_google_native_e2e.py @@ -21,7 +21,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e -UPSTREAM_MODEL = "gemini/gemini-2.5-flash" +UPSTREAM_MODEL = "gemini/gemini-3.8-flash" class _GenerateContentPart(BaseModel): diff --git a/tests/e2e/llm_translation/test_image_edits_e2e.py b/tests/e2e/llm_translation/test_image_edits_e2e.py index 66b0ebfc538..917662b9e52 100644 --- a/tests/e2e/llm_translation/test_image_edits_e2e.py +++ b/tests/e2e/llm_translation/test_image_edits_e2e.py @@ -24,7 +24,7 @@ from sdk_clients import SdkClients pytestmark = pytest.mark.e2e -IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1" +IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst" _TEST_PNG = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index 84e0674d959..59b1224a479 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -41,7 +41,7 @@ from proxy_client import ProxyClient from pydantic import BaseModel from sdk_clients import NO_PROXY_CACHE, SdkClients -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] FLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5" UNFLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 52d1c10d829..3eaeef6bddd 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row (call_type="pass_through_endpoint"), correlated by the id the caller was served: the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic. -Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming + +Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming + non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md. A passthrough call returning non-2xx fails hard (never a skip); once it returns @@ -32,7 +32,7 @@ from passthrough_client import ( completed_responses_object, ) -GEMINI_MODEL: Final = "gemini-2.5-flash" +GEMINI_MODEL: Final = "gemini-3.8-flash" ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5" EMBEDDING_MODEL = "text-embedding-3-small" REALTIME_MODEL = "gpt-realtime-2" @@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist: result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}") assert result.status_code == 403, ( - "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " + "a key restricted to gemini-3.8-flash must be denied a claude passthrough call, " f"got {result.status_code}: {result.body[:300]}" ) diff --git a/tests/e2e/llm_translation/test_realtime_http_e2e.py b/tests/e2e/llm_translation/test_realtime_http_e2e.py index ee3ce3893c2..3abb74308e8 100644 --- a/tests/e2e/llm_translation/test_realtime_http_e2e.py +++ b/tests/e2e/llm_translation/test_realtime_http_e2e.py @@ -1,14 +1,18 @@ """Vendor §9.19: realtime client_secrets + calls HTTP surface (LIT-4778). Websocket coverage already lives under realtime/; this file pins the HTTP -client-secret mint and the missing-auth contract. +client-secret mint (realtime, translation and transcription sessions), the +translation token scope, and the missing-auth contract. """ from __future__ import annotations +from dataclasses import dataclass +from typing import Final + import pytest from e2e_config import unique_marker -from e2e_http import NoBody, assert_auth_denied, unwrap +from e2e_http import NoBody, PartialBody, UnknownApiError, assert_auth_denied, unwrap from e2e_metadata import Domain, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -18,6 +22,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e REALTIME_BACKEND = "openai/gpt-realtime" +TRANSLATIONS_CLIENT_SECRETS: Final = "/v1/realtime/translations/client_secrets" class RealtimeSession(BaseModel): @@ -38,8 +43,65 @@ class RealtimeClientSecretRequest(BaseModel): session: RealtimeSession | None = None +class TurnDetection(BaseModel): + type: str + + +class SessionTranscription(BaseModel): + model: str | None = None + + +class SessionAudioInput(BaseModel): + transcription: SessionTranscription | None = None + turn_detection: TurnDetection | None = None + + +class SessionAudioOutput(BaseModel): + language: str | None = None + + +class SessionAudio(BaseModel): + input: SessionAudioInput | None = None + output: SessionAudioOutput | None = None + + +class TranslationSession(BaseModel): + model: str + audio: SessionAudio + + +class TranscriptionSession(BaseModel): + type: str = "transcription" + audio: SessionAudio + + +class SessionClientSecretRequest(BaseModel): + session: TranslationSession | TranscriptionSession + + +class NullableAudioInput(PartialBody): + turn_detection: TurnDetection | None = None + + +class NullableAudio(PartialBody): + input: NullableAudioInput + + +class NullableRealtimeSession(PartialBody): + type: str + model: str + instructions: str | None = None + audio: NullableAudio + + +class NullableClientSecretRequest(PartialBody): + model: str + session: NullableRealtimeSession + + class RealtimeClientSecretSession(BaseModel): type: str | None = None + audio: SessionAudio | None = None class RealtimeClientSecretResponse(BaseModel): @@ -49,15 +111,85 @@ class RealtimeClientSecretResponse(BaseModel): def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: - model = f"e2e-realtime-http-{unique_marker()}" - model_id = proxy.create_model( - model, - LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY"), + return _register_backend( + proxy, resources, LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY") ) + + +def _register_backend( + proxy: ProxyClient, resources: ResourceManager, litellm_params: LiteLLMParamsBody +) -> tuple[str, str]: + model: Final = f"e2e-realtime-http-{unique_marker()}" + model_id: Final = proxy.create_model(model, litellm_params) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() +@dataclass(frozen=True, slots=True) +class _AudioBackend: + id: str + registry_route: str + provider: Provider + litellm_params: LiteLLMParamsBody + + @property + def upstream_model(self) -> str: + return self.litellm_params.model.split("/", 1)[1] + + +def _openai_backend(model: str) -> _AudioBackend: + return _AudioBackend( + "openai", "openai", Provider.OPENAI, LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + ) + + +def _azure_backend(model: str) -> _AudioBackend: + return _AudioBackend( + "azure", + "azure_openai", + Provider.AZURE, + LiteLLMParamsBody(model=model, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"), + ) + + +TRANSLATION_BACKENDS: Final = ( + _openai_backend("openai/gpt-realtime-translate"), + _azure_backend("azure/gpt-realtime-translate"), +) +TRANSCRIPTION_BACKENDS: Final = ( + _openai_backend("openai/gpt-live-transcribe"), + _azure_backend("azure/gpt-live-transcribe"), +) + + +def _subject(backend: _AudioBackend) -> pytest.MarkDecorator: + return meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(backend.provider,), + models=(backend.litellm_params.model,), + ) + ) + + +TRANSLATION_PARAMS: Final = tuple( + pytest.param( + b, + id=b.id, + marks=(pytest.mark.covers(f"llm.realtime.{b.registry_route}.translation.stream.works"), _subject(b)), + ) + for b in TRANSLATION_BACKENDS +) +TRANSCRIPTION_PARAMS: Final = tuple(pytest.param(b, id=b.id, marks=_subject(b)) for b in TRANSCRIPTION_BACKENDS) + + +def _french_translation_request(model: str) -> SessionClientSecretRequest: + return SessionClientSecretRequest( + session=TranslationSession(model=model, audio=SessionAudio(output=SessionAudioOutput(language="fr"))) + ) + + class TestRealtimeHttp: @pytest.mark.covers("llm.realtime.openai.basic.nonstream.works") @meta( @@ -110,3 +242,150 @@ class TestRealtimeHttp: json=NoBody(), ) assert_auth_denied(result, "realtime calls missing auth") + + +class TestRealtimeTranslationClientSecrets: + @pytest.mark.parametrize("backend", TRANSLATION_PARAMS) + def test_translations_route_mints_a_french_translation_session( + self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend + ) -> None: + model, key = _register_backend(proxy, resources, backend.litellm_params) + secret: Final = unwrap( + proxy.transport.post( + TRANSLATIONS_CLIENT_SECRETS, + headers=proxy.transport.bearer(key), + json=_french_translation_request(model), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + assert secret.session is not None and secret.session.type == "translation", ( + f"the translations route did not mint a translation session: {secret.session}" + ) + output: Final = secret.session.audio.output if secret.session.audio else None + assert output is not None and output.language == "fr", ( + f"the requested target language did not reach the session: {secret.session}" + ) + + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(TRANSLATION_BACKENDS[0].litellm_params.model,), + ) + ) + def test_translation_token_is_refused_by_the_realtime_calls_route( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params) + secret: Final = unwrap( + proxy.transport.post( + TRANSLATIONS_CLIENT_SECRETS, + headers=proxy.transport.bearer(key), + json=_french_translation_request(model), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + result: Final = proxy.transport.send( + "/v1/realtime/calls", headers=proxy.transport.bearer(secret.value), json=NoBody() + ) + assert result.status_code == 401, ( + f"a translation token opened a plain realtime call: {result.status_code} {result.body[:300]}" + ) + assert "not valid for this Realtime endpoint" in result.body, ( + f"the refusal does not say the token belongs to another endpoint: {result.body[:300]}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(TRANSLATION_BACKENDS[0].litellm_params.model,), + ) + ) + def test_translation_session_on_the_realtime_route_is_rejected( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params) + result: Final = proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=RealtimeClientSecretRequest(model=model, session=RealtimeSession(type="translation", model=model)), + response_type=RealtimeClientSecretResponse, + ) + match result: + case UnknownApiError(status_code=400, body=body): + assert "translations endpoint" in body, f"the 400 does not point at the translations route: {body}" + case other: + pytest.fail(f"a translation session on /v1/realtime/client_secrets expected a 400, got {other!r}") + + +class TestRealtimeTranscriptionClientSecrets: + @pytest.mark.parametrize("backend", TRANSCRIPTION_PARAMS) + def test_transcription_session_runs_on_the_deployment_model( + self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend + ) -> None: + model, key = _register_backend(proxy, resources, backend.litellm_params) + secret: Final = unwrap( + proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=SessionClientSecretRequest( + session=TranscriptionSession( + audio=SessionAudio(input=SessionAudioInput(transcription=SessionTranscription(model=model))) + ) + ), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + assert secret.session is not None and secret.session.type == "transcription", ( + f"the client secret did not mint a transcription session: {secret.session}" + ) + audio_input: Final = secret.session.audio.input if secret.session.audio else None + transcription: Final = audio_input.transcription if audio_input else None + assert transcription is not None and transcription.model == backend.upstream_model, ( + f"the session does not transcribe with the deployment's model {backend.upstream_model}: {secret.session}" + ) + + +class TestRealtimeClientSecretNulls: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(REALTIME_BACKEND,), + ) + ) + def test_top_level_null_is_dropped_and_nested_null_is_kept( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register(proxy, resources) + secret: Final = unwrap( + proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=NullableClientSecretRequest( + model=model, + session=NullableRealtimeSession( + type="realtime", + model=model, + instructions=None, + audio=NullableAudio(input=NullableAudioInput(turn_detection=None)), + ), + ), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"a session with instructions: null was not minted: {secret}" + audio_input: Final = secret.session.audio.input if secret.session and secret.session.audio else None + assert audio_input is not None and "turn_detection" in audio_input.model_fields_set, ( + f"the session does not report turn detection: {secret.session}" + ) + assert audio_input.turn_detection is None, ( + f"turn_detection: null was dropped, so the session fell back to {audio_input.turn_detection}" + ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index a0f185c4077..6b299fad18c 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" -GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash" OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 4dd681f59e8..3903d1a8363 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -18,10 +18,11 @@ import json import os import time from dataclasses import dataclass -from typing import Callable, Literal +from datetime import datetime, timedelta, timezone +from typing import Callable, Final, Literal import pytest -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation from e2e_metadata import step @@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel): guardrail_name: str | None = None +_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + class LangfuseObservation(BaseModel): - model_config = ConfigDict(extra="ignore", populate_by_name=True) + model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True) id: str trace_id: str | None = Field(default=None, alias="traceId") name: str | None = None type: str | None = None - calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost") + calculated_total_cost: float | None = Field( + default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost") + ) level: str | None = None - input: object | None = None - output: object | None = None - metadata: object | None = None + input: JsonValue = None + output: JsonValue = None + metadata: JsonValue = None usage: object | None = None usage_details: object | None = Field(default=None, alias="usageDetails") model: str | None = None + @field_validator("input", "output", "metadata", mode="before") + @classmethod + def decode_json_fields(cls, value: object) -> JsonValue: + if not isinstance(value, str): + return _LANGFUSE_JSON.validate_python(value) + try: + return _LANGFUSE_JSON.validate_json(value) + except ValidationError: + return value + + class LangfuseObservationList(BaseModel): model_config = ConfigDict(extra="ignore") @@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: class LangfuseListParams(BaseModel): - model_config = ConfigDict(populate_by_name=True) + model_config = ConfigDict(populate_by_name=True, frozen=True) limit: int = 100 trace_id: str | None = Field(default=None, alias="traceId") name: str | None = None - from_start_time: str | None = Field(default=None, alias="fromStartTime") + from_start_time: str = Field(alias="fromStartTime") + to_start_time: str = Field(alias="toStartTime") + fields: str = "core,basic,io,metadata,model,usage" + expand_metadata: str = Field(default="attributes", alias="expandMetadata") @dataclass(frozen=True, slots=True) @@ -580,14 +600,16 @@ class LoggingClient: name: str | None = None, from_start_time: str | None = None, ) -> list[LangfuseObservation]: - result = get( - URL(f"{creds.host}/api/public/observations"), + now: Final = datetime.now(timezone.utc) + result: Final = get( + URL(f"{creds.host}/api/public/v2/observations"), headers=creds.auth_headers, params=LangfuseListParams( limit=100, traceId=trace_id, name=name, - fromStartTime=from_start_time, + fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(), + toStartTime=now.isoformat(), ), response_type=LangfuseObservationList, timeout=30.0, diff --git a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py index e2dadf9e696..37f363772ae 100644 --- a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py +++ b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py @@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e class LangsmithHeaders(Headers): x_api_key: str = Field(serialization_alias="x-api-key") + x_tenant_id: str | None = Field(default=None, serialization_alias="x-tenant-id") class LangsmithRunExtra(BaseModel): @@ -50,6 +51,7 @@ class LangsmithCreds: api_key: str base_url: str project: str + tenant_id: str | None def load_langsmith_creds() -> LangsmithCreds: @@ -62,6 +64,7 @@ def load_langsmith_creds() -> LangsmithCreds: api_key=api_key, base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"), project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e", + tenant_id=os.getenv("LANGSMITH_TENANT_ID"), ) @@ -69,7 +72,7 @@ def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None: result = get_external( f"{creds.base_url}/runs/{run_id}", response_type=LangsmithRun, - headers=LangsmithHeaders(x_api_key=creds.api_key), + headers=LangsmithHeaders(x_api_key=creds.api_key, x_tenant_id=creds.tenant_id), ) match result: case Success(data=run): @@ -102,7 +105,10 @@ class TestLangsmithBatchSerialization: async def test_non_json_native_metadata_reaches_langsmith(self) -> None: creds: Final = load_langsmith_creds() logger: Final = LangsmithLogger( - langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url + langsmith_api_key=creds.api_key, + langsmith_project=creds.project, + langsmith_base_url=creds.base_url, + langsmith_tenant_id=creds.tenant_id, ) assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API" marker: Final = unique_marker() diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 81a6407f642..fb6e481dc33 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool: return False -def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: +def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: """The enforced behavior: the destination holds exactly one call-id-tagged trace for the call, rooted at the SERVER span, with auth/db children and the gen-AI span all connected into that one tree - no dangling parent @@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - roots = [span for span in trace.spans if not span.references] assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}" root = roots[0] - assert root.operation_name == f"POST {route}", ( - f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}" - ) assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}" + request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target") + assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}" assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}" assert any(name.startswith(DB_SPAN_PREFIX) for name in names), ( @@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces: return otel_reader.poll_traces_for_call( call_id=call_id, - settled_names={f"POST {route}", f"auth {route}", genai_span}, + settled_names={f"auth {route}", genai_span}, settled_prefixes={DB_SPAN_PREFIX}, ) -def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: +def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: """The enforced behavior: the gen-AI span for the attempt that served the stream records a TTFT that is a real measurement - present, numeric, positive, and strictly less than that span's own total duration. A TTFT of @@ -282,7 +281,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -315,7 +314,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) @meta( @@ -333,7 +332,7 @@ class TestOtelTraceCompleteness: """This test verifies that one successful non-streaming /v1/messages request produces exactly one complete OTEL trace. - The trace must have a single root span named "POST /v1/messages". The + The trace must have a single root SERVER span identifying /v1/messages. The authentication, database, and model-call spans must all belong to the same trace and have valid parent relationships leading back to that root. @@ -353,7 +352,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"]) @meta( @@ -392,7 +391,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -443,7 +442,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -500,7 +499,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -557,7 +556,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) one_served_genai_span(traces.hits[0], genai_span) @@ -614,7 +613,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"]) @meta( @@ -661,7 +660,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"]) @meta( @@ -708,7 +707,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -763,7 +762,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( @@ -825,7 +824,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py index ae4e6db4e15..ea4afee78ca 100644 --- a/tests/e2e/management/test_budget_customer_user_org_e2e.py +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -418,7 +418,7 @@ class TestBudgetListV1: @pytest.mark.covers("mgmt.budget.list_v1.happy_path") @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None: - assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400 + assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400 assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400 @pytest.mark.covers("mgmt.budget.list_v1.admin_only") diff --git a/tests/e2e/mcp/test_mcp_semantic_tool_filter_e2e.py b/tests/e2e/mcp/test_mcp_semantic_tool_filter_e2e.py index 6e564f4625a..1ec2a6822e3 100644 --- a/tests/e2e/mcp/test_mcp_semantic_tool_filter_e2e.py +++ b/tests/e2e/mcp/test_mcp_semantic_tool_filter_e2e.py @@ -5,7 +5,8 @@ embedding deployment, and turns on `litellm_settings.mcp_semantic_tool_filter` t /config/update. A chat completion that attaches the server and asks to search logs must report, in the x-litellm-semantic-filter headers, that the catalog was cut to at most top_k tools, all of them tools the key lists. Requires DD_API_KEY + DD_APP_KEY and the proxy's -OPENAI_API_KEY for the embedding deployment. +OPENAI_API_KEY for the embedding deployment. The filter is proxy-wide, so the test is +marked `proxy_wide_config` and runs on a stack the main suite is not driving. """ from __future__ import annotations @@ -23,7 +24,7 @@ from mcp_client import McpClient from models import ChatBody, ChatMessage, LiteLLMParamsBody, McpChatTool from pydantic import BaseModel -pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] +pytestmark = [pytest.mark.e2e, pytest.mark.proxy_wide_config] TOP_K: Final = 3 diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 9c12e2146c9..86af30f413a 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -653,6 +653,35 @@ class ResponsesStreamBody(BaseModel): cache: dict[str, bool] | None = {"no-cache": True} +class ResponsesFunctionTool(BaseModel): + model_config = ConfigDict(frozen=True) + type: Literal["function"] = "function" + name: str + description: str | None = None + parameters: ToolInputSchema + + +class ResponsesToolBody(BaseModel): + model_config = ConfigDict(frozen=True) + model: str + input: str + tools: list[ResponsesFunctionTool] + tool_choice: Literal["required"] | None = None + guardrails: list[str] + max_output_tokens: int = 128 + + +class ResponsesOutputItem(BaseModel): + model_config = ConfigDict(frozen=True) + type: str + name: str | None = None + + +class ResponsesToolOutput(BaseModel): + model_config = ConfigDict(frozen=True) + output: list[ResponsesOutputItem] + + class CountTokensBody(BaseModel): """POST /v1/messages/count_tokens body: the /v1/messages shape minus max_tokens (the endpoint only counts the prompt).""" @@ -1262,27 +1291,6 @@ class CostMap(RootModel[dict[str, CostMapEntry]]): pass -class ConfigPatchResponse(BaseModel): - status: str - values: dict[str, float | dict[str, float]] - - -class CostDiscountConfig(RootModel[dict[str, float]]): - pass - - -class CostMarginConfig(RootModel[dict[str, float | dict[str, float]]]): - pass - - -class CostDiscountConfigResponse(BaseModel): - values: dict[str, float] - - -class CostMarginConfigResponse(BaseModel): - values: dict[str, float | dict[str, float]] - - class BedrockGuardrailParams(BaseModel): guardrail: Literal["bedrock"] = "bedrock" mode: Literal["pre_call"] = "pre_call" diff --git a/tests/e2e/otel_client.py b/tests/e2e/otel_client.py index 5f93c677d89..b42235484dc 100644 --- a/tests/e2e/otel_client.py +++ b/tests/e2e/otel_client.py @@ -28,11 +28,10 @@ from dataclasses import dataclass from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, Field - from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT from e2e_http import URL, NetworkError, NoBody, Result, Success, get from e2e_metadata import step +from pydantic import BaseModel, ConfigDict, Field #: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default). JAEGER_SERVICE = "litellm" @@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]: time.sleep(POLL_INTERVAL) -def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: - present = set(trace.span_names()) - return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) +def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: + root: Final = root_span(trace) + present: Final = frozenset(trace.span_names()) + return ( + root is not None + and root.kind == "server" + and names.issubset(present) + and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) + ) def root_span(trace: JaegerTrace) -> JaegerSpan | None: @@ -172,7 +177,7 @@ class _Observation: def settled(self, names: set[str], prefixes: set[str]) -> bool: hits: Final = self.traces.hits - return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes) + return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes) @dataclass(frozen=True, slots=True) diff --git a/tests/e2e/perf/setup/upgrade.py b/tests/e2e/perf/setup/upgrade.py deleted file mode 100644 index c9114ccbff2..00000000000 --- a/tests/e2e/perf/setup/upgrade.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import math -import time -from collections.abc import Callable -from dataclasses import replace -from typing import Final - -import psycopg -from cases.small_bodies_5rps import CASE -from e2e_config import MASTER_KEY -from e2e_http import send_text_external -from prometheus_client.parser import text_string_to_metric_families -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter - -from setup.performance_test_case import PerformanceTestCase -from setup.upgrade_worker import UpgradeState, fingerprints, missing_indexes, read_state, release_indexes -from setup.wire import WireRecord - -MAX_TRAFFIC_SECONDS: Final = 6 * 3600 -_ROWS: Final = TypeAdapter(tuple[tuple[str, str | None], ...]) -_ACTIVITY: Final = TypeAdapter(tuple[list[int], int]) -_COUNT: Final = TypeAdapter(tuple[int]) -_SERVICES: Final = ("postgres", "batch_write_to_db") - - -def upgrade_case() -> PerformanceTestCase: - return replace( - CASE, - name="database_upgrade_5rps", - traffic=replace(CASE.traffic, total_requests=int(CASE.traffic.rps * MAX_TRAFFIC_SECONDS)), - limits=tuple( - limit - for limit in CASE.limits - if limit.metric - in { - "unexpected_status_count", - "gateway_added_seconds", - "time_to_first_token_seconds", - "memory_growth_megabytes", - } - ), - ) - - -class MetricsHeaders(BaseModel): - model_config = ConfigDict(frozen=True) - authorization: str = Field(default=f"Bearer {MASTER_KEY}", alias="Authorization") - - -def postgres_failures(text: str) -> float: - families: Final = tuple(text_string_to_metric_families(text)) - names: Final = frozenset(family.name for family in families) - required: Final = frozenset(f"litellm_{service}_failed_requests" for service in _SERVICES) - samples: Final = tuple(sample for family in families for sample in family.samples) - has_database_activity: Final = any( - sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0 - for sample in samples - ) - assert required <= names or has_database_activity, f"Missing database service counters: {required - names}" - failures: Final = tuple(sample.value for sample in samples if sample.name in {f"{name}_total" for name in required}) - assert all(math.isfinite(value) and value >= 0 for value in failures), "Invalid database failure counter" - return sum(failures) - - -def assert_database_metrics(urls: tuple[str, ...]) -> None: - assert urls, "E2E_DATABASE_METRICS_URLS must name every gateway/backend /metrics endpoint" - for url in urls: - assert_database_metrics_url(url) - - -def assert_database_metrics_url(url: str) -> None: - result: Final = send_text_external("GET", url, headers=MetricsHeaders(), timeout=10) - assert result.status_code == 200, f"Database metrics unavailable at {url}: {result.status_code}" - assert postgres_failures(result.body) == 0, f"Failed PostgreSQL operations reported at {url}" - - -def persisted(connection: psycopg.Connection[tuple[object, ...]], records: tuple[WireRecord, ...]) -> frozenset[str]: - ids: Final = tuple(record.call_id for record in records if record.call_id) - if not ids: - return frozenset() - rows: Final = _ROWS.validate_python( - connection.execute( - 'SELECT request_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=ANY(%s)', (list(ids),) - ).fetchall() - ) - return matched_response_ids(records, rows) - - -def matched_response_ids(records: tuple[WireRecord, ...], rows: tuple[tuple[str, str | None], ...]) -> frozenset[str]: - call_ids: Final = frozenset(call_id for _, call_id in rows if call_id) - return frozenset(record.response_id for record in records if record.response_id and record.call_id in call_ids) - - -def activity(connection: psycopg.Connection[tuple[object, ...]]) -> tuple[list[int], int]: - return _ACTIVITY.validate_python( - connection.execute("""SELECT - ARRAY(SELECT pid FROM pg_stat_progress_create_index WHERE datname=current_database()), - (SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock') - """).fetchone() - ) - - -def assert_finished(connection: psycopg.Connection[tuple[object, ...]], state: UpgradeState) -> None: - assert state.phase == "complete", f"Upgrade did not finish: {state.phase}" - assert not missing_indexes(connection, state.required_indexes), "Required target indexes are missing or invalid" - (missing,) = _COUNT.validate_python( - connection.execute( - """ - SELECT count(*) FROM unnest(%s::text[]) name WHERE NOT EXISTS ( - SELECT 1 FROM "_prisma_migrations" WHERE migration_name=name - AND finished_at IS NOT NULL AND rolled_back_at IS NULL)""", - (list(state.required_migrations),), - ).fetchone() - ) - assert missing == 0, f"{missing} target migrations were not applied" - (unfinished,) = _COUNT.validate_python( - connection.execute(""" - SELECT count(*) FROM "_prisma_migrations" WHERE finished_at IS NULL AND rolled_back_at IS NULL - """).fetchone() - ) - assert unfinished == 0, f"{unfinished} unfinished migrations" - assert fingerprints(connection, tuple(state.baseline_rows)) == state.baseline_rows, ( - "Baseline rows changed or disappeared" - ) - - -class UpgradeMonitor: - def __init__( - self, - *, - state: Callable[[], UpgradeState], - check_metrics: Callable[[], None], - read_activity: Callable[[], tuple[list[int], int]], - read_persisted: Callable[[tuple[WireRecord, ...]], frozenset[str]], - start_indexes: Callable[[], None], - clock: Callable[[], float] = time.monotonic, - wall_clock: Callable[[], float] = time.time, - ) -> None: - self.state: Final = state - self.check_metrics: Final = check_metrics - self.read_activity: Final = read_activity - self.read_persisted: Final = read_persisted - self.start_indexes: Final = start_indexes - self.clock: Final = clock - self.wall_clock: Final = wall_clock - self.started: Final = clock() - self.status_limit: Final = next( - limit.max for limit in CASE.limits if limit.metric == "unexpected_status_count" and limit.max is not None - ) - self.pending: tuple[WireRecord, ...] = () - self.processed = 0 - self.successes = 0 - self.persisted_count = 0 - self.overlap_writes = 0 - self.max_lock_waiters = 0 - self.previous_builds: frozenset[int] = frozenset() - self.build_seen_at: float | None = None - self.released = False - self.completed_at: float | None = None - - def poll(self, records: tuple[WireRecord, ...]) -> bool: - self.check_metrics() - current: Final = self.state() - assert current.phase != "failed", "Target migration/index worker failed" - assert self.clock() - self.started < MAX_TRAFFIC_SECONDS, "Upgrade exceeded its six-hour operational timeout" - successful: Final = tuple(record for record in records if record.status == 200) - self.processed += len(records) - self.successes += len(successful) - assert self.processed - self.successes <= self.status_limit, "Too many unexpected non-200 responses" - assert all(record.ok and record.text for record in successful), "Failed or empty inference response" - assert all(record.response_id and record.call_id for record in successful), ( - "Response missing spend-log identifiers" - ) - builds, lock_waiters = self.read_activity() - if not self.previous_builds.intersection(builds): - self.build_seen_at = self.wall_clock() if builds else None - self.max_lock_waiters = max(self.max_lock_waiters, lock_waiters) - self.pending += successful - found: Final = self.read_persisted(self.pending) - self.persisted_count += len(found) - still_building, _ = self.read_activity() - since: Final = self.build_seen_at - if since is not None and frozenset(builds).intersection(still_building): - self.overlap_writes += sum( - 1 for record in self.pending if record.response_id in found and record.started_at >= since - ) - self.previous_builds = frozenset(builds) - self.pending = tuple(record for record in self.pending if record.response_id not in found) - if self.successes and not self.released: - self.start_indexes() - self.released = True - if current.phase == "complete" and self.completed_at is None: - self.completed_at = self.clock() - return self.completed_at is not None and self.clock() - self.completed_at >= 60 and self.successes >= 300 - - def assert_complete(self, state: UpgradeState) -> None: - assert self.successes >= 300, f"Only {self.successes} requests completed" - assert not self.pending, f"{len(self.pending)} successful requests have no matching spend log" - assert self.persisted_count == self.successes, "Spend-log count does not match successful requests" - assert not state.pending_indexes or self.overlap_writes > 0, "No persisted requests observed during index work" - - -def upgrade_properties(state: UpgradeState, monitor: UpgradeMonitor) -> tuple[tuple[str, float], ...]: - """The upgrade's own numbers under the `perf_` prefix the dashboard ingests, next to the perf rows.""" - return ( - ("perf_upgrade_schema_seconds", state.schema_seconds), - ("perf_upgrade_index_seconds", state.index_seconds), - ("perf_upgrade_overlap_writes", float(monitor.overlap_writes)), - ("perf_upgrade_max_lock_waiters", float(monitor.max_lock_waiters)), - ("perf_upgrade_pending_indexes", float(len(state.pending_indexes))), - ("perf_upgrade_unmatched_requests", float(len(monitor.pending))), - ) - - -def monitor_for(connection: psycopg.Connection[tuple[object, ...]], urls: tuple[str, ...]) -> UpgradeMonitor: - return UpgradeMonitor( - state=lambda: read_state(connection), - check_metrics=lambda: assert_database_metrics(urls), - read_activity=lambda: activity(connection), - read_persisted=lambda records: persisted(connection, records), - start_indexes=lambda: release_indexes(connection), - ) diff --git a/tests/e2e/perf/test_database_upgrade.py b/tests/e2e/perf/test_database_upgrade.py deleted file mode 100644 index fba6af30014..00000000000 --- a/tests/e2e/perf/test_database_upgrade.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import os -import sys -import time -from pathlib import Path -from typing import Final - -import pytest -from idp import Keycloak -from lifecycle import ResourceManager -from memory_readings import read_rss_everywhere -from proxy_client import ProxyClient -from setup.datastore import read_db_counters, read_redis_counters -from setup.observe import ( - Observation, - evaluate, - junit_properties, - read_mock_requests, - render, - reset_mock, - set_mock_latency, -) -from setup.plan import build_plan -from setup.podcpu import gateway_cpu_sampler -from setup.provision import MOCK_PROVIDERS, build_setup, mock_url -from setup.run import run_plan -from setup.upgrade import assert_finished, monitor_for, upgrade_case, upgrade_properties -from setup.upgrade_worker import connect, read_state - -pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] - - -@pytest.mark.skipif(os.environ.get("E2E_DATABASE_UPGRADE") != "1", reason="Dedicated disposable snapshot lane only") -def test_database_upgrade( - proxy: ProxyClient, idp: Keycloak, resources: ResourceManager, request: pytest.FixtureRequest -) -> None: - case: Final = upgrade_case() - urls: Final = tuple(mock_url(provider) for provider in MOCK_PROVIDERS) - for url in urls: - _ = reset_mock(url) - _ = set_mock_latency(url, case.mock_latency) - setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources) - metrics: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip()) - with connect() as connection: - assert read_state(connection).phase == "schema_ready", "Expected the target schema ready, index work waiting" - rss_before: Final = read_rss_everywhere(proxy) - db_before: Final = read_db_counters() - redis_before: Final = read_redis_counters() - monitor: Final = monitor_for(connection, metrics) - with gateway_cpu_sampler() as cpu: - outcome: Final = run_plan(case, build_plan(case), setup, stop_when=monitor.poll, request_timeout=30) - assert outcome.locust_exit == 0, outcome.locust_output - remaining: Final = outcome.records[monitor.processed :] - _ = monitor.poll(remaining) - deadline: Final = time.monotonic() + 120 - while monitor.pending and time.monotonic() < deadline: - time.sleep(1) - _ = monitor.poll(()) - state: Final = read_state(connection) - _ = sys.stdout.write( - f"successful_requests={monitor.successes} persisted_requests={monitor.persisted_count} " - f"unmatched_requests={len(monitor.pending)}\n" - ) - rss_after: Final = read_rss_everywhere(proxy) - observed: Final = Observation( - records=outcome.records, - mock_requests=tuple(seen for url in urls for seen in read_mock_requests(url)), - guardrail_calls=(), - rss_before=rss_before, - rss_after=rss_after, - wall_seconds=outcome.wall_seconds, - spend_logged_share=1, - db_before=db_before, - db_after=read_db_counters(), - redis_before=redis_before, - redis_after=read_redis_counters(), - gateway_cpu_seconds=cpu.seconds, - ) - rows, exceeded = evaluate(case, observed) - item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped - assert isinstance(item, pytest.Item) - item.user_properties[:] = [ - entry for entry in item.user_properties if not entry[0].startswith("perf_") - ] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set - item.user_properties.extend(junit_properties(rows)) - item.user_properties.extend(upgrade_properties(state, monitor)) - _ = sys.stdout.write(render(case, observed, rows) + "\n") - _ = sys.stdout.write( - f"schema_seconds={state.schema_seconds:.3f} index_seconds={state.index_seconds:.3f} " - f"overlap_writes={monitor.overlap_writes} max_lock_waiters={monitor.max_lock_waiters} " - f"pending_indexes={state.pending_indexes}\n" - ) - artifact: Final = os.environ.get("E2E_ARTIFACT_DIR") - if artifact: - _ = (Path(artifact) / "database-upgrade.json").write_text(state.model_dump_json(), encoding="utf-8") - monitor.assert_complete(state) - assert_finished(connection, state) - assert {(r.replica, r.hostname) for r in rss_before.readings} == { - (r.replica, r.hostname) for r in rss_after.readings - }, "Gateway pods replaced during the upgrade" - assert not exceeded, "\n".join(exceeded) diff --git a/tests/e2e/perf/test_perf_cases.py b/tests/e2e/perf/test_perf_cases.py index db657a04b00..0a137e84572 100644 --- a/tests/e2e/perf/test_perf_cases.py +++ b/tests/e2e/perf/test_perf_cases.py @@ -48,7 +48,7 @@ from setup.ramp import step_properties from setup.run import run_plan from setup.telemetry import read_gateway_telemetry, telemetry_sampler -pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] +pytestmark = pytest.mark.perf _SPEND_SAMPLE: Final = 40 _WARMUP_SECONDS: Final = 60 diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/perf/test_reliability_memory_e2e.py similarity index 77% rename from tests/e2e/router/test_reliability_memory_e2e.py rename to tests/e2e/perf/test_reliability_memory_e2e.py index fb114092303..2066137bb72 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/perf/test_reliability_memory_e2e.py @@ -1,17 +1,19 @@ """Live e2e: a few hundred requests that fail before any provider answers must not -grow the proxy's resident memory past a loose ceiling once the proxy is warm. +grow the proxy's resident memory past a loose ceiling once the proxy is warm. It reads +the proxy's own RSS, so it runs in the perf lane, where nothing else is hitting the stack. The regression this guards shipped in v1.100.0: every retry breadcrumb copied the whole request and the copies nested into one router-global list, so a proxy under retry-heavy failing traffic grew until it was OOM-killed. The traffic here has that shape: a model group whose deployments refuse at the socket (an unreachable base -URL) with cooldown_time 0 so the router keeps retrying them, per-request retries, -and a fallback group that refuses the same way, each request carrying a long chat +URL, with a literal key since the perf stack carries no provider keys) with +cooldown_time 0 so the router keeps retrying them, per-request retries, and a +fallback group that refuses the same way, each request carrying a long chat transcript so every whole-request copy costs hundreds of containers instead of a -handful. Under the stack's cooldown policy a -deployment that fails a handful of times in a row is benched (a bad-credential 401 -included), the router answers "No deployments available" without retrying, and the -retry loop that leaks stops running; cooldown_time 0 keeps it running. +handful. Under the stack's cooldown policy a deployment that fails a handful of +times in a row is benched (a bad-credential 401 included), the router answers "No +deployments available" without retrying, and the retry loop that leaks stops running; +cooldown_time 0 keeps it running. Four identical batches run back to back. The first two are the warmup that grows the proxy's caches and allocator arenas, the last two are the measured window, and @@ -35,19 +37,11 @@ single load-balanced address reaches the workers behind it one answer at a time, a worker that answered only one checkpoint would otherwise drop out of the comparison, which is where a leaking worker could hide. -RSS alone is a coarse gauge: on the release stack (spend logs storing prompts, -json logs, prometheus and otel callbacks) the same v1.100.0 breadcrumbs grew RSS -by only about 15 MB per 300 failing requests, while every failing request's stored -request snapshot carried a copy of the request per failed attempt, over 100 KB on -the first call and a couple of MB once the copies nested, against tens of KB with -the fix. So the first check sends one failing request before the phases, reads its -spend log back through /spend/logs, and holds the stored request body to a fixed -size budget: the deterministic catch for a breadcrumb that copies the whole -request. It runs before the phases because the leaking writer drops its own rows -under the phases' traffic (a queue budget hit, a recursion limit on the nested -copies), which would turn the size check into a missing-row check. +The stored-request half of that regression (every failing request's spend log carrying a +copy of the request per failed attempt) is a deterministic size check, so it lives in +tests/integration/spend/test_failing_request_stored_snapshot.py. -A second, cheaper check holds the idle footprint: every worker's RSS as the harness +A cheaper check holds the idle footprint: every worker's RSS as the harness read it at collection time, before this pytest process sent any traffic (see conftest.pytest_collection_finish), must sit under a fixed budget. On the release gate that is a fresh stack right after its readiness gate, one worker per @@ -74,7 +68,6 @@ from typing import Final import pytest -from complexity_router_client import ComplexityRouterClient from e2e_config import ( MEMORY_CONCURRENCY, MEMORY_IDLE_RSS_BUDGET_MB, @@ -83,7 +76,6 @@ from e2e_config import ( MEMORY_RSS_CEILING_MB, MEMORY_RSS_SAMPLE_INTERVAL_SECONDS, MEMORY_RSS_SETTLE_SAMPLES, - MEMORY_STORED_REQUEST_BUDGET_KB, MEMORY_TRANSCRIPT_TURNS, PROXY_REPLICA_URLS, unique_marker, @@ -91,11 +83,12 @@ from e2e_config import ( from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere -from models import ChatMessage, RouterSettingsOverride, SpendLogRow +from models import ChatMessage, LiteLLMParamsBody, ReliabilityChatBody, RouterSettingsOverride from proxy_client import ProxyClient -from reliability_support import REAL_MODEL, chat_override, create_never_benched_refusing_deployment -pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] +pytestmark = [pytest.mark.e2e, pytest.mark.perf] + +REFUSING_MODEL: Final = "openai/gpt-5.5" DEPLOYMENTS_PER_GROUP: Final = 2 RSS_SAMPLE_CAP: Final = 4 * MEMORY_RSS_SETTLE_SAMPLES @@ -110,7 +103,6 @@ class FailedCall: status_code: int seconds: float body_head: str - call_id: str | None @dataclass(frozen=True, slots=True) @@ -123,8 +115,17 @@ class WorkerGrowth: return self.after.ram_usage_mb - self.warm.ram_usage_mb +def _create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> str: + return proxy.create_model( + name, + LiteLLMParamsBody( + model=REFUSING_MODEL, api_key="sk-unreachable", api_base="http://127.0.0.1:9/v1", cooldown_time=0 + ), + ) + + def _register_refusing_group(proxy: ProxyClient, resources: ResourceManager, name: str) -> None: - for model_id in tuple(create_never_benched_refusing_deployment(proxy, name) for _ in range(DEPLOYMENTS_PER_GROUP)): + for model_id in tuple(_create_never_benched_refusing_deployment(proxy, name) for _ in range(DEPLOYMENTS_PER_GROUP)): resources.defer(lambda model_id=model_id: proxy.delete_model(model_id)) @@ -141,10 +142,18 @@ TRANSCRIPT: Final = _transcript(MEMORY_TRANSCRIPT_TURNS) def _fail_once(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> FailedCall: started: Final = time.perf_counter() - resp: Final = chat_override( - proxy, key, model, f"memory regression {unique_marker()}", override=override, history=TRANSCRIPT + resp: Final = proxy.transport.send( + "/chat/completions", + headers=proxy.transport.bearer(key), + json=ReliabilityChatBody( + model=model, + messages=[*TRANSCRIPT, ChatMessage(role="user", content=f"memory regression {unique_marker()}")], + max_tokens=512, + router_settings_override=override, + cache={"no-cache": True}, + ), ) - return FailedCall(resp.status_code, time.perf_counter() - started, resp.body[:300], resp.call_id) + return FailedCall(resp.status_code, time.perf_counter() - started, resp.body[:300]) def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> tuple[FailedCall, ...]: @@ -259,24 +268,6 @@ def _assert_every_call_failed_through_fallback(calls: Sequence[FailedCall], fall ) -def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: - assert call.call_id, ( - f"the failing call carried no x-litellm-call-id header, so its spend log cannot be read back: {call.body_head}" - ) - rows: Final[Sequence[SpendLogRow]] = proxy.poll_logs_for_request_id(call.call_id) - assert rows, ( - f"no spend log row appeared for failing call {call.call_id} within the poll window: either the stack " - "writes no spend logs or its writer dropped the row, which the v1.100.0 one did once the stored " - "request outgrew the writer's queue budget" - ) - snapshot: Final = rows[0].proxy_server_request - assert snapshot, ( - f"spend log {call.call_id} stored no request body, so the stack is not running with " - "general_settings.store_prompts_in_spend_logs and the stored-request check would pass vacuously" - ) - return len(json.dumps(snapshot).encode()) / 1024 - - class TestReliabilityMemory: @pytest.mark.covers("reliability.perf.idle_memory.under_slo") @meta( @@ -309,13 +300,13 @@ class TestReliabilityMemory: Subject( domain=Domain.ROUTING, providers=(Provider.OPENAI,), - models=(REAL_MODEL,), + models=(REFUSING_MODEL,), mode=Mode.NONSTREAM, ) ) - def test_failing_requests_do_not_grow_rss_or_stored_request( + def test_failing_requests_do_not_grow_rss( self, - client: ComplexityRouterClient, + proxy: ProxyClient, resources: ResourceManager, scoped_key: str, record_property: Callable[[str, object], None], @@ -323,35 +314,25 @@ class TestReliabilityMemory: marker: Final = unique_marker() primary: Final = f"reliability-memory-{marker}" fallback: Final = f"reliability-memory-fb-{marker}" - _register_refusing_group(client.proxy, resources, primary) - _register_refusing_group(client.proxy, resources, fallback) + _register_refusing_group(proxy, resources, primary) + _register_refusing_group(proxy, resources, fallback) override: Final = RouterSettingsOverride( num_retries=MEMORY_RETRIES_PER_REQUEST, fallbacks=[{primary: [fallback]}] ) - probe: Final = _fail_once(client.proxy, scoped_key, primary, override) - _assert_every_call_failed_through_fallback((probe,), fallback) - stored_kb: Final = _stored_request_kb(client.proxy, probe) - assert stored_kb <= MEMORY_STORED_REQUEST_BUDGET_KB, ( - f"the spend log of one failing request stored a {stored_kb:.0f} KB request body, past the " - f"{MEMORY_STORED_REQUEST_BUDGET_KB:.0f} KB budget for a {len(TRANSCRIPT)}-message transcript with " - f"{MEMORY_RETRIES_PER_REQUEST} retries and a fallback; the retry breadcrumbs are copying the whole " - f"request into the stored snapshot the way the v1.100.0 ones did" - ) - warm: Final = _settled_rss_after_batches( - client.proxy, + proxy, scoped_key, primary, override, fallback, record_property, - _settled_rss_per_worker(client.proxy), + _settled_rss_per_worker(proxy), 1, WARMUP_BATCHES, ) after: Final = _settled_rss_after_batches( - client.proxy, + proxy, scoped_key, primary, override, diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py index 63eefedec68..11b2cd1051d 100644 --- a/tests/e2e/perf/test_ui_journeys.py +++ b/tests/e2e/perf/test_ui_journeys.py @@ -33,7 +33,6 @@ from transport import HttpTransport pytestmark = [ pytest.mark.perf, - pytest.mark.quiet_stack, pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"), ] diff --git a/tests/e2e/provider_cache_redis.py b/tests/e2e/provider_cache_redis.py index 2c7419cfc0f..c1a3eada4a9 100644 --- a/tests/e2e/provider_cache_redis.py +++ b/tests/e2e/provider_cache_redis.py @@ -14,8 +14,8 @@ from typing import Final, Protocol, cast from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease from pydantic import TypeAdapter, ValidationError -from redis import Redis -from redis.exceptions import RedisError +from redis import Redis, RedisCluster +from redis.exceptions import RedisClusterException, RedisError REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes]) @@ -83,7 +83,7 @@ class RedisResponseStore: started: Final = time.monotonic() try: result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms) - except (RedisError, OSError): + except (RedisError, RedisClusterException, OSError): return CacheUnavailable() try: parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True)) @@ -102,28 +102,46 @@ class RedisResponseStore: result: Final = self.client.eval( PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms, ) - except (RedisError, OSError): + except (RedisError, RedisClusterException, OSError): return False return result == 1 def release(self, key: str, lease: CaptureLease) -> bool: try: result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token) - except (RedisError, OSError): + except (RedisError, RedisClusterException, OSError): return False return result == 1 def discard(self, key: str, payload: bytes) -> bool: try: result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload) - except (RedisError, OSError): + except (RedisError, RedisClusterException, OSError): return False return result == 1 +@functools.lru_cache(maxsize=16) +def _redis_commands(url: str, cluster: bool) -> RedisCommands: + if cluster: + return cast( + RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False) + ) + return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)) + + +@dataclass(frozen=True, slots=True) +class LazyRedisCommands: + url: str + cluster: bool + + def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object: + return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args) + + def redis_store(url: str, namespace: str) -> RedisResponseStore: - client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False) - return RedisResponseStore(cast(RedisCommands, client), namespace) + client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1") + return RedisResponseStore(client, namespace) def write_metrics(cache: CacheEdge) -> None: diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 8cc7bf0c2b4..9b842a723d8 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -11,7 +11,7 @@ from __future__ import annotations import os import time import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime from functools import reduce @@ -155,6 +155,21 @@ class NotServableOn: last_result: Result[ModelsListResponse] | None +@dataclass(frozen=True, slots=True) +class ModelRegistered: + model_name: str + model_id: str + + +@dataclass(frozen=True, slots=True) +class ModelNotRegistered: + model_name: str + reason: str + + +ModelRegistration = ModelRegistered | ModelNotRegistered + + def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], *, @@ -461,6 +476,7 @@ class ProxyClient: model_servable_db_sync_seconds: float = MODEL_SERVABLE_DB_SYNC_SECONDS model_servable_interval: float = MODEL_SERVABLE_INTERVAL model_servable_request_timeout: float = MODEL_SERVABLE_REQUEST_TIMEOUT + settle: Callable[[float], None] = settle_propagation def with_caller(self, caller: Caller) -> ProxyClient: return replace(self, caller=caller) @@ -694,31 +710,59 @@ class ProxyClient: balancer address is configured (every request opens a fresh connection, so the caller's next request re-rolls), so waiting out PROPAGATION_TIMEOUT is what makes the model safe to use anywhere.""" - model_id = unwrap( - self.transport.post( - "/model/new", - headers=self.management_headers(), - json=body.model_copy(update={"litellm_params": route_cache_model( - body.litellm_params, provider_edge_base, - enabled=os.environ.get("E2E_PROVIDER_CACHE", "0") == "1" and not provider_live, - mode=body.model_info.mode, - )}), - response_type=ModelNewResponse, - ) - ).model_id + model_id = unwrap(self._post_model(body, provider_live=provider_live)).model_id written_at = time.monotonic() try: self._await_model_servable(body.model_name, listed_for) except BaseException: self.delete_model(model_id) raise - settle_propagation(written_at) + self.settle(written_at) return model_id + @step("Add a batch of deployments, then wait once for all of them to propagate") + def register_models( + self, bodies: Sequence[ModelNewBody], *, provider_live: bool = False + ) -> tuple[ModelRegistration, ...]: + posted: Final = tuple((body.model_name, self._post_model(body, provider_live=provider_live)) for body in bodies) + written_at: Final = time.monotonic() + registrations: Final = tuple(self._servable_registration(name, result) for name, result in posted) + self.settle(written_at) + return registrations + + def _post_model(self, body: ModelNewBody, *, provider_live: bool) -> Result[ModelNewResponse]: + return self.transport.post( + "/model/new", + headers=self.management_headers(), + json=body.model_copy(update={"litellm_params": route_cache_model( + body.litellm_params, provider_edge_base, + enabled=os.environ.get("E2E_PROVIDER_CACHE", "0") == "1" and not provider_live, + mode=body.model_info.mode, + )}), + response_type=ModelNewResponse, + ) + + def _servable_registration(self, model_name: str, posted: Result[ModelNewResponse]) -> ModelRegistration: + if not isinstance(posted, Success): + return ModelNotRegistered(model_name=model_name, reason=f"/model/new failed: {posted!r}") + match self._servable_outcome(model_name, None): + case Servable(): + return ModelRegistered(model_name=model_name, model_id=posted.data.model_id) + case NotServableOn() as missing: + self.delete_model(posted.data.model_id) + return ModelNotRegistered(model_name=model_name, reason=self._not_servable_message(model_name, missing)) + def _await_model_servable(self, model_name: str, listed_for: str | None = None) -> None: """Block until every replica lists `model_name`, or fail at model_servable_timeout.""" + match self._servable_outcome(model_name, listed_for): + case Servable(): + return + case NotServableOn() as missing: + raise AssertionError(self._not_servable_message(model_name, missing)) + + def _servable_outcome(self, model_name: str, listed_for: str | None) -> Servable | NotServableOn: headers: Final = self.management_headers(listed_for) - outcome: Final = await_servable_everywhere( + return await_servable_everywhere( {url: self._models_poller(transport, headers) for url, transport in self.replicas.items()}, model_name=model_name, timeout=self.model_servable_timeout, @@ -728,19 +772,15 @@ class ProxyClient: now=time.monotonic, sleep=time.sleep, ) - match outcome: - case Servable(): - return - case NotServableOn(replica=replica, last_result=last_result): - raise AssertionError( - servable_timeout_message( - model_name=model_name, - replica=replica, - timeout=self.model_servable_timeout, - db_sync_seconds=self.model_servable_db_sync_seconds, - last_result=last_result, - ) - ) + + def _not_servable_message(self, model_name: str, missing: NotServableOn) -> str: + return servable_timeout_message( + model_name=model_name, + replica=missing.replica, + timeout=self.model_servable_timeout, + db_sync_seconds=self.model_servable_db_sync_seconds, + last_result=missing.last_result, + ) @staticmethod def _models_poller(transport: Transport, headers: AuthHeaders) -> ModelsPoller: diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index f15cfd4e98b..891c5fc282c 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -13,7 +13,6 @@ markers = prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_PERF_TESTS is set - quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set owned_gateway: boots its own proxy from source against the stack's Postgres, so it needs DATABASE_URL on the pytest host; deselected unless E2E_OWNED_GATEWAY is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set @@ -21,3 +20,4 @@ markers = secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py) sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA_ENDPOINT names it sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it + proxy_wide_config: turns on a proxy-wide setting through /config/update, so it runs on a stack no other suite is driving; deselected unless E2E_PROXY_WIDE_CONFIG_STACK is set diff --git a/tests/e2e/quota_management/spend_tracking/test_pricing_config_spend_e2e.py b/tests/e2e/quota_management/spend_tracking/test_guardrail_cost_spend_e2e.py similarity index 52% rename from tests/e2e/quota_management/spend_tracking/test_pricing_config_spend_e2e.py rename to tests/e2e/quota_management/spend_tracking/test_guardrail_cost_spend_e2e.py index 8f3bed3f89b..a77ca91ad3a 100644 --- a/tests/e2e/quota_management/spend_tracking/test_pricing_config_spend_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_guardrail_cost_spend_e2e.py @@ -17,11 +17,6 @@ from models import ( ChatBody, ChatMessage, ChatResponse, - ConfigPatchResponse, - CostDiscountConfig, - CostDiscountConfigResponse, - CostMarginConfig, - CostMarginConfigResponse, GuardrailCreateBody, GuardrailCreateResponse, KeyDeleteBody, @@ -31,14 +26,11 @@ from models import ( ) from spend_e2e_client import SpendClient -pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] +pytestmark = pytest.mark.e2e BACKEND_MODEL: Final = "openai/gpt-6.1-sol" INPUT_RATE: Final = 0.00005 OUTPUT_RATE: Final = 0.0001 -DISCOUNT: Final = 0.25 -MARGIN_PERCENT: Final = 0.1 -MARGIN_FIXED: Final = 0.0005 def _register_bedrock_guardrail( @@ -92,52 +84,6 @@ def _guarded_chat(client: SpendClient, key: str, model: str, name: str) -> Strea ) -def _set_discount(client: SpendClient, values: dict[str, float]) -> float: - unwrap( - client.proxy.transport.patch( - "/config/cost_discount_config", - headers=client.proxy.transport.master, - json=CostDiscountConfig(values), - response_type=ConfigPatchResponse, - ) - ) - return time.monotonic() - - -def _set_margin(client: SpendClient, values: dict[str, float | dict[str, float]]) -> float: - unwrap( - client.proxy.transport.patch( - "/config/cost_margin_config", - headers=client.proxy.transport.master, - json=CostMarginConfig(values), - response_type=ConfigPatchResponse, - ) - ) - return time.monotonic() - - -def _get_discount(client: SpendClient) -> dict[str, float]: - return unwrap( - client.proxy.transport.get( - "/config/cost_discount_config", - headers=client.proxy.transport.master, - params=NoBody(), - response_type=CostDiscountConfigResponse, - ) - ).values - - -def _get_margin(client: SpendClient) -> dict[str, float | dict[str, float]]: - return unwrap( - client.proxy.transport.get( - "/config/cost_margin_config", - headers=client.proxy.transport.master, - params=NoBody(), - response_type=CostMarginConfigResponse, - ) - ).values - - def _register_model(client: SpendClient, resources: ResourceManager, prefix: str) -> str: model: Final = f"{prefix}-{unique_marker()}" model_id: Final = client.proxy.create_model( @@ -199,100 +145,7 @@ def scoped_key(client: SpendClient, strict_resources: ResourceManager) -> str: return key -@pytest.fixture -def restored_pricing_config(client: SpendClient) -> Iterator[None]: - discount: Final = _get_discount(client) - margin: Final = _get_margin(client) - try: - _set_discount(client, {}) - reset_at: Final = _set_margin(client, {}) - settle_propagation(reset_at) - yield - finally: - try: - _set_discount(client, discount) - finally: - restored_at: Final = _set_margin(client, margin) - settle_propagation(restored_at) - - -class TestPricingConfigSpend: - @pytest.mark.covers( - "quota_management.spend_tracking.discount_config.logs_cost", - exercised_on=["chat_completions"], - ) - @meta( - Subject( - domain=Domain.SPEND_BUDGETS, - route=Route.SPEND_REPORTING, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_configured_discount_reaches_persisted_spend_row( - self, - client: SpendClient, - strict_resources: ResourceManager, - scoped_key: str, - restored_pricing_config: None, - ) -> None: - discount_set_at: Final = _set_discount(client, {"openai": DISCOUNT}) - settle_propagation(discount_set_at) - model: Final = _register_model(client, strict_resources, "discount-priced") - chat: Final = unwrap(client.chat(scoped_key, model, f"reply with one word {unique_marker()}", max_tokens=16)) - assert chat.id and chat.usage - assert chat.usage.prompt_tokens is not None and chat.usage.prompt_tokens > 0 - assert chat.usage.completion_tokens is not None and chat.usage.completion_tokens >= 0 - - row: Final = poll_cost_row(client.proxy, chat.id) - assert row is not None - base_cost: Final = _base_cost(row, chat.usage.prompt_tokens, chat.usage.completion_tokens) - breakdown: Final = row.breakdown - assert breakdown.discount_percent is not None and approx_equal(breakdown.discount_percent, DISCOUNT) - assert breakdown.discount_amount is not None and approx_equal(breakdown.discount_amount, base_cost * DISCOUNT) - assert row.spend is not None and approx_equal(row.spend, base_cost * (1 - DISCOUNT)) - - @pytest.mark.covers( - "quota_management.spend_tracking.margin_config.logs_cost", - exercised_on=["chat_completions"], - ) - @meta( - Subject( - domain=Domain.SPEND_BUDGETS, - route=Route.SPEND_REPORTING, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_configured_margin_reaches_persisted_spend_row( - self, - client: SpendClient, - strict_resources: ResourceManager, - scoped_key: str, - restored_pricing_config: None, - ) -> None: - margin_set_at: Final = _set_margin( - client, {"openai": {"percentage": MARGIN_PERCENT, "fixed_amount": MARGIN_FIXED}} - ) - settle_propagation(margin_set_at) - model: Final = _register_model(client, strict_resources, "margin-priced") - chat: Final = unwrap(client.chat(scoped_key, model, f"reply with one word {unique_marker()}", max_tokens=16)) - assert chat.id and chat.usage - assert chat.usage.prompt_tokens is not None and chat.usage.prompt_tokens > 0 - assert chat.usage.completion_tokens is not None and chat.usage.completion_tokens >= 0 - - row: Final = poll_cost_row(client.proxy, chat.id) - assert row is not None - base_cost: Final = _base_cost(row, chat.usage.prompt_tokens, chat.usage.completion_tokens) - breakdown: Final = row.breakdown - expected_margin: Final = base_cost * MARGIN_PERCENT + MARGIN_FIXED - assert breakdown.margin_percent is not None and approx_equal(breakdown.margin_percent, MARGIN_PERCENT) - assert breakdown.margin_fixed_amount is not None and approx_equal(breakdown.margin_fixed_amount, MARGIN_FIXED) - assert breakdown.margin_total_amount is not None and approx_equal(breakdown.margin_total_amount, expected_margin) - assert row.spend is not None and approx_equal(row.spend, base_cost + expected_margin) - +class TestGuardrailCostSpend: @pytest.mark.covers( "quota_management.spend_tracking.guardrail_cost.logs_cost", exercised_on=["chat_completions"], @@ -311,7 +164,6 @@ class TestPricingConfigSpend: client: SpendClient, strict_resources: ResourceManager, scoped_key: str, - restored_pricing_config: None, ) -> None: identifier: Final = os.environ["BEDROCK_GUARDRAIL_IDENTIFIER"] version: Final = os.environ["BEDROCK_GUARDRAIL_VERSION"] diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index e0c19ea1b6b..1d0878d073a 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -55,7 +55,7 @@ from spend_e2e_client import ( pytestmark = pytest.mark.e2e -CHAT_MODEL: Final = "gemini-2.5-flash" +CHAT_MODEL: Final = "gemini-3.8-flash" MESSAGES_MODEL: Final = "claude-haiku-4-5" RESPONSES_MODEL: Final = "openai-responses-codex" EMBED_MODEL: Final = "openai-text-embedding-3-small" diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index c7ef81f826a..d452525ec91 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -3,7 +3,7 @@ Run against a proxy started with the gateway config. Coverage rationale: SPEND_TRACKING_COVERAGE_MATRIX.md. -Model names are literals from that config: chat tests hit "gemini-2.5-flash", +Model names are literals from that config: chat tests hit "gemini-3.8-flash", embedding tests hit "openai-text-embedding-3-small". Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard @@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND pytestmark = pytest.mark.e2e -GEMINI_MODEL = "gemini-2.5-flash" +GEMINI_MODEL = "gemini-3.8-flash" CLAUDE_MODEL = "claude-haiku-4-5" CODEX_MODEL = "openai-responses-codex" EMBEDDING_MODEL = "openai-text-embedding-3-small" @@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: GEMINI_MODEL, "estimate the cost of this request" ) assert cost > 0, ( - "/spend/calculate returned 0 for gemini-2.5-flash; " + "/spend/calculate returned 0 for gemini-3.8-flash; " "cost map may be missing this model" ) diff --git a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py index ab7c2370365..84a598a2bbb 100644 --- a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py @@ -30,7 +30,7 @@ from models import ( from proxy_client import ProxyClient from pydantic import BaseModel, ValidationError -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" SEARCH_CALL_TYPE: Final = "asearch" diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 03e585273c0..193a94cb581 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = ( COOLDOWN_SECONDS = 30.0 REPLICA_PROPAGATION_SECONDS = 15.0 -# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt -# past that limit comes back as a real `context_length_exceeded` 400, which is -# what litellm maps to ContextWindowExceededError. -SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo" -SMALL_CONTEXT_LIMIT_TOKENS = 16385 +SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano" +# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02 +SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000 def oversized_prompt(marker: str) -> str: @@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: @step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}") def create_small_context_deployment(proxy: ProxyClient, name: str) -> str: - """Register a deployment on the smallest-context model OpenAI still serves, so an + """Register a deployment with a smaller context window than the backup, so an oversized prompt earns a real context-window refusal from the provider.""" return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY)) @@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None: @step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic") def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str: - """The always-picked half of a retry pair on the smallest-context model OpenAI - still serves: it holds all of the model group's shuffle weight, so an oversized + """The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized prompt opens on it and earns a real context-window refusal, which never benches a deployment, so only the retry itself can steer the request off it.""" return proxy.register_model( diff --git a/tests/e2e/stack_lock.py b/tests/e2e/stack_lock.py deleted file mode 100644 index 06df7a20b6a..00000000000 --- a/tests/e2e/stack_lock.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Cross-process reader/writer lock over the proxy stack every xdist worker shares. -Every collected test holds it shared, marker or not, since the Claude Code cells and -other unmarked suites drive the same stack; a `quiet_stack` test holds it exclusive, -and the `gate` file makes a waiting exclusive holder win over readers that arrive -after it.""" - -from __future__ import annotations - -import fcntl -import hashlib -import tempfile -from collections.abc import Generator -from contextlib import ExitStack, contextmanager -from pathlib import Path -from typing import Final - -from e2e_config import PROXY_BASE_URL - -STACK_DIGEST: Final = hashlib.sha256(PROXY_BASE_URL.encode()).hexdigest()[:12] -LOCK_DIR: Final = Path(tempfile.gettempdir()) / f"litellm-e2e-stack-{STACK_DIGEST}" -GATE_FILE: Final = LOCK_DIR / "gate" -STACK_FILE: Final = LOCK_DIR / "stack" - - -@contextmanager -def _flock(path: Path, operation: int) -> Generator[None]: - with path.open("a") as handle: - fcntl.flock(handle, operation) - try: - yield - finally: - fcntl.flock(handle, fcntl.LOCK_UN) - - -@contextmanager -def stack_lock(exclusive: bool) -> Generator[None]: - LOCK_DIR.mkdir(parents=True, exist_ok=True) - if exclusive: - with _flock(GATE_FILE, fcntl.LOCK_EX), _flock(STACK_FILE, fcntl.LOCK_EX): - yield - return - with ExitStack() as held: - with _flock(GATE_FILE, fcntl.LOCK_SH): - held.enter_context(_flock(STACK_FILE, fcntl.LOCK_SH)) - yield diff --git a/tests/e2e/ui/helpers/navigation.ts b/tests/e2e/ui/helpers/navigation.ts index be46fcf0395..3943526aebf 100644 --- a/tests/e2e/ui/helpers/navigation.ts +++ b/tests/e2e/ui/helpers/navigation.ts @@ -37,7 +37,7 @@ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Prom // assert against the default page. for (let attempt = 0; attempt < 3; attempt++) { await page.goto(`/ui?page=${pageEnum}`); - await page.waitForLoadState("networkidle"); + await expect(page.getByRole("complementary")).toBeVisible(); const url = new URL(page.url()); const onLegacyRoot = url.pathname.replace(/\/+$/, "").endsWith("/ui"); if (!onLegacyRoot || url.searchParams.get("page") === pageEnum) { diff --git a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts index 04b02d9ccc4..517ec64c7df 100644 --- a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts +++ b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts @@ -1,8 +1,8 @@ -import { test, expect, type APIRequestContext } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -import { masterKey } from "../../helpers/traffic"; +import { masterKey, uniqueSuffix } from "../../helpers/traffic"; test.describe("AI Hub (internal admin view)", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -62,106 +62,93 @@ test.describe("AI Hub (internal admin view)", () => { }); test.describe("Public model hub (/ui/model_hub_table)", () => { - // No storageState — the public page is reached anonymously with a `key` query param. - - test("Public model_hub_table loads and renders the Model Hub tab", async ({ page }) => { - // The page expects the proxy key as the `key` query param. Use the master - // key the e2e runner already exports — this matches what the AI Hub copy - // button hands out. - await page.goto(`/ui/model_hub_table?key=${masterKey()}`); - - // Dismiss the feedback popup before asserting on the tab, so a popup - // race can't briefly mask the tab while we're evaluating visibility. + test("shows the Model Hub and Skills Hub catalogs", async ({ page }) => { + await page.goto("/ui/model_hub_table"); await dismissFeedbackPopup(page); - // Page loads (no auth redirect) and the Model Hub tab is always present. - // Agent Hub and MCP Hub tabs are conditionally rendered only when public - // agents/MCP servers exist, so we don't assert on them in a fresh CI run. - await expect(page.getByRole("tab", { name: "Model Hub" })).toBeVisible({ timeout: 10_000 }); + await expect( + page.getByRole("tab", { name: "Model Hub", exact: true }), + ).toBeVisible(); + await expect( + page.getByRole("textbox", { name: "Search models", exact: true }), + ).toBeVisible(); + await expect(page.getByRole("tab")).toHaveCount(2); + await page.getByRole("tab", { name: "Skills Hub", exact: true }).click(); + await expect( + page.getByRole("textbox", { name: "Search skills", exact: true }), + ).toBeVisible(); }); - test("Agent Hub and MCP Hub tabs render their public entries", async ({ page, request }) => { - const suffix = `${Date.now()}`; - const agentName = `e2e-public-agent-${suffix}`; - const mcpServerName = `e2e_public_mcp_${suffix}`; + test("finds a published skill and opens its installation instructions", async ({ + page, + request, + }) => { + const skillName = `e2e-catalog-${uniqueSuffix()}`; + const domain = `catalog-${uniqueSuffix()}`; const auth = { Authorization: `Bearer ${masterKey()}` }; - - const publicMcpServerIds = async (api: APIRequestContext): Promise => { - const res = await api.get("/public/mcp_hub"); - expect(res.ok(), `public mcp_hub read failed (${res.status()}): ${await res.text()}`).toBe(true); - const servers: { server_id: string }[] = await res.json(); - return servers.map((server) => server.server_id); - }; - - const seedPublicEntries = async ( - api: APIRequestContext, - priorMcpIds: string[], - ): Promise<{ agentId: string; serverId: string }> => { - const agentRes = await api.post("/v1/agents", { - headers: auth, - data: { - agent_name: agentName, - agent_card_params: { - name: agentName, - description: "E2E public agent", - version: "1.0.0", - url: "http://127.0.0.1:9999/", - capabilities: {}, - skills: [], - defaultInputModes: ["text"], - defaultOutputModes: ["text"], - }, - }, - }); - expect(agentRes.ok(), `agent create failed (${agentRes.status()}): ${await agentRes.text()}`).toBe(true); - const agentId = (await agentRes.json()).agent_id as string; - - const serverRes = await api.post("/v1/mcp/server", { - headers: auth, - data: { - server_name: mcpServerName, - url: "http://127.0.0.1:9999/mcp", - transport: "http", - description: "E2E public MCP server", - }, - }); - expect(serverRes.ok(), `mcp server create failed (${serverRes.status()}): ${await serverRes.text()}`).toBe(true); - const serverId = (await serverRes.json()).server_id as string; - - const agentPublicRes = await api.post(`/v1/agents/${agentId}/make_public`, { headers: auth }); - expect(agentPublicRes.ok(), `agent make_public failed: ${await agentPublicRes.text()}`).toBe(true); - const mcpPublicRes = await api.post("/v1/mcp/make_public", { - headers: auth, - data: { mcp_server_ids: [...priorMcpIds, serverId] }, - }); - expect(mcpPublicRes.ok(), `mcp make_public failed: ${await mcpPublicRes.text()}`).toBe(true); - - return { agentId, serverId }; - }; - - const priorMcpIds = await publicMcpServerIds(request); - const { agentId, serverId } = await seedPublicEntries(request, priorMcpIds); + const created = await request.post("/claude-code/plugins", { + headers: auth, + data: { + name: skillName, + source: { source: "github", repo: "example/catalog-skills" }, + description: "Catalog browser test skill", + domain, + version: "1.0.0", + }, + }); + expect( + created.ok(), + `skill registration failed (${created.status()}): ${await created.text()}`, + ).toBe(true); try { - await page.goto(`/ui/model_hub_table?key=${masterKey()}`); + await page.goto("/ui/model_hub_table?tab=skills"); await dismissFeedbackPopup(page); + await page.getByRole("checkbox", { name: domain, exact: true }).check(); + await page + .getByRole("textbox", { name: "Search skills", exact: true }) + .fill(skillName); + await page.getByRole("link").filter({ hasText: skillName }).click(); - const agentHubTab = page.getByRole("tab", { name: "Agent Hub" }); - await expect(agentHubTab).toBeVisible({ timeout: 15_000 }); - await agentHubTab.click(); - await expect(page.getByText("Available Agents")).toBeVisible({ timeout: 10_000 }); - await expect(page.getByRole("row").filter({ hasText: agentName })).toHaveCount(1, { timeout: 10_000 }); - await expect(page.getByText("E2E public agent").first()).toBeVisible(); - - const mcpHubTab = page.getByRole("tab", { name: "MCP Hub" }); - await expect(mcpHubTab).toBeVisible(); - await mcpHubTab.click(); - await expect(page.getByText("Available MCP Servers")).toBeVisible({ timeout: 10_000 }); - await expect(page.getByRole("row").filter({ hasText: mcpServerName })).toHaveCount(1, { timeout: 10_000 }); - await expect(page.getByText("E2E public MCP server").first()).toBeVisible(); + const details = page.getByRole("dialog"); + await expect( + details.getByRole("heading", { name: skillName, exact: true }), + ).toBeVisible(); + await expect( + details.getByText(`/plugin install ${skillName}@litellm`, { + exact: true, + }), + ).toBeVisible(); + await expect(page).toHaveURL(new RegExp(`open=${skillName}`)); + await page.reload(); + await expect( + page + .getByRole("dialog") + .getByRole("heading", { name: skillName, exact: true }), + ).toBeVisible(); } finally { - await request.post("/v1/mcp/make_public", { headers: auth, data: { mcp_server_ids: priorMcpIds } }); - await request.delete(`/v1/agents/${agentId}`, { headers: auth }); - await request.delete(`/v1/mcp/server/${serverId}`, { headers: auth }); + const deleted = await request.delete( + `/claude-code/plugins/${skillName}`, + { headers: auth }, + ); + expect(deleted.ok(), `skill cleanup failed (${deleted.status()})`).toBe( + true, + ); + } + }); + + test("retired catalog links return to Models and clear the old selection", async ({ + page, + }) => { + for (const tab of ["agents", "mcp"]) { + await page.goto( + `/ui/model_hub_table?tab=${tab}&open=retired-entry&key=preserved`, + ); + await dismissFeedbackPopup(page); + await expect( + page.getByRole("tab", { name: "Model Hub", exact: true }), + ).toHaveAttribute("aria-selected", "true"); + await expect(page).toHaveURL(/model_hub_table\?key=preserved$/); + await expect(page.getByRole("dialog")).toHaveCount(0); } }); }); diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index a99fb937b83..b5e876b36d2 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -391,7 +391,7 @@ test.describe("Add Model", () => { // Navigate to Deployed Models tab await page.getByRole("tab", { name: "Deployed Models" }).click(); - await page.waitForLoadState("networkidle"); + await expect(page.getByPlaceholder("Search model names")).toBeVisible(); // Search for the model we just added await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5"); @@ -470,7 +470,7 @@ test.describe("Add Model", () => { // The Models table renders team-scoped models with the team id in the row. await page.getByRole("tab", { name: "Deployed Models" }).click(); - await page.waitForLoadState("networkidle"); + await expect(page.getByPlaceholder("Search model names")).toBeVisible(); await page.getByPlaceholder("Search model names").fill("cohere"); @@ -515,7 +515,7 @@ test.describe("Add Model", () => { // Navigate to Deployed Models tab await page.getByRole("tab", { name: "Deployed Models" }).click(); - await page.waitForLoadState("networkidle"); + await expect(page.getByPlaceholder("Search model names")).toBeVisible(); // Search for the wildcard model await page.getByPlaceholder("Search model names").fill("cohere"); diff --git a/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts index 4480515ae59..0831c003891 100644 --- a/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts @@ -27,7 +27,7 @@ async function readStoredParams( } function paramsEditor(page: PlaywrightPage) { - return page.getByPlaceholder('"rpm": 100'); + return page.getByRole("textbox", { name: "LiteLLM Params", exact: true }); } async function editParams( diff --git a/tests/e2e/ui/tests/navigation/commandPalette.spec.ts b/tests/e2e/ui/tests/navigation/commandPalette.spec.ts new file mode 100644 index 00000000000..5e80afb8b18 --- /dev/null +++ b/tests/e2e/ui/tests/navigation/commandPalette.spec.ts @@ -0,0 +1,54 @@ +import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { dismissFeedbackPopup, navigateToPage } from "../../helpers/navigation"; +import { createVirtualKey, deleteVirtualKey, uniqueSuffix } from "../../helpers/traffic"; + +const paletteSearch = (page: PlaywrightPage) => page.getByRole("combobox", { name: "Search" }); +const headerTrigger = (page: PlaywrightPage) => page.getByRole("button", { name: "Search keys…", exact: true }); + +test.describe("Command palette", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test.beforeEach(async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await dismissFeedbackPopup(page); + await expect(headerTrigger(page)).toBeVisible(); + }); + + test("opens a key found by its alias", async ({ page }) => { + const alias = `e2e-palette-key-${uniqueSuffix()}`; + const created = await createVirtualKey(page.request, { key_alias: alias }); + try { + await page.keyboard.press("ControlOrMeta+K"); + await paletteSearch(page).fill(alias); + await expect(page.getByRole("option", { name: new RegExp(`^${alias}`) })).toBeVisible(); + await page.keyboard.press("Enter"); + + await expect(page).toHaveURL(new RegExp(`[?&]key=${created.token}`)); + await expect(paletteSearch(page)).toBeHidden(); + } finally { + await deleteVirtualKey(page.request, created.token); + } + }); + + test("does not open over the Create New Key dialog", async ({ page }) => { + await page.getByRole("button", { name: /Create New Key/i }).click(); + const keyName = page.getByLabel(/Key Name/); + await keyName.click(); + + await page.keyboard.press("ControlOrMeta+K"); + await page.keyboard.type("still-in-the-dialog"); + + await expect(keyName).toHaveValue("still-in-the-dialog"); + await expect(paletteSearch(page)).toBeHidden(); + }); + + test("opens from the header trigger and closes with Escape", async ({ page }) => { + await headerTrigger(page).click(); + await expect(paletteSearch(page)).toBeFocused(); + + await page.keyboard.press("Escape"); + await expect(paletteSearch(page)).toBeHidden(); + }); +}); diff --git a/tests/e2e_harness/AGENTS.md b/tests/e2e_harness/AGENTS.md index 83e573ae249..4f507c2fe7b 100644 --- a/tests/e2e_harness/AGENTS.md +++ b/tests/e2e_harness/AGENTS.md @@ -1,6 +1,6 @@ # e2e harness tests -Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network +Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder and probe, `cli_harness/` covers the version resolver, the CLI env lockdown and the rate limiter, and `codex/` covers how the Codex driver reads the CLI's JSONL events and the compactions in a thread's rollout file. Put a new harness test under the folder that mirrors the suite folder whose module it covers diff --git a/tests/e2e_harness/logging/test_logging_client.py b/tests/e2e_harness/logging/test_logging_client.py new file mode 100644 index 00000000000..c396a6c68e5 --- /dev/null +++ b/tests/e2e_harness/logging/test_logging_client.py @@ -0,0 +1,49 @@ +import json +from typing import Final + +import pytest +from pydantic import JsonValue + +from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend + + +@pytest.mark.parametrize("encoded", [False, True]) +@pytest.mark.parametrize("cost", [0.0, 0.000023]) +def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None: + content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}] + output: Final[JsonValue] = [{"role": "assistant", "content": "response"}] + metadata: Final[JsonValue] = { + "attributes": { + "langfuse.observation.type": "generation", + "litellm.metadata.user_api_key_alias": "test-key-alias", + } + } + usage: Final[JsonValue] = {"input": 17, "output": 3} + observation: Final = LangfuseObservation.model_validate( + { + "id": "observation", + "input": json.dumps(content) if encoded else content, + "output": json.dumps(output) if encoded else output, + "metadata": json.dumps(metadata) if encoded else metadata, + "usageDetails": usage, + "totalCost": cost, + } + ) + + assert observation.input == content + assert observation.output == output + assert observation.usage_details == usage + assert observation_spend(observation) == cost + assert is_otel_v2_generation(observation, key_alias="test-key-alias") + assert not is_otel_v2_generation(observation, key_alias="another-key") + + +def test_observation_preserves_plain_text_and_legacy_cost() -> None: + cost: Final = 0.000031 + observation: Final = LangfuseObservation.model_validate( + {"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost} + ) + + assert observation.input == "plain prompt" + assert observation.output == "plain reply" + assert observation_spend(observation) == cost diff --git a/tests/e2e_harness/logging/test_span_selection.py b/tests/e2e_harness/logging/test_span_selection.py index e7487562e43..76f70e4d8bb 100644 --- a/tests/e2e_harness/logging/test_span_selection.py +++ b/tests/e2e_harness/logging/test_span_selection.py @@ -8,8 +8,21 @@ helper is worth pinning here. from __future__ import annotations +from typing import Final + import pytest -from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans +from otel_client import ( + TTFT_TAG, + CallTraces, + JaegerReference, + JaegerSpan, + JaegerTag, + JaegerTrace, + one_served_genai_span, + served_genai_spans, + trace_settled, +) +from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft GENAI_SPAN = "chat claude-haiku-4-5" @@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None: trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3)) assert len(served_genai_spans(trace, GENAI_SPAN)) == 1 + + +def _request_trace( + *, + root_name: str = "POST", + root_kind: str = "server", + route: str = "/v1/messages", + genai_parent: str = "root", + ttft: float = 0.2, +) -> JaegerTrace: + parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root") + return JaegerTrace( + traceID="request", + spans=[ + JaegerSpan( + spanID="root", + operationName=root_name, + tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)], + ), + JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]), + JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]), + JaegerSpan( + spanID="genai", + operationName=GENAI_SPAN, + duration=1_000_000, + references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)], + tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)], + ), + ], + ) + + +@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages")) +def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None: + trace: Final = _request_trace(root_name=root_name) + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + assert_real_ttft((trace,), genai_span=GENAI_SPAN) + assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None: + trace: Final = _request_trace() + incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]}) + assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +@pytest.mark.parametrize("route", ("/v1/responses", "")) +def test_request_trace_must_still_identify_the_requested_route(route: str) -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", route=route) + with pytest.raises(AssertionError, match="route"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_be_rooted_at_a_server_span() -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client") + with pytest.raises(AssertionError, match="kind=server"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None: + trace: Final = _request_trace(genai_parent="absent") + with pytest.raises(AssertionError, match="orphaned trace"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +@pytest.mark.parametrize("ttft", (0.0, 1.0)) +def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None: + trace: Final = _request_trace(ttft=ttft) + with pytest.raises(AssertionError, match="TTFT"): + assert_real_ttft((trace,), genai_span=GENAI_SPAN) diff --git a/tests/e2e_harness/perf/test_run.py b/tests/e2e_harness/perf/test_run.py index 23f62c9c560..480af96a12a 100644 --- a/tests/e2e_harness/perf/test_run.py +++ b/tests/e2e_harness/perf/test_run.py @@ -8,7 +8,29 @@ from typing import Final import pytest from setup.run import run_until from setup.wire import WireRecord -from test_upgrade import record + + +def record(seq: int = 0, *, started_at: float = 100) -> WireRecord: + return WireRecord( + seq=seq, + marker=f"request-{seq}", + caller="key", + endpoint="chat_completions", + stream=False, + blocked=False, + started_at=started_at, + status=200, + header_s=1, + first_byte_s=1, + first_token_s=None, + total_s=1, + text="answer", + ok=True, + mock_delay_s=1, + call_id=f"call-{seq}", + error=None, + response_id=f"response-{seq}", + ) class StopAfterRecord: diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py deleted file mode 100644 index 3414476ba4c..00000000000 --- a/tests/e2e_harness/perf/test_upgrade.py +++ /dev/null @@ -1,375 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import Final - -import pytest -from cases.small_bodies_5rps import CASE -from memory_readings import RssCapture, RssReading -from setup.observe import Observation, evaluate -from setup.upgrade import ( - MAX_TRAFFIC_SECONDS, - UpgradeMonitor, - matched_response_ids, - postgres_failures, - upgrade_case, - upgrade_properties, -) -from setup.upgrade_worker import UpgradeState -from setup.wire import PlanProgress, WireRecord, response_id - - -def record(seq: int = 0, *, started_at: float = 100) -> WireRecord: - return WireRecord( - seq=seq, - marker=f"request-{seq}", - caller="key", - endpoint="chat_completions", - stream=False, - blocked=False, - started_at=started_at, - status=200, - header_s=1, - first_byte_s=1, - first_token_s=None, - total_s=1, - text="answer", - ok=True, - mock_delay_s=1, - call_id=f"call-{seq}", - error=None, - response_id=f"response-{seq}", - ) - - -def test_spend_logs_match_call_ids_when_response_ids_are_transformed() -> None: - records: Final = (record(0), record(1), record(2)) - rows: Final = (("provider-response-id", "call-0"), ("response-1", "unrelated-call")) - assert matched_response_ids(records, rows) == frozenset({"response-0"}) - - -def state(phase: str = "schema_ready", *, pending: bool = True) -> UpgradeState: - return UpgradeState.model_validate( - { - "phase": phase, - "started_at": 0, - "required_indexes": ["index"], - "required_migrations": ["migration"], - "pending_indexes": ["index"] if pending else [], - "baseline_rows": {"seed-row": "hash"}, - } - ) - - -class Probe: - def __init__(self) -> None: - self.now = 0.0 - self.current = state() - self.builds: list[int] = [] - self.found: frozenset[str] = frozenset() - self.releases = 0 - self.metric_checks = 0 - self.errors = False - - def check_metrics(self) -> None: - self.metric_checks += 1 - assert not self.errors, "Failed PostgreSQL operation" - - def persisted(self, records: tuple[WireRecord, ...]) -> frozenset[str]: - return frozenset(row.response_id for row in records if row.response_id in self.found and row.response_id) - - def release(self) -> None: - self.releases += 1 - - def monitor(self) -> UpgradeMonitor: - return UpgradeMonitor( - state=lambda: self.current, - check_metrics=self.check_metrics, - read_activity=lambda: (self.builds, 2), - read_persisted=self.persisted, - start_indexes=self.release, - clock=lambda: self.now, - wall_clock=lambda: self.now, - ) - - -def test_stop_drains_issued_requests_without_issuing_the_rest() -> None: - progress: Final = PlanProgress(100) - assert progress.take(stop=False) - assert progress.take(stop=False) - progress.completed = 1 - assert not progress.take(stop=True) - assert not progress.drained - assert not progress.take(stop=False) - progress.completed = 2 - assert progress.drained - assert progress.issued == 2 - - -def test_finite_plan_still_waits_for_its_last_response() -> None: - progress: Final = PlanProgress(1) - assert progress.take(stop=False) - assert not progress.take(stop=False) - assert not progress.drained - progress.completed = 1 - assert progress.drained - - -@pytest.mark.parametrize( - "payload,expected", - [ - ({"id": "chat-id"}, "chat-id"), - ({"type": "response.created", "response": {"id": "response-id"}}, "response-id"), - ({"type": "message_start", "message": {"id": "message-id"}}, "message-id"), - ({"type": "response.output_item.added", "item": {"id": "item-id"}}, None), - ({"type": "content_block_delta", "delta": {"text": "hello"}}, None), - ], -) -def test_response_id_uses_response_not_output_item(payload: Mapping[str, object], expected: str | None) -> None: - assert response_id(payload) == expected - - -def metrics(*, failure: str = "0", batch: str = "0") -> str: - return f"""# TYPE litellm_postgres_failed_requests_total counter -litellm_postgres_failed_requests_total{{function_name="query",error_class="Timeout"}} {failure} -# TYPE litellm_batch_write_to_db_failed_requests_total counter -litellm_batch_write_to_db_failed_requests_total{{function_name="flush",error_class="DatabaseError"}} {batch} -""" - - -def test_failed_sql_and_async_batch_writes_are_both_counted() -> None: - assert postgres_failures(metrics(failure="2", batch="3")) == 5 - assert postgres_failures(metrics()) == 0 - - -def test_no_error_label_series_is_a_valid_zero() -> None: - assert ( - postgres_failures("""# TYPE litellm_postgres_failed_requests_total counter -# TYPE litellm_batch_write_to_db_failed_requests_total counter -""") - == 0 - ) - - -@pytest.mark.parametrize("service", [None, "postgres", "batch_write_to_db"]) -def test_multiprocess_metrics_omit_failure_families_until_a_failure(service: str | None) -> None: - total: Final = """# TYPE litellm_postgres_total_requests_total counter -litellm_postgres_total_requests_total{postgres="postgres"} 1 -""" - failure: Final = ( - f"# TYPE litellm_{service}_failed_requests_total counter\n" - f'litellm_{service}_failed_requests_total{{function_name="query",error_class="Timeout"}} 1\n' - if service is not None - else "" - ) - assert postgres_failures(total + failure) == (1 if service is not None else 0) - - -@pytest.mark.parametrize("total", ["0", "-1", "NaN", "+Inf"]) -def test_missing_failure_families_require_observed_database_activity(total: str) -> None: - with pytest.raises(AssertionError): - _ = postgres_failures( - "# TYPE litellm_postgres_total_requests_total counter\n" - f'litellm_postgres_total_requests_total{{postgres="postgres"}} {total}\n' - ) - - -@pytest.mark.parametrize("body", ["", "# TYPE unrelated counter\n", metrics(failure="NaN"), metrics(batch="-1")]) -def test_missing_or_invalid_counters_do_not_pass_as_zero(body: str) -> None: - with pytest.raises(AssertionError): - _ = postgres_failures(body) - - -def test_index_completion_stops_after_the_post_upgrade_window() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - assert not monitor.poll((record(),)) - assert probe.releases == 1 - probe.builds = [17] - probe.now = 101 - assert not monitor.poll(()) - probe.found = frozenset({"response-0", "response-1"}) - probe.now = 103 - assert not monitor.poll((record(1, started_at=102),)) - assert monitor.overlap_writes == 1 - assert monitor.persisted_count == 2 - assert not monitor.pending - assert monitor.max_lock_waiters == 2 - probe.current = state("complete") - assert not monitor.poll(()) - probe.now = 162 - assert not monitor.poll(()) - probe.now = 163 - assert not monitor.poll(()) - probe.found = frozenset(f"response-{i}" for i in range(300)) - assert monitor.poll(tuple(record(i) for i in range(2, 300))) - assert probe.releases == 1 - assert probe.metric_checks == 7 - - -def test_http_200_without_spend_row_cannot_complete() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - records: Final = tuple(record(i) for i in range(300)) - _ = monitor.poll(records) - with pytest.raises(AssertionError, match="no matching spend log"): - monitor.assert_complete(state("complete", pending=False)) - - -def test_no_pending_index_work_still_requires_300_persisted_requests() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - records: Final = tuple(record(i) for i in range(300)) - probe.found = frozenset(f"response-{i}" for i in range(300)) - _ = monitor.poll(records) - monitor.assert_complete(state("complete", pending=False)) - assert monitor.successes == monitor.persisted_count == 300 - with pytest.raises(AssertionError, match="during index work"): - monitor.assert_complete(state("complete", pending=True)) - - -def test_postgres_failure_fails_even_when_inference_succeeded() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - probe.errors = True - with pytest.raises(AssertionError, match="PostgreSQL"): - _ = monitor.poll((record(),)) - - -def test_failed_migration_and_operational_timeout_are_not_success() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - probe.current = state("failed") - with pytest.raises(AssertionError, match="worker failed"): - _ = monitor.poll(()) - probe.current = state() - probe.now = MAX_TRAFFIC_SECONDS - with pytest.raises(AssertionError, match="operational timeout"): - _ = monitor.poll(()) - - -def test_upgrade_case_uses_5rps_with_the_same_request_and_memory_limits() -> None: - case: Final = upgrade_case() - assert case.traffic.rps == 5 - assert not {"cpu_seconds_per_request", "postgres_calls_per_request", "redis_calls_per_request"}.intersection( - limit.metric for limit in case.limits - ) - assert case.limits == tuple( - limit - for limit in CASE.limits - if limit.metric - in { - "unexpected_status_count", - "gateway_added_seconds", - "time_to_first_token_seconds", - "memory_growth_megabytes", - } - ) - - -def failed_record(seq: int) -> WireRecord: - return record(seq).model_copy(update={"status": 503, "ok": False, "text": "", "response_id": None, "call_id": None}) - - -def test_three_non_200s_do_not_start_indexes_or_require_spend_rows() -> None: - probe: Final = Probe() - monitor: Final = probe.monitor() - assert not monitor.poll(tuple(failed_record(i) for i in range(3))) - assert probe.releases == 0 - assert monitor.processed == 3 - assert monitor.successes == 0 - assert not monitor.pending - probe.found = frozenset(f"response-{i}" for i in range(3, 303)) - assert not monitor.poll(tuple(record(i) for i in range(3, 303))) - monitor.assert_complete(state("complete", pending=False)) - assert probe.releases == 1 - assert monitor.processed == 303 - assert monitor.successes == monitor.persisted_count == 300 - - -def test_fourth_non_200_fails_across_poll_batches() -> None: - monitor: Final = Probe().monitor() - assert not monitor.poll(tuple(failed_record(i) for i in range(3))) - with pytest.raises(AssertionError, match="Too many unexpected non-200"): - _ = monitor.poll((failed_record(3),)) - - -@pytest.mark.parametrize( - "failed", [record().model_copy(update={"text": ""}), record().model_copy(update={"ok": False})] -) -def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None: - with pytest.raises(AssertionError, match="Failed or empty inference"): - _ = Probe().monitor().poll((failed,)) - - -def _ceiling(metric: str, percentile: int | None = None) -> float: - return next( - limit.max - for limit in upgrade_case().limits - if limit.metric == metric and limit.percentile == percentile and limit.max is not None - ) - - -@pytest.mark.parametrize( - "over,expected", - [ - ("", ()), - ("unexpected_status_count", ("unexpected_status_count",)), - ("gateway_added_seconds", ("gateway_added_seconds percentile 50",)), - ("time_to_first_token_seconds", ("time_to_first_token_seconds percentile 99",)), - ("memory_growth_megabytes", ("memory_growth_megabytes",)), - ], -) -def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples( - over: str, expected: tuple[str, ...] -) -> None: - """Each sample sits exactly on the case's ceiling, so only the one nudged past it goes over.""" - failures: Final = int(_ceiling("unexpected_status_count")) + (over == "unexpected_status_count") - overhead: Final = _ceiling("gateway_added_seconds", 50) * (1.01 if over == "gateway_added_seconds" else 1) - ttft: Final = _ceiling("time_to_first_token_seconds", 99) * (1.01 if over == "time_to_first_token_seconds" else 1) - growth: Final = _ceiling("memory_growth_megabytes") + (over == "memory_growth_megabytes") - nonstream: Final = tuple(record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100)) - stream: Final = record(100).model_copy(update={"stream": True, "first_token_s": ttft}) - observed: Final = Observation( - records=(*nonstream, stream, *(failed_record(i) for i in range(101, 101 + failures))), - mock_requests=(), - guardrail_calls=(), - rss_before=RssCapture((RssReading("gateway", "host", 1, 1000),), ()), - rss_after=RssCapture((RssReading("gateway", "host", 1, 1000 + growth),), ()), - wall_seconds=60, - ) - rows, exceeded = evaluate(upgrade_case(), observed) - assert tuple(row.label for row in rows if row.over or row.missing) == expected - assert len(exceeded) == len(expected) - - -def test_upgrade_properties_carry_the_upgrade_timings_and_what_the_monitor_saw() -> None: - state: Final = UpgradeState( - phase="complete", - started_at=0, - schema_seconds=12.5, - index_seconds=240.25, - required_indexes=("idx_a", "idx_b"), - required_migrations=("20260101_a",), - pending_indexes=("idx_b",), - baseline_rows={}, - ) - monitor: Final = UpgradeMonitor( - state=lambda: state, - check_metrics=lambda: None, - read_activity=lambda: ([], 0), - read_persisted=lambda records: frozenset(), - start_indexes=lambda: None, - ) - monitor.overlap_writes = 37 - monitor.max_lock_waiters = 3 - monitor.pending = (record(0), record(1)) - - assert dict(upgrade_properties(state, monitor)) == { - "perf_upgrade_schema_seconds": 12.5, - "perf_upgrade_index_seconds": 240.25, - "perf_upgrade_overlap_writes": 37, - "perf_upgrade_max_lock_waiters": 3, - "perf_upgrade_pending_indexes": 1, - "perf_upgrade_unmatched_requests": 2, - } diff --git a/tests/e2e_harness/pytest.ini b/tests/e2e_harness/pytest.ini index a29166d31bd..9ccf8ba83a0 100644 --- a/tests/e2e_harness/pytest.ini +++ b/tests/e2e_harness/pytest.ini @@ -5,5 +5,8 @@ addopts = --strict-markers --strict-config -p no:cacheprovider pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf markers = + e2e: live assertion modules imported by harness regression tests + meta(subject): metadata on live assertion modules imported by harness regression tests + otel_tls: TLS coverage in live assertion modules imported by harness regression tests covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it cli_determinism: drives the real claude CLI for several seconds diff --git a/tests/e2e_harness/test_proxy_client.py b/tests/e2e_harness/test_proxy_client.py index e1615ec5f65..d352e5c8d36 100644 --- a/tests/e2e_harness/test_proxy_client.py +++ b/tests/e2e_harness/test_proxy_client.py @@ -12,10 +12,11 @@ injected clock, so nothing here monkeypatches anything. from __future__ import annotations import json +import time from builtins import ExceptionGroup from collections.abc import Callable, Generator, Iterable, Mapping from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from itertools import chain, repeat from queue import SimpleQueue @@ -40,7 +41,10 @@ from models import ( LiteLLMParamsBody, McpServerCreateBody, McpServerUpdateBody, + ModelDeleteBody, + ModelInfoBody, ModelListEntry, + ModelNewBody, ModelsListResponse, OrgNewBody, OrgUpdateBody, @@ -59,6 +63,8 @@ from proxy_client import ( ConvergeOutcome, CredentialKind, EverywhereConverged, + ModelNotRegistered, + ModelRegistered, ModelsPoller, NeverConvergedOn, NotConverged, @@ -691,3 +697,85 @@ def test_partial_updates_preserve_explicit_null_at_the_http_boundary(operation: ) assert json.loads(bodies.get_nowait()) == expected assert bodies.empty() + + +@contextmanager +def model_registry( + listed: frozenset[str], refused: frozenset[str] +) -> Generator[tuple[ProxyClient, SimpleQueue[float], SimpleQueue[str]]]: + writes: Final[SimpleQueue[float]] = SimpleQueue() + deleted: Final[SimpleQueue[str]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + pass + + def reply(self, status: int, body: object) -> None: + self.send_response(status) + self.end_headers() + self.wfile.write(json.dumps(body).encode()) + + def do_GET(self) -> None: + self.reply(200, {"data": [{"id": name} for name in sorted(listed)]}) + + def do_POST(self) -> None: + raw: Final = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path == "/model/delete": + deleted.put(ModelDeleteBody.model_validate_json(raw).id) + self.reply(200, {}) + return + writes.put(time.monotonic()) + name: Final = ModelNewBody.model_validate_json(raw).model_name + if name in refused: + self.reply(500, {"error": "refused"}) + return + self.reply(200, {"model_id": f"id-{name}"}) + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) + thread.start() + url: Final = f"http://127.0.0.1:{server.server_port}" + proxy: Final = build_proxy_client( + base_url=url, control_plane_base_url=url, replica_urls=(url,), control_replica_urls=(url,), master_key="bootstrap" + ) + try: + yield replace(proxy, model_servable_timeout=0.3, model_servable_interval=0.01), writes, deleted + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +class TestRegisterModels: + def test_a_batch_waits_once_after_its_last_write_and_reports_each_model(self) -> None: + names: Final = ("first", "refused", "ghost", "last") + settled: Final[SimpleQueue[float]] = SimpleQueue() + with ( + model_registry(listed=frozenset({"first", "last"}), refused=frozenset({"refused"})) as ( + proxy, + writes, + deleted, + ), + without_retries(), + ): + registrations: Final = replace(proxy, settle=settled.put).register_models( + tuple( + ModelNewBody( + model_name=name, litellm_params=LiteLLMParamsBody(model="synthetic"), model_info=ModelInfoBody() + ) + for name in names + ) + ) + last_write: Final = max(writes.get_nowait() for _ in names) + assert writes.empty() + assert settled.get_nowait() >= last_write + assert settled.empty(), "a batch must wait out propagation once, not once per model" + assert tuple((type(r), r.model_name) for r in registrations) == ( + (ModelRegistered, "first"), + (ModelNotRegistered, "refused"), + (ModelNotRegistered, "ghost"), + (ModelRegistered, "last"), + ) + assert tuple(r.model_id for r in registrations if isinstance(r, ModelRegistered)) == ("id-first", "id-last") + assert deleted.get_nowait() == "id-ghost" + assert deleted.empty() diff --git a/tests/e2e_harness/test_stack_lock.py b/tests/e2e_harness/test_stack_lock.py deleted file mode 100644 index b275ca8c755..00000000000 --- a/tests/e2e_harness/test_stack_lock.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Cross-process behavior of the stack lock: readers share it, an exclusive holder waits for -every reader and keeps them out, and a reader arriving behind a waiting exclusive holder -queues behind it instead of starving it.""" - -from __future__ import annotations - -import fcntl -import inspect -import os -import subprocess -import sys -import time -from contextlib import ExitStack -from pathlib import Path -from typing import Final - -import pytest -import stack_lock - -HARNESS_DIR: Final = Path(inspect.getfile(stack_lock)).resolve().parent -DEADLINE_SECONDS: Final = 30.0 -SETTLE_SECONDS: Final = 0.5 -HOLDER_SCRIPT: Final = """ -import sys, time -from pathlib import Path -from stack_lock import stack_lock -name, mode, release_path, log_path = sys.argv[1:] - - -def record(event): - with Path(log_path).open("a") as log: - log.write(f"{name} {event}\\n") - - -record("waiting") -with stack_lock(exclusive=mode == "exclusive"): - record("enter") - while not Path(release_path).exists(): - time.sleep(0.02) - record("exit") -""" - - -def _events(log_path: Path) -> tuple[str, ...]: - return tuple(log_path.read_text().splitlines()) if log_path.exists() else () - - -def _wait_for_event(log_path: Path, event: str) -> None: - deadline: Final = time.monotonic() + DEADLINE_SECONDS - while event not in _events(log_path): - if time.monotonic() > deadline: - pytest.fail(f"{event!r} never appeared; events so far: {_events(log_path)}") - time.sleep(0.02) - - -def _wait_until_gate_is_held_exclusively(gate_path: Path) -> None: - deadline: Final = time.monotonic() + DEADLINE_SECONDS - with gate_path.open("a") as handle: - while True: - try: - fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB) - except BlockingIOError: - return - fcntl.flock(handle, fcntl.LOCK_UN) - if time.monotonic() > deadline: - pytest.fail("no exclusive holder ever took the gate") - time.sleep(0.02) - - -def _start_holder(held: ExitStack, tmp_path: Path, name: str, mode: str) -> subprocess.Popen[bytes]: - holder: Final = held.enter_context( - subprocess.Popen( - ( - sys.executable, - "-P", - "-c", - HOLDER_SCRIPT, - name, - mode, - str(tmp_path / f"release-{name}"), - str(tmp_path / "events"), - ), - cwd=HARNESS_DIR, - env={**os.environ, "TMPDIR": str(tmp_path), "PYTHONPATH": str(HARNESS_DIR)}, - ) - ) - held.callback(holder.kill) - return holder - - -def test_readers_share_exclusive_waits_and_a_waiting_exclusive_beats_later_readers(tmp_path: Path) -> None: - lock_dir: Final = tmp_path / f"litellm-e2e-stack-{stack_lock.STACK_DIGEST}" - lock_dir.mkdir() - log_path: Final = tmp_path / "events" - with ExitStack() as held: - first_reader: Final = _start_holder(held, tmp_path, "A", "shared") - _wait_for_event(log_path, "A enter") - second_reader: Final = _start_holder(held, tmp_path, "R", "shared") - _wait_for_event(log_path, "R enter") - (tmp_path / "release-R").touch() - _wait_for_event(log_path, "R exit") - writer: Final = _start_holder(held, tmp_path, "W", "exclusive") - _wait_until_gate_is_held_exclusively(lock_dir / "gate") - late_reader: Final = _start_holder(held, tmp_path, "B", "shared") - _wait_for_event(log_path, "B waiting") - time.sleep(SETTLE_SECONDS) - (tmp_path / "release-A").touch() - _wait_for_event(log_path, "W enter") - (tmp_path / "release-W").touch() - _wait_for_event(log_path, "B enter") - (tmp_path / "release-B").touch() - for holder in (first_reader, second_reader, writer, late_reader): - assert holder.wait(timeout=DEADLINE_SECONDS) == 0 - events: Final = _events(log_path) - assert events.index("R enter") < events.index("A exit") - assert events.index("W enter") > events.index("A exit") - assert events.index("B enter") > events.index("W exit") diff --git a/tests/integration/_support/custom_video_provider.py b/tests/integration/_support/custom_video_provider.py new file mode 100644 index 00000000000..3513d1e3186 --- /dev/null +++ b/tests/integration/_support/custom_video_provider.py @@ -0,0 +1,181 @@ +"""A customer-owned video provider for the proxy's `custom_provider_map`: a job queue with its own wire shape.""" + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from httpx._types import RequestFiles + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_llm import CustomLLM +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id + +_STATUS_BY_STATE: Final = MappingProxyType({"queued": "queued", "running": "in_progress", "done": "completed"}) + + +def _video_object(job: Mapping[str, object], model: str | None, custom_llm_provider: str | None) -> VideoObject: + job_id: Final = str(job["job_id"]) + seconds: Final = job.get("seconds") + video: Final = VideoObject( + id=encode_video_id_with_provider(job_id, custom_llm_provider, model) if custom_llm_provider else job_id, + object="video", + status=_STATUS_BY_STATE[str(job["state"])], + created_at=0, + model=model, + seconds=str(seconds) if seconds is not None else None, + ) + return video + + +class QueueVideoConfig(BaseVideoConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseVideoConfig contract + return ["seconds"] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: BaseVideoConfig contract + return {key: value for key, value in video_create_optional_params.items() if key == "seconds"} + + def validate_environment( + self, + headers: dict, # mutable-ok: BaseVideoConfig contract + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> dict: # mutable-ok: BaseVideoConfig contract + key: Final = (litellm_params.api_key if litellm_params is not None else None) or api_key or "" + return {**headers, "X-Api-Key": key} + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: BaseVideoConfig contract + ) -> str: + if api_base is None: + raise ValueError("api_base is required") + return f"{api_base.rstrip('/')}/jobs" + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: dict, # mutable-ok: BaseVideoConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[dict, RequestFiles, str]: # mutable-ok: BaseVideoConfig contract + seconds: Final = video_create_optional_request_params.get("seconds") + return ({"model": model, "prompt": prompt, "seconds": seconds}, [], api_base) + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + request_data: dict | None = None, + ) -> VideoObject: + return _video_object(raw_response.json(), model, custom_llm_provider) + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (f"{api_base}/{extract_original_video_id(video_id)}", {}) + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + client: object = None, + ) -> VideoObject: + return _video_object(raw_response.json(), None, custom_llm_provider) + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + variant: str | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (f"{api_base}/{extract_original_video_id(video_id)}/file", {}) + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + client: object = None, + ) -> bytes: + return raw_response.content + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, object] | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (api_base, {}) + + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseVideoConfig contract + return raw_response.json() + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("the queue backend has no remix") + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("the queue backend has no remix") + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("the queue backend has no delete") + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> VideoObject: + raise NotImplementedError("the queue backend has no delete") + + +VIDEO_CONFIG: Final = QueueVideoConfig() +IMAGE_ONLY_HANDLER: Final = CustomLLM() diff --git a/tests/integration/_support/interception_vendor.py b/tests/integration/_support/interception_vendor.py index 3ca40087441..b46eb998f7a 100644 --- a/tests/integration/_support/interception_vendor.py +++ b/tests/integration/_support/interception_vendor.py @@ -27,7 +27,7 @@ JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) _CHAT_TARGET: Final = re.compile(r"^/openai/deployments/([^/]+)/chat/completions$") -Mode: TypeAlias = Literal["search", "plain", "search-down", "followup-fails", "missing"] +Mode: TypeAlias = Literal["search", "multi-search", "plain", "search-down", "followup-fails", "missing"] def deployment(mode: Mode) -> str: @@ -42,8 +42,8 @@ def plain(name: str) -> str: return f"plain-{name}" -def query(name: str) -> str: - return f"q-{name}" +def query(name: str, turn: int = 1) -> str: + return f"q-{name}" if turn == 1 else f"q-{name}-{turn}" def _minted(prefix: str) -> str: @@ -98,9 +98,20 @@ def _chat(name: str, body: Mapping[str, JsonValue]) -> Reply: content_type="text/event-stream", chunks=chat_stream(_minted(f"chatcmpl-{name}-s-"), "gpt-4o", plain(name)) ) messages: Final = _ITEMS.validate_python(body["messages"]) - if any(message.get("role") == "tool" for message in messages): + tool_results: Final = tuple(message for message in messages if message.get("role") == "tool") + if tool_results: if name.endswith("-followup-fails"): return _azure_error(500, FOLLOWUP_FAILURE) + if name.endswith("-multi-search") and len(tool_results) == 1: + call: Final = { + "id": f"call_{name[4:36]}-2", + "type": "function", + "function": {"name": SEARCH_TOOL, "arguments": json.dumps({"query": query(name, turn=2)})}, + } + message: Final = JSON_OBJECT.validate_python({"role": "assistant", "content": None, "tool_calls": [call]}) + return _chat_completion( + _minted(f"chatcmpl-{name}-2-"), message, "tool_calls", FOLLOWUP_USAGE + ) return _chat_completion( _minted(f"chatcmpl-{name}-2-"), {"role": "assistant", "content": answer(name)}, "stop", FOLLOWUP_USAGE ) @@ -153,9 +164,20 @@ def _responses(body: Mapping[str, JsonValue]) -> Reply: content_type="text/event-stream", chunks=responses_stream(_minted(f"resp_{name}_s_"), name, plain(name)) ) items: Final = _ITEMS.validate_python(body["input"]) if isinstance(body.get("input"), list) else [] - if any(item.get("type") == "function_call_output" for item in items): + function_call_outputs: Final = tuple(item for item in items if item.get("type") == "function_call_output") + if function_call_outputs: if name.endswith("-followup-fails"): return _azure_error(500, FOLLOWUP_FAILURE) + if name.endswith("-multi-search") and len(function_call_outputs) == 1: + call: Final = { + "type": "function_call", + "id": f"fc_{name[4:36]}-2", + "call_id": f"call_{name[4:36]}-2", + "name": SEARCH_TOOL, + "arguments": json.dumps({"query": query(name, turn=2)}), + "status": "completed", + } + return _response(_minted(f"resp_{name}_2_"), name, (call,), RESPONSES_FOLLOWUP_USAGE) return _response( _minted(f"resp_{name}_2_"), name, (_text_item(f"{name}_2", answer(name)),), RESPONSES_FOLLOWUP_USAGE ) diff --git a/tests/integration/_support/riva.py b/tests/integration/_support/riva.py index 8552bc2197d..438d877386b 100644 --- a/tests/integration/_support/riva.py +++ b/tests/integration/_support/riva.py @@ -39,6 +39,7 @@ class RivaCall: rpc: Rpc audio: bytes cancelled: bool + authorization: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -68,6 +69,10 @@ def pcm_bytes(seconds: float) -> int: return int(seconds * SAMPLE_RATE_HZ) * 2 +def _authorization(context: grpc.ServicerContext) -> tuple[str, ...]: + return tuple(str(value) for key, value in context.invocation_metadata() if key == "authorization") + + def carries_clip(audio: bytes, seconds: float) -> bool: sent: Final = _clip_samples(seconds) if len(audio) != len(sent) * 2: @@ -100,9 +105,9 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer): released: Final = threading.Event() context.add_callback(released.set) released.wait(timeout=_STALL_LIMIT_SECONDS) - self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=not context.is_active())) + self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, not context.is_active(), _authorization(context))) return riva_asr_pb2.RecognizeResponse() - self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=False)) + self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, False, _authorization(context))) return riva_asr_pb2.RecognizeResponse( results=[riva_asr_pb2.SpeechRecognitionResult(alternatives=[self._alternative(request.config)])] ) @@ -111,13 +116,12 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer): self, request_iterator: Iterable[riva_asr_pb2.StreamingRecognizeRequest], context: grpc.ServicerContext ) -> Iterator[riva_asr_pb2.StreamingRecognizeResponse]: if self._offline_only: - self._calls.put(RivaCall(Rpc.STREAMING, b"", cancelled=False)) + self._calls.put(RivaCall(Rpc.STREAMING, b"", False, _authorization(context))) context.abort(grpc.StatusCode.INVALID_ARGUMENT, OFFLINE_ONLY_REFUSAL) requests: Final = tuple(request_iterator) config: Final = requests[0].streaming_config.config if requests else riva_asr_pb2.RecognitionConfig() - self._calls.put( - RivaCall(Rpc.STREAMING, b"".join(request.audio_content for request in requests), cancelled=False) - ) + audio: Final = b"".join(request.audio_content for request in requests) + self._calls.put(RivaCall(Rpc.STREAMING, audio, False, _authorization(context))) yield riva_asr_pb2.StreamingRecognizeResponse( results=[riva_asr_pb2.StreamingRecognitionResult(alternatives=[self._alternative(config)], is_final=True)] ) diff --git a/tests/integration/authorization/test_jwt_auto_register_map_existing_key.py b/tests/integration/authorization/test_jwt_auto_register_map_existing_key.py index 62eda977438..c8fbcbb1c97 100644 --- a/tests/integration/authorization/test_jwt_auto_register_map_existing_key.py +++ b/tests/integration/authorization/test_jwt_auto_register_map_existing_key.py @@ -7,13 +7,15 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from hashlib import sha256 from pathlib import Path -from typing import Final +from typing import Final, Literal import httpx import jwt import pytest import yaml from cryptography.hazmat.primitives.asymmetric import rsa +from pydantic import JsonValue +from redis import Redis from tests.integration._support.client import Gateway, eventually, string_value from tests.integration._support.database import read_rows @@ -31,7 +33,15 @@ def _hash(key: str) -> str: return sha256(key.encode()).hexdigest() -def _config(directory: Path, claim_field: str) -> Path: +def _config( + directory: Path, + claim_field: str, + *, + behavior: Literal["auto_register", "link_only", "reject"] = "auto_register", + map_existing_key: bool = True, + user_id_upsert: bool = False, + issuers: tuple[dict[str, JsonValue], ...] = (), +) -> Path: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["general_settings"] = { **config["general_settings"], @@ -40,8 +50,10 @@ def _config(directory: Path, claim_field: str) -> Path: "user_id_jwt_field": "sub", "user_email_jwt_field": "email", "virtual_key_claim_field": claim_field, - "unregistered_jwt_client_behavior": "auto_register", - "auto_register_map_existing_key": True, + "unregistered_jwt_client_behavior": behavior, + "auto_register_map_existing_key": map_existing_key, + "user_id_upsert": user_id_upsert, + "issuers": list(issuers), }, } path: Final = directory / f"jwt_map_existing_key_{claim_field}.yaml" @@ -107,7 +119,7 @@ def _billed_key(response: httpx.Response) -> str: return string_value(rows[0]["api_key"]) -def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_ineligible_newer_key( +def test_first_jwt_call_reuses_the_newest_durable_key_and_skips_every_ineligible_newer_key( gateway: Gateway, tmp_path: Path ) -> None: with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: @@ -118,7 +130,6 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig skipped: Final = { "older_durable": older_durable, "expiring": scenario.key(user_id=user, duration="1h"), - "management_only": scenario.key(user_id=user, allowed_routes=["management_routes"]), "auto_registered_look_alike": scenario.key(user_id=user, metadata={"auto_registered": True}), "other_team": scenario.key(user_id=user, team_id=scenario.team()), "blocked": scenario.key(user_id=user), @@ -140,6 +151,84 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig assert _billed_key(response) == _hash(durable) +@pytest.mark.parametrize( + ("behavior", "map_existing_key"), [("auto_register", True), ("link_only", False), ("link_only", True)] +) +def test_route_restricted_keys_are_linked_without_bypassing_other_key_checks( + gateway: Gateway, + tmp_path: Path, + behavior: Literal["auto_register", "link_only"], + map_existing_key: bool, +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + scenario.key(user_id=user) + restricted: Final = scenario.key(user_id=user, allowed_routes=["/v1/chat/completions"]) + scenario.key(user_id=user, duration="1h") + scenario.key(user_id=user, metadata={"auto_registered": True}) + scenario.key(user_id=user, team_id=scenario.team()) + blocked: Final = scenario.key(user_id=user) + gateway.post("/key/block", {"key": blocked}) + keys_before: Final = _user_key_hashes(user) + token: Final = _token(private_key, user) + + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config( + tmp_path, + "sub", + behavior=behavior, + map_existing_key=map_existing_key, + ), + ) as candidate: + response: Final = _chat(candidate, model, token) + repeated: Final = _chat(candidate, model, token) + restricted_route: Final = candidate.request("GET", "/v1/models", key=token) + + assert response.status_code == 200, response.text + assert repeated.status_code == 200, repeated.text + assert restricted_route.status_code == 403, restricted_route.text + assert _mapped_token("sub", user) == _hash(restricted) + assert _user_key_hashes(user) == keys_before, "a new key was minted instead of linking an existing key" + assert _billed_key(response) == _hash(restricted) + assert _billed_key(repeated) == _hash(restricted) + + +@pytest.mark.parametrize("behavior", ["auto_register", "link_only"]) +def test_management_route_keys_are_linked_by_default_and_keep_their_route_permissions( + gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"] +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + target: Final = f"integration-{uuid.uuid4().hex}" + scenario.key(user_id=user) + management_key: Final = scenario.key(user_id=user, allowed_routes=["/user/new"]) + keys_before: Final = _user_key_hashes(user) + token: Final = _token(private_key, user) + with owned_proxy( + gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=_config(tmp_path, "sub", behavior=behavior) + ) as candidate: + denied: Final = _chat(candidate, model, token) + allowed: Final = candidate.request( + "POST", + "/user/new", + {"user_id": target, "auto_create_key": False, "user_role": "internal_user"}, + key=token, + ) + assert allowed.status_code == 200, allowed.text + scenario.cleanups.callback(scenario.delete_user, target) + repeated: Final = _chat(candidate, model, token) + assert denied.status_code == 403, denied.text + assert repeated.status_code == 403, repeated.text + assert allowed.json()["user_id"] == target + assert _mapped_token("sub", user) == _hash(management_key) + assert _user_key_hashes(user) == keys_before + + def test_user_matched_by_email_instead_of_sub_still_reuses_their_existing_key(gateway: Gateway, tmp_path: Path) -> None: with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: model: Final = scenario.model() @@ -183,8 +272,9 @@ def test_shared_client_claim_never_maps_a_second_user_onto_the_first_users_perso assert read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (_hash(personal),)) == [] -def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviving_mapped_key( - gateway: Gateway, tmp_path: Path +@pytest.mark.parametrize("behavior", ["auto_register", "link_only"]) +def test_concurrent_first_jwt_calls_both_succeed_on_one_surviving_mapped_key( + gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"] ) -> None: writer_url: Final = os.environ.get("INTEGRATION_PROXY_DATABASE_URL") or os.environ["DATABASE_URL"] with ( @@ -194,6 +284,7 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv ): model: Final = scenario.model() user: Final = scenario.user(user_role="internal_user") + existing: Final = scenario.key(user_id=user) if behavior == "link_only" else None token: Final = _token(private_key, user) overrides: Final = { "JWT_PUBLIC_KEY_URL": jwks_url, @@ -202,7 +293,12 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv } with ( - owned_proxy(gateway, tmp_path, overrides, config=_config(tmp_path, "sub")) as candidate, + owned_proxy( + gateway, + tmp_path, + overrides, + config=_config(tmp_path, "sub", behavior=behavior, map_existing_key=False), + ) as candidate, ThreadPoolExecutor(max_workers=1) as pool, ): held_call: Final = pool.submit(_chat, candidate, model, token) @@ -215,5 +311,242 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv assert held.status_code == 200, held.text keys: Final = _user_key_hashes(user) assert len(keys) == 1, keys + if existing is not None: + assert keys == frozenset({_hash(existing)}) assert _mapped_token("sub", user) in keys assert (_billed_key(held), _billed_key(racing)) == (_mapped_token("sub", user),) * 2 + + +@pytest.mark.parametrize("ineligible_keys", [False, True]) +def test_link_only_rejects_without_writes_then_links_a_newly_available_key( + gateway: Gateway, tmp_path: Path, ineligible_keys: bool +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + if ineligible_keys: + scenario.key(user_id=user, duration="1h") + scenario.key(user_id=user, metadata={"auto_registered": True}) + scenario.key(user_id=user, team_id=scenario.team()) + blocked: Final = scenario.key(user_id=user) + gateway.post("/key/block", {"key": blocked}) + keys_before: Final = _user_key_hashes(user) + token: Final = _token(private_key, user) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False), + ) as candidate: + first: Final = _chat(candidate, model, token) + repeated: Final = _chat(candidate, model, token) + assert first.status_code == 403, first.text + assert repeated.status_code == 403, repeated.text + assert _user_key_hashes(user) == keys_before + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == [] + available: Final = scenario.key(user_id=user) + accepted: Final = _chat(candidate, model, token) + assert accepted.status_code == 200, accepted.text + assert _mapped_token("sub", user) == _hash(available) + assert _user_key_hashes(user) == keys_before | {_hash(available)} + assert _billed_key(accepted) == _hash(available) + + +@pytest.mark.parametrize("claim_field", ["client_id", "email"]) +def test_link_only_rejects_shared_or_missing_claims_without_linking_a_personal_key( + gateway: Gateway, tmp_path: Path, claim_field: str +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=user) + shared_claim: Final = f"integration-client-{uuid.uuid4().hex}" + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, claim_field, behavior="link_only", map_existing_key=False), + ) as candidate: + response: Final = _chat(candidate, model, _token(private_key, user, client_id=shared_claim)) + assert response.status_code == 403, response.text + assert _user_key_hashes(user) == frozenset({_hash(key)}) + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE token=%s', (_hash(key),)) == [] + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (shared_claim,)) == [] + + +@pytest.mark.parametrize("restriction", ["models", "allowed_routes"]) +def test_link_only_enforces_key_permissions_on_the_first_and_cached_requests( + gateway: Gateway, tmp_path: Path, restriction: str +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + allowed: Final = scenario.model() + denied: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key( + user_id=user, + **({"models": [allowed]} if restriction == "models" else {"allowed_routes": ["/v1/models"]}), + ) + token: Final = _token(private_key, user) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False), + ) as candidate: + first: Final = _chat(candidate, denied, token) + repeated: Final = _chat(candidate, denied, token) + control: Final = ( + _chat(candidate, allowed, token) + if restriction == "models" + else candidate.request("GET", "/v1/models", key=token) + ) + assert first.status_code == 403, first.text + assert repeated.status_code == 403, repeated.text + assert control.status_code == 200, control.text + assert _mapped_token("sub", user) == _hash(key) + assert _user_key_hashes(user) == frozenset({_hash(key)}) + + +def test_link_only_does_not_replace_an_existing_mapping_to_a_blocked_key(gateway: Gateway, tmp_path: Path) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + blocked: Final = scenario.key(user_id=user) + available: Final = scenario.key(user_id=user) + gateway.post("/jwt/key/mapping/new", {"jwt_claim_name": "sub", "jwt_claim_value": user, "key": blocked}) + gateway.post("/key/block", {"key": blocked}) + token: Final = _token(private_key, user) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False), + ) as candidate: + first: Final = _chat(candidate, model, token) + repeated: Final = _chat(candidate, model, token) + assert first.status_code == 401, first.text + assert repeated.status_code == 401, repeated.text + assert _mapped_token("sub", user) == _hash(blocked) + assert _user_key_hashes(user) == frozenset({_hash(blocked), _hash(available)}) + + +def test_link_only_exempts_proxy_admins_but_rechecks_the_role_after_a_cached_exemption( + gateway: Gateway, tmp_path: Path +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + admin: Final = _token(private_key, user, scope="litellm_proxy_admin") + ordinary: Final = _token(private_key, user) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False), + ) as candidate: + first: Final = candidate.request("GET", "/v1/models", key=admin) + repeated: Final = candidate.request("GET", "/v1/models", key=admin) + denied: Final = candidate.request("GET", "/v1/models", key=ordinary) + forbidden_route: Final = _chat(candidate, model, admin) + assert first.status_code == 200, first.text + assert repeated.status_code == 200, repeated.text + assert denied.status_code == 403, denied.text + assert forbidden_route.status_code == 401, forbidden_route.text + assert _user_key_hashes(user) == frozenset() + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == [] + + +@pytest.mark.parametrize("user_id_upsert", [False, True]) +def test_link_only_keeps_user_provisioning_independent_of_key_creation( + gateway: Gateway, tmp_path: Path, user_id_upsert: bool +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + subject: Final = f"integration-link-only-{uuid.uuid4().hex}" + try: + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config( + tmp_path, "sub", behavior="link_only", map_existing_key=False, user_id_upsert=user_id_upsert + ), + ) as candidate: + response: Final = _chat(candidate, model, _token(private_key, subject)) + assert response.status_code == (403 if user_id_upsert else 401), response.text + users: Final = read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,)) + assert users == ([{"user_id": subject}] if user_id_upsert else []) + assert _user_key_hashes(subject) == frozenset() + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (subject,)) == [] + finally: + if read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,)): + scenario.delete_user(subject) + + +@pytest.mark.parametrize("issuer_behavior", ["auto_register", "link_only"]) +def test_issuer_specific_registration_mode_overrides_the_global_mode( + gateway: Gateway, tmp_path: Path, issuer_behavior: Literal["auto_register", "link_only"] +) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + issuer: Final = f"https://integration.example.com/{uuid.uuid4().hex}" + token: Final = _token(private_key, user, iss=issuer, aud="integration") + path: Final = _config( + tmp_path, + "sub", + behavior="auto_register" if issuer_behavior == "link_only" else "link_only", + map_existing_key=False, + issuers=( + { + "issuer": issuer, + "jwks_url": jwks_url, + "audience": "integration", + "user_id_jwt_field": "sub", + "virtual_key_claim_field": "sub", + "unregistered_jwt_client_behavior": issuer_behavior, + }, + ), + ) + with owned_proxy(gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=path) as candidate: + response: Final = _chat(candidate, model, token) + assert response.status_code == (403 if issuer_behavior == "link_only" else 200), response.text + if issuer_behavior == "link_only": + assert _user_key_hashes(user) == frozenset() + assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == [] + else: + assert len(_user_key_hashes(user)) == 1 + assert _mapped_token("sub", user) in _user_key_hashes(user) + assert _billed_key(response) == _mapped_token("sub", user) + + +def test_link_only_links_after_a_cached_rejection_from_the_previous_mode(gateway: Gateway, tmp_path: Path) -> None: + with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=user) + token: Final = _token(private_key, user) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="reject", map_existing_key=False), + ) as rejecting: + denied: Final = _chat(rejecting, model, token) + assert denied.status_code == 403, denied.text + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + eventually( + lambda: cache.get(f"jwt_key_mapping:sub:{user}"), + lambda value: value is not None and b"__NO_MAPPING__" in value, + ) + with owned_proxy( + gateway, + tmp_path, + {"JWT_PUBLIC_KEY_URL": jwks_url}, + config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False), + ) as linking: + accepted: Final = _chat(linking, model, token) + assert accepted.status_code == 200, accepted.text + assert _mapped_token("sub", user) == _hash(key) + assert _user_key_hashes(user) == frozenset({_hash(key)}) + assert _billed_key(accepted) == _hash(key) diff --git a/tests/integration/compatibility/test_audio_transcriptions_wire.py b/tests/integration/compatibility/test_audio_transcriptions_wire.py index 3deeaebaf7d..faebfda715b 100644 --- a/tests/integration/compatibility/test_audio_transcriptions_wire.py +++ b/tests/integration/compatibility/test_audio_transcriptions_wire.py @@ -345,10 +345,6 @@ def test_audio_transcription_forwards_nested_and_repeated_bracket_fields_verbati def test_audio_transcription_stream_returns_server_sent_events(gateway: Gateway) -> None: - pytest.skip( - "BUG: stream=true transcriptions reach the upstream but its SSE body comes back as application/json " - '{"text": "data: ..."}, so OpenAI SDK stream=True callers get no transcript events' - ) wav: Final = _WAV_BYTES + uuid.uuid4().bytes def respond(request: Request) -> Reply: diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..18fb692da6e --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,39 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import refused_boot_log + + +def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "general_settings": {"fips_mode": fips_mode}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True)) + assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced")) + assert "general_settings.fips_mode='enforced' is not a boolean" in log, log diff --git a/tests/integration/database/migrations/test_drop_accounting_event_tables.py b/tests/integration/database/migrations/test_drop_accounting_event_tables.py new file mode 100644 index 00000000000..583223a573b --- /dev/null +++ b/tests/integration/database/migrations/test_drop_accounting_event_tables.py @@ -0,0 +1,64 @@ +import os +from pathlib import Path +from typing import Final + +import psycopg +import pytest + +_MIGRATIONS: Final = Path(__file__).parents[4] / "litellm-proxy-extras/litellm_proxy_extras/migrations" +_OLD_LEDGER: Final = _MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql" +_SPEND_LOG_DELIVERY: Final = _MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql" +_DROP: Final = _MIGRATIONS / "20261104010000_drop_accounting_event_tables/migration.sql" +_DROPPED: Final = ( + "AccountingDeliveryTarget", + "AccountingEventKind", + "LiteLLM_AccountingDelivery", + "LiteLLM_AccountingEvent", +) + + +def _apply(connection: psycopg.Connection[tuple[object, ...]], migration: Path) -> None: + with connection.transaction(): + connection.execute(migration.read_bytes()) + + +def _present(connection: psycopg.Connection[tuple[object, ...]], names: tuple[str, ...]) -> tuple[object, ...]: + return tuple( + row[0] + for row in connection.execute( + "SELECT relname FROM pg_class WHERE relname = ANY(%s) " + "UNION SELECT typname FROM pg_type WHERE typname = ANY(%s) ORDER BY 1", + (list(names), list(names)), + ).fetchall() + ) + + +@pytest.mark.parametrize("old_states", [(), ("applied", "pending", "parked")], ids=["empty", "with_rows"]) +def test_drop_removes_the_old_ledger_and_keeps_spend_log_deliveries(old_states: tuple[str, ...]) -> None: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + _apply(connection, _OLD_LEDGER) + connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id text PRIMARY KEY)') + _apply(connection, _SPEND_LOG_DELIVERY) + for index, state in enumerate(old_states): + connection.execute( + 'INSERT INTO "LiteLLM_AccountingEvent" (event_id, kind, schema_version, payload_hash, occurred_at, ' + "payload) VALUES (%s, 'cost', 1, 'hash', now(), '{}')", + (f"event-{index}",), + ) + connection.execute( + 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state) ' + "VALUES (%s, 'LiteLLM_UserTable', 1, %s)", + (f"event-{index}", state), + ) + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, accounting_payload, is_batch) ' + "VALUES ('request', 'pending', '{}', false)" + ) + + _apply(connection, _DROP) + _apply(connection, _DROP) + + assert _present(connection, _DROPPED) == () + assert connection.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("request", "pending") + ] diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py deleted file mode 100644 index 15cd5413149..00000000000 --- a/tests/integration/database/test_accounting_ledger.py +++ /dev/null @@ -1,610 +0,0 @@ -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from collections.abc import AsyncGenerator, AsyncIterator, Iterator, Mapping, Sequence -from contextlib import asynccontextmanager -from datetime import datetime, timezone -from decimal import Decimal -from itertools import chain -from pathlib import Path -from typing import TYPE_CHECKING, Final, TypeAlias - -import psycopg -import pytest -import pytest_asyncio -from psycopg import sql -from pydantic import JsonValue - -from litellm.accounting import ( - CostEvent, - DeliveryState, - DeliveryTarget, - EventKind, - InboundUsageEvent, - Metric, - MetricName, - OutboundUsageEvent, - Outcome, - PricingSnapshot, -) -from litellm.proxy.db import accounting_ledger -from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite -from litellm.types.utils import CallTypes -from tests.integration._support.database import scratch_database -from tests.integration._support.prisma_clients import connected_proxy_prisma - -if TYPE_CHECKING: - from prisma import Prisma - - from litellm.proxy.utils import PrismaClient - -_Writer: TypeAlias = tuple["PrismaClient", AccountingLedger] - -_SCHEMA: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/schema.prisma" -_EVENT_INSERT: Final = ( - 'INSERT INTO "LiteLLM_AccountingEvent" ' - "(event_id, kind, schema_version, payload_hash, occurred_at, payload) " - "VALUES ('event', %s, %s, 'hash', '2026-01-01', '{}') RETURNING kind::text, schema_version" -) -_DELIVERY_INSERT: Final = ( - 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state, attempts) ' - "VALUES ('event', %s, %s, %s, %s) RETURNING target::text, version, state::text, attempts" -) - - -@pytest.fixture(scope="module") -def ledger_database() -> Iterator[str]: - with scratch_database("accounting") as database_url: - result: Final = subprocess.run( - [sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(_SCHEMA)], - env={**os.environ, "DATABASE_URL": database_url}, - capture_output=True, - text=True, - timeout=300, - ) - assert result.returncode == 0, result.stdout + result.stderr - yield database_url - - -@pytest.fixture -def ledger(ledger_database: str) -> Iterator[psycopg.Connection[tuple[object, ...]]]: - with psycopg.connect(ledger_database) as connection: - try: - yield connection - finally: - connection.rollback() - - -@pytest.mark.parametrize("kind", tuple(EventKind)) -def test_ledger_accepts_every_public_event_kind( - ledger: psycopg.Connection[tuple[object, ...]], kind: EventKind -) -> None: - row: Final = ledger.execute(_EVENT_INSERT, (kind.value, 1)).fetchone() - assert row == (kind.value, 1) - - -@pytest.mark.parametrize("target", tuple(DeliveryTarget)) -@pytest.mark.parametrize("state", tuple(DeliveryState)) -def test_delivery_accepts_every_public_target_and_state( - ledger: psycopg.Connection[tuple[object, ...]], target: DeliveryTarget, state: DeliveryState -) -> None: - row: Final = ledger.execute(_DELIVERY_INSERT, (target.value, 1, state.value, 0)).fetchone() - assert row == (target.value, 1, state.value, 0) - - -@pytest.mark.parametrize( - ("enum_name", "values"), - [ - ("AccountingEventKind", tuple(value.value for value in EventKind)), - ("AccountingDeliveryState", tuple(value.value for value in DeliveryState)), - ("AccountingDeliveryTarget", tuple(value.value for value in DeliveryTarget)), - ], -) -def test_migrated_enums_match_the_public_contract( - ledger: psycopg.Connection[tuple[object, ...]], enum_name: str, values: tuple[str, ...] -) -> None: - rows: Final = ledger.execute( - "SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid WHERE t.typname = %s", - (enum_name,), - ).fetchall() - assert frozenset(row[0] for row in rows) == frozenset(values) - - -@pytest.mark.parametrize("version", [0, -1]) -def test_event_rejects_nonpositive_schema_versions( - ledger: psycopg.Connection[tuple[object, ...]], version: int -) -> None: - with pytest.raises(psycopg.errors.CheckViolation, match="AccountingEvent_schema_version_check"): - ledger.execute(_EVENT_INSERT, (EventKind.OUTBOUND_USAGE.value, version)) - - -@pytest.mark.parametrize( - ("version", "attempts", "constraint"), - [ - (0, 0, "AccountingDelivery_version_check"), - (-1, 0, "AccountingDelivery_version_check"), - (1, -1, "AccountingDelivery_attempts_check"), - ], -) -def test_delivery_rejects_invalid_versions_and_attempts( - ledger: psycopg.Connection[tuple[object, ...]], version: int, attempts: int, constraint: str -) -> None: - with pytest.raises(psycopg.errors.CheckViolation, match=constraint): - ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, version, "pending", attempts)) - - -@pytest.mark.parametrize("column", ["state", "target"]) -def test_delivery_rejects_unknown_enum_values(ledger: psycopg.Connection[tuple[object, ...]], column: str) -> None: - ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, 1, "pending", 0)) - with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"): - ledger.execute( - sql.SQL('UPDATE "LiteLLM_AccountingDelivery" SET {} = %s').format(sql.Identifier(column)), ("invalid",) - ) - - -def test_event_rejects_unknown_kind(ledger: psycopg.Connection[tuple[object, ...]]) -> None: - with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"): - ledger.execute(_EVENT_INSERT, ("invalid", 1)) - - -def test_new_delivery_defaults_to_pending_with_no_attempts(ledger: psycopg.Connection[tuple[object, ...]]) -> None: - row: Final = ledger.execute( - 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) ' - "VALUES ('event', %s, 1) RETURNING state::text, attempts", - (DeliveryTarget.KEY_SPEND.value,), - ).fetchone() - assert row == ("pending", 0) - - -_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) -_CUTOFF: Final = datetime(2026, 2, 1, tzinfo=timezone.utc) -_TARGETS: Final = (DeliveryTarget.USER_SPEND, DeliveryTarget.TEAM_SPEND) - - -def _usage(event_id: str = "usage", *, inbound: bool = False) -> InboundUsageEvent | OutboundUsageEvent: - fields: Final = dict( - event_id=event_id, - request_id="request", - call_type=CallTypes.completion, - started_at=_AT, - ended_at=_AT, - outcome=Outcome.SUCCEEDED, - dimensions=(), - ) - if inbound: - return InboundUsageEvent(**fields, metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),)) - return OutboundUsageEvent(**fields, attempt_id=event_id, retry_of_attempt_id=None, metrics=None) - - -def _cost(source: str = "usage", event_id: str = "cost", amount: str = "0.125") -> CostEvent: - return CostEvent( - event_id=event_id, - usage_event_id=source, - occurred_at=_AT, - dimensions=(), - charges=(Metric(name=MetricName.SPEND, value=Decimal(amount), unit="USD"),), - snapshot=PricingSnapshot( - calculator_version="fixture", - model="fixture", - provider=None, - pricing_source="cost_map", - priced_model_key="fixture", - total_cost=Decimal(amount), - ), - ) - - -def _envelope(event: AccountingEvent, *, context: Mapping[str, JsonValue] | None = None) -> str: - return json.dumps({"event": event.model_dump(mode="json"), "context": context or {}}) - - -async def _insert( - store: AccountingLedger, - event: AccountingEvent, - targets: tuple[DeliveryTarget, ...] = _TARGETS, - *, - context: Mapping[str, JsonValue] | None = None, -) -> str: - return await store.insert_event(event, targets, envelope=_envelope(event, context=context)) - - -def _rows(database_url: str, statement: str, parameters: tuple[object, ...] = ()) -> Sequence[tuple[object, ...]]: - with psycopg.connect(database_url) as connection: - cursor: Final = connection.execute(statement, parameters) - return cursor.fetchall() if cursor.description is not None else [] - - -def _complete_and_age(database_url: str) -> None: - with psycopg.connect(database_url) as connection: - connection.execute('UPDATE "LiteLLM_AccountingEvent" SET recorded_at = %s', (_AT,)) - connection.execute("UPDATE \"LiteLLM_AccountingDelivery\" SET state = 'applied', applied_at = %s", (_AT,)) - - -@pytest_asyncio.fixture(loop_scope="function") -async def writer(ledger_database: str, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[_Writer]: - monkeypatch.setenv("DATABASE_URL", ledger_database) - monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) - _rows(ledger_database, 'TRUNCATE "LiteLLM_AccountingDelivery", "LiteLLM_AccountingEvent"') - async with connected_proxy_prisma(ledger_database) as client: - yield client, AccountingLedger.for_client(client) - - -@pytest.mark.asyncio -async def test_event_and_complete_delivery_set_commit_together(writer: _Writer, ledger_database: str) -> None: - _, store = writer - event: Final = _usage() - assert await _insert(store, event, context={"requested_region": "eu", "quantity": "1.25"}) == "inserted" - rows: Final = _rows(ledger_database, 'SELECT kind::text, source_event_id, payload FROM "LiteLLM_AccountingEvent"') - assert rows == [ - ("outbound_usage", None, json.loads(_envelope(event, context={"requested_region": "eu", "quantity": "1.25"}))) - ] - deliveries: Final = _rows( - ledger_database, 'SELECT target::text, state::text, attempts FROM "LiteLLM_AccountingDelivery"' - ) - assert set(deliveries) == {(target.value, "pending", 0) for target in _TARGETS} - - -@pytest.mark.asyncio -async def test_concurrent_duplicates_preserve_completed_deliveries(writer: _Writer, ledger_database: str) -> None: - _, store = writer - event: Final = _usage(inbound=True) - results: Final = await asyncio.gather(*(_insert(store, event) for _ in range(8))) - assert results.count("inserted") == 1 and results.count("duplicate") == 7 - _complete_and_age(ledger_database) - before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY target') - reordered: Final = json.dumps(json.loads(_envelope(event)), sort_keys=True, indent=2) - assert await store.insert_event(event, tuple(reversed(_TARGETS)) + _TARGETS, envelope=reordered) == "duplicate" - assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY target') == before - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(1,)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("change", ["payload", "context", "targets", "schema_version", "version"]) -async def test_conflicting_acceptance_cannot_overwrite_or_add_work( - writer: _Writer, ledger_database: str, change: str -) -> None: - _, store = writer - event: Final = _usage() - await _insert(store, event) - altered: Final = event.model_copy(update={"outcome": Outcome.FAILED}) if change == "payload" else event - targets: Final = (DeliveryTarget.KEY_SPEND,) if change == "targets" else _TARGETS - context: Final = {"override": "0.5"} if change == "context" else {} - options: Final = {change: 2} if change in ("schema_version", "version") else {} - with pytest.raises(AccountingConflictError, match="conflicts"): - await store.insert_event(altered, targets, envelope=_envelope(altered, context=context), **options) - assert _rows(ledger_database, 'SELECT payload FROM "LiteLLM_AccountingEvent"') == [(json.loads(_envelope(event)),)] - assert set(_rows(ledger_database, 'SELECT target::text FROM "LiteLLM_AccountingDelivery"')) == { - (t.value,) for t in _TARGETS - } - - -@pytest.mark.asyncio -async def test_competing_cost_identities_accept_only_one_authoritative_charge( - writer: _Writer, ledger_database: str -) -> None: - _, store = writer - await _insert(store, _usage()) - results: Final = await asyncio.gather( - _insert(store, _cost()), _insert(store, _cost(event_id="other-cost")), return_exceptions=True - ) - assert sum(result == "inserted" for result in results) == 1 - assert sum(isinstance(result, AccountingConflictError) for result in results) == 1 - accepted: Final = _rows( - ledger_database, "SELECT event_id, payload FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" - ) - assert len(accepted) == 1 - event: Final = CostEvent.model_validate_json(json.dumps(accepted[0][1]["event"])) - assert await _insert(store, event) == "duplicate" - assert event.snapshot.total_cost == Decimal("0.125") - with pytest.raises(AccountingConflictError): - await _insert(store, _cost(event_id=event.event_id, amount="0.5")) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("source", ["missing", "inbound", "cost"]) -async def test_cost_requires_an_existing_outbound_source(writer: _Writer, ledger_database: str, source: str) -> None: - _, store = writer - if source == "inbound": - await _insert(store, _usage("source", inbound=True)) - if source == "cost": - await _insert(store, _usage()) - await _insert(store, _cost(event_id="source")) - with pytest.raises(AccountingConflictError, match="outbound usage"): - await _insert(store, _cost(source="source", event_id="invalid")) - assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'invalid'") == [ - (0,) - ] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "cancellation"]) -async def test_interruption_rolls_back_or_retry_recognizes_commit( - writer: _Writer, ledger_database: str, failure: str -) -> None: - client, store = writer - - @asynccontextmanager - async def interrupted() -> AsyncGenerator[Prisma]: - async with client.writer_db.tx() as tx: - yield tx - if failure == "before_commit": - raise RuntimeError("injected pre-commit failure") - if failure == "cancellation": - raise asyncio.CancelledError - raise RuntimeError("injected lost acknowledgement") - - error: Final = asyncio.CancelledError if failure == "cancellation" else RuntimeError - with pytest.raises(error): - await _insert(AccountingLedger(interrupted), _usage()) - expected: Final = 1 if failure == "after_commit" else 0 - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(expected,)] - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(expected * len(_TARGETS),)] - assert await _insert(store, _usage()) == ("duplicate" if expected else "inserted") - - -@pytest.mark.asyncio -async def test_delivery_insert_failure_rolls_back_event(writer: _Writer, ledger_database: str) -> None: - from prisma.errors import RawQueryError - - _, store = writer - with psycopg.connect(ledger_database) as connection: - connection.execute( - 'ALTER TABLE "LiteLLM_AccountingDelivery" ADD CONSTRAINT reject_delivery CHECK (version < 1)' - ) - try: - with pytest.raises(RawQueryError, match="reject_delivery"): - await _insert(store, _usage()) - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] - finally: - with psycopg.connect(ledger_database) as connection: - connection.execute('ALTER TABLE "LiteLLM_AccountingDelivery" DROP CONSTRAINT reject_delivery') - - -@pytest.mark.asyncio -@pytest.mark.parametrize("blocked", ["pending", "parked", "unpriced", "recent_cost", "recovery"]) -async def test_retention_preserves_incomplete_or_recovery_needed_families( - writer: _Writer, ledger_database: str, blocked: str -) -> None: - _, store = writer - await _insert(store, _usage()) - if blocked != "unpriced": - await _insert(store, _cost()) - _complete_and_age(ledger_database) - with psycopg.connect(ledger_database) as connection: - if blocked in ("pending", "parked"): - connection.execute( - "UPDATE \"LiteLLM_AccountingDelivery\" SET state = %s WHERE event_id = 'cost'", (blocked,) - ) - if blocked == "recent_cost": - connection.execute( - "UPDATE \"LiteLLM_AccountingEvent\" SET recorded_at = %s WHERE event_id = 'cost'", (_CUTOFF,) - ) - before: Final = _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') - assert ( - await store.purge_completed( - redelivery_before=_CUTOFF, recovery_before=_AT if blocked == "recovery" else _CUTOFF - ) - == 0 - ) - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == before - - -@pytest.mark.asyncio -async def test_retention_removes_completed_family_and_receipts_together(writer: _Writer, ledger_database: str) -> None: - _, store = writer - await _insert(store, _usage()) - await _insert(store, _cost()) - _complete_and_age(ledger_database) - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(0,)] - - -@pytest.mark.asyncio -async def test_retention_respects_batch_bound_and_skips_locked_families(writer: _Writer, ledger_database: str) -> None: - client, store = writer - for name in ("a", "b", "c"): - await _insert(store, _usage(name, inbound=True)) - _complete_and_age(ledger_database) - async with client.writer_db.tx() as tx: - await tx.query_raw("SELECT event_id FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'a' FOR UPDATE") - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF, limit=1) == 1 - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ - ("a",), - ("c",), - ] - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 - - -@pytest.mark.asyncio -async def test_retention_makes_progress_past_pending_and_unpriced_events(writer: _Writer, ledger_database: str) -> None: - _, store = writer - await _insert(store, _usage("a", inbound=True)) - await _insert(store, _usage("b")) - await _insert(store, _usage("c", inbound=True)) - _complete_and_age(ledger_database) - _rows(ledger_database, "UPDATE \"LiteLLM_AccountingDelivery\" SET state = 'pending' WHERE event_id = 'a'") - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF, limit=1) == 1 - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ - ("a",), - ("b",), - ] - - -@pytest.mark.asyncio -async def test_zero_cost_and_empty_delivery_sets_are_durable(writer: _Writer, ledger_database: str) -> None: - _, store = writer - await _insert(store, _usage(), targets=()) - event: Final = _cost(amount="0") - assert await _insert(store, event, targets=()) == "inserted" - assert await _insert(store, event, targets=()) == "duplicate" - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(0,)] - _complete_and_age(ledger_database) - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 - - -@pytest.mark.asyncio -async def test_retention_cannot_delete_source_while_cost_acceptance_is_uncommitted( - writer: _Writer, ledger_database: str -) -> None: - client, store = writer - await _insert(store, _usage()) - _complete_and_age(ledger_database) - inserted: Final = asyncio.Event() - release: Final = asyncio.Event() - - @asynccontextmanager - async def paused_commit() -> AsyncGenerator[Prisma]: - async with client.writer_db.tx() as tx: - yield tx - inserted.set() - await release.wait() - - task: Final = asyncio.create_task(_insert(AccountingLedger(paused_commit), _cost())) - try: - await asyncio.wait_for(inserted.wait(), timeout=5) - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 0 - finally: - release.set() - await task - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent" ORDER BY event_id') == [ - ("cost",), - ("usage",), - ] - - -@pytest.mark.asyncio -async def test_retention_failure_rolls_back_event_and_receipt_deletion(writer: _Writer, ledger_database: str) -> None: - client, store = writer - await _insert(store, _usage(inbound=True)) - _complete_and_age(ledger_database) - - @asynccontextmanager - async def interrupted() -> AsyncGenerator[Prisma]: - async with client.writer_db.tx() as tx: - yield tx - raise RuntimeError("injected cleanup failure") - - with pytest.raises(RuntimeError, match="cleanup failure"): - await AccountingLedger(interrupted).purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(1,)] - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 1 - - -@pytest.mark.asyncio -async def test_ledger_reads_and_writes_use_primary_when_replica_is_configured( - writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch -) -> None: - with scratch_database("replica") as replica_url: - monkeypatch.setenv("DATABASE_URL_READ_REPLICA", replica_url) - async with connected_proxy_prisma(ledger_database) as client: - store: Final = AccountingLedger.for_client(client) - await _insert(store, _usage()) - assert await _insert(store, _usage()) == "duplicate" - assert await _insert(store, _cost()) == "inserted" - _complete_and_age(ledger_database) - assert await store.purge_completed(redelivery_before=_CUTOFF, recovery_before=_CUTOFF) == 2 - assert _rows(replica_url, "SELECT to_regclass('public.\"LiteLLM_AccountingEvent\"')") == [(None,)] - - -def _index_conditions(plan: object) -> tuple[tuple[object, object], ...]: - if isinstance(plan, dict): - own: Final = ((plan.get("Index Name"), plan.get("Index Cond")),) if "Index Name" in plan else () - children: Final = (plan.get("Plan"), *plan.get("Plans", ())) - return own + tuple(chain.from_iterable(_index_conditions(child) for child in children if child is not None)) - return tuple(chain.from_iterable(_index_conditions(item) for item in plan)) if isinstance(plan, list) else () - - -@pytest.mark.parametrize("statement", [accounting_ledger._DELETE_EVENTS, accounting_ledger._DELETE_DELIVERIES]) -def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str, statement: str) -> None: - explained: Final = "EXPLAIN (FORMAT JSON) " + statement.replace("$1", "%s") - with psycopg.connect(ledger_database) as connection: - connection.execute("SET enable_seqscan = off") - plan: Final = connection.execute(explained, ("usage",) * statement.count("$1")).fetchone() - assert plan is not None - conditions: Final = _index_conditions(plan[0]) - assert any( - name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition) - for name, condition in conditions - ), conditions - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "conflict"]) -async def test_usage_batch_is_atomic_and_replay_preserves_receipts( - writer: _Writer, ledger_database: str, failure: str -) -> None: - client, store = writer - events: Final = ( - _usage("caller", inbound=True), - _usage("attempt").model_copy( - update={"metrics": (Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),)} - ), - ) - writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events) - - @asynccontextmanager - async def interrupted() -> AsyncGenerator[Prisma]: - async with client.writer_db.tx() as tx: - yield tx - if failure == "before_commit": - raise RuntimeError("injected pre-commit failure") - if failure == "after_commit": - raise RuntimeError("injected lost acknowledgement") - - if failure == "conflict": - await store.insert_usage_batch((writes[1],)) - _complete_and_age(ledger_database) - rejected: Final = (writes[0], UsageWrite(events[1], _envelope(events[1], context={"changed": True}), _TARGETS)) - with pytest.raises(AccountingConflictError): - await store.insert_usage_batch(rejected) - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("attempt",)] - else: - with pytest.raises(RuntimeError, match="injected"): - await AccountingLedger(interrupted).insert_usage_batch(writes) - count: Final = 2 if failure == "after_commit" else 0 - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(count,)] - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(count * len(_TARGETS),)] - await store.insert_usage_batch(writes) - recovered: Final = await store.load_usage_batch(tuple(event.event_id for event in events)) - assert len(recovered) == 1 - assert recovered[0].event == events[1] - assert json.loads(recovered[0].envelope) == json.loads(writes[1].envelope) - assert set(recovered[0].targets) == set(_TARGETS) - _complete_and_age(ledger_database) - before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') - await store.insert_usage_batch((*writes, *writes)) - assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)] - - -@pytest.mark.asyncio -async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit( - writer: _Writer, ledger_database: str -) -> None: - from prisma.errors import DataError - - from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings - - _, store = writer - invalid: Final = _usage("invalid") - valid: Final = _usage("valid") - meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2)) - rejected: Final = meter.submit( - (UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),) - ) - accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),)) - try: - with pytest.raises(DataError): - await rejected - await accepted - finally: - await meter.close() - assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)] - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] - assert (meter.pending_bytes, meter.pending_records) == (0, 0) diff --git a/tests/integration/database/test_spendlogs_delivery.py b/tests/integration/database/test_spendlogs_delivery.py new file mode 100644 index 00000000000..19f01a6b902 --- /dev/null +++ b/tests/integration/database/test_spendlogs_delivery.py @@ -0,0 +1,512 @@ +import asyncio +import os +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import psycopg +import pytest +import pytest_asyncio +from prisma import Prisma + +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerTransaction, +) +from litellm.repositories.spend_log_insert import spend_log_insert_statement +from tests.integration._support.database import scratch_database + +_MIGRATIONS: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/migrations" + + +@pytest.fixture +def delivery_database(request: pytest.FixtureRequest) -> Iterator[psycopg.Connection[tuple[object, ...]]]: + with scratch_database("spendlogs_delivery") as url, psycopg.connect(url, autocommit=True) as connection: + if getattr(request, "param", "public") == "ledger_tenant": + connection.execute("CREATE SCHEMA ledger_tenant") + connection.execute("SET search_path TO ledger_tenant") + connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text()) + connection.execute("""CREATE TABLE "LiteLLM_SpendLogs" ( + request_id text PRIMARY KEY, call_type text NOT NULL, spend float8 NOT NULL DEFAULT 0, + status text, + metadata jsonb DEFAULT '{}', "startTime" timestamp(3) NOT NULL, + created_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP + )""") + connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text()) + yield connection + + +def _insert( + connection: psycopg.Connection[tuple[object, ...]], rows: Sequence[Mapping[str, object]], *, ledger: bool = True +) -> None: + from psycopg.types.json import Jsonb + + statement: Final = spend_log_insert_statement(rows) + with connection.transaction(): + inserted: Final = { + row[0] for row in connection.execute(statement[0].replace("$1", "%s"), (statement[1],)).fetchall() + } + for row in rows: + if ledger and row["request_id"] in inserted: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) ' + "VALUES (%s, %s, %s, %s)", + (row["request_id"], row["request_id"], Jsonb(dict(row)), row.get("call_type") == "aretrieve_batch"), + ) + + +def _with_payload(row: Mapping[str, object]) -> Mapping[str, object]: + return {**row, "accounting_payload": dict(row)} + + +def _row(request_id: str) -> Mapping[str, object]: + return { + "request_id": request_id, + "call_type": "acompletion", + "spend": 0.5, + "startTime": "2026-11-02T12:00:00+00:00", + "metadata": '{"accounting_savings":{"compression":0.2}}', + } + + +@pytest.mark.parametrize("call_type", ["acompletion", "aretrieve_batch"]) +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.parametrize("partitioned", [False, True]) +@pytest.mark.asyncio +async def test_delivery_first_acceptance_deduplicates_all_storage_modes( + delivery_database, delivery_client, mode, partitioned, call_type +): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + if partitioned: + delivery_database.execute('ALTER TABLE "LiteLLM_SpendLogs" RENAME TO spend_template') + delivery_database.execute( + 'CREATE TABLE "LiteLLM_SpendLogs" (LIKE spend_template INCLUDING DEFAULTS INCLUDING CONSTRAINTS, PRIMARY KEY(request_id, "startTime")) PARTITION BY RANGE ("startTime")' + ) + delivery_database.execute('CREATE TABLE "LiteLLM_SpendLogs_p20261102" PARTITION OF "LiteLLM_SpendLogs" DEFAULT') + delivery_database.execute("DROP TABLE spend_template") + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + row = {**_with_payload({**_row("shared"), "call_type": call_type}), "accounting_log_disabled": mode == "disabled"} + results = await asyncio.gather( + repository.insert_with_deliveries((row,), write_spend_logs=mode != "url"), + repository.insert_with_deliveries( + ({**row, "startTime": "2026-11-03T12:00:00"},), write_spend_logs=mode != "url" + ), + ) + assert sorted(results) == [0, 1] + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(mode == "local"),) + assert delivery_database.execute( + "SELECT accounting_payload->>'spend', state::text FROM \"LiteLLM_SpendLogDelivery\"" + ).fetchall() == [("0.5", "pending")] + if partitioned: + delivery_database.execute('DROP TABLE "LiteLLM_SpendLogs_p20261102"') + else: + delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"') + + async def apply(transaction, deliveries): + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(deliveries[0].row["spend"])) + + worker = AccountingDeliveryRepository(delivery_client) + assert (await worker.apply_pending(apply)).applied == 1 + assert (await worker.apply_pending(apply)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.asyncio +async def test_duplicate_rows_in_one_flush_keep_first_accounting_payload(delivery_database, delivery_client, mode): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + first = {**_with_payload(_row("repeated")), "accounting_log_disabled": mode == "disabled"} + later = { + **_with_payload({**_row("repeated"), "spend": 9.0, "startTime": "2026-11-03T12:00:00+00:00"}), + "accounting_log_disabled": mode == "disabled", + } + assert await repository.insert_with_deliveries((first, later), write_spend_logs=mode != "url") == 1 + assert delivery_database.execute( + 'SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"' + ).fetchone() == (first["accounting_payload"],) + assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == ( + [(first["spend"],)] if mode == "local" else [] + ) + + +@pytest.mark.parametrize(("legacy_spend", "accepted"), [(0.5, 0), (0.0, 1)]) +@pytest.mark.asyncio +async def test_batch_charged_before_the_ledger_is_not_charged_again( + delivery_database, delivery_client, legacy_spend, accepted +): + from types import SimpleNamespace + + from litellm.repositories.table_repositories import SpendLogsRepository + + batch: Final = {**_row("batch_legacy"), "call_type": "aretrieve_batch", "status": "success"} + _insert(delivery_database, ({**batch, "spend": legacy_spend},), ledger=False) + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + assert await repository.insert_with_deliveries((_with_payload({**batch, "spend": 1.0}),)) == accepted + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (accepted,) + assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [(legacy_spend,)] + + +@pytest.mark.parametrize("table", ["LiteLLM_SpendLogs", "LiteLLM_SpendLogDelivery"]) +@pytest.mark.asyncio +async def test_transaction_a_rolls_back_payload_and_log_on_failure(delivery_database, delivery_client, table): + from types import SimpleNamespace + + from prisma.errors import PrismaError + from psycopg import sql + + from litellm.repositories.table_repositories import SpendLogsRepository + + delivery_database.execute( + sql.SQL("ALTER TABLE {} ADD CONSTRAINT reject_poison CHECK (request_id <> 'poison')").format( + sql.Identifier(table) + ) + ) + repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client)) + with pytest.raises(PrismaError): + await repository.insert_with_deliveries(tuple(_with_payload(_row(name)) for name in ("good", "poison"))) + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + + +@pytest.mark.asyncio +async def test_worker_rolls_back_destinations_and_parks_only_isolated_failure( + delivery_database: psycopg.Connection[tuple[object, ...]], +) -> None: + url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + client: Final = Prisma(datasource={"url": url}) + delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)") + delivery_database.execute("INSERT INTO totals VALUES (0)") + _insert(delivery_database, (_row("bad"), _row("good"))) + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(str(row.row["spend"])) for row in rows) + ) + if any(row.row["request_id"] == "bad" for row in rows): + raise ValueError("Invalid projected destination") + + await client.connect() + try: + repository: Final = AccountingDeliveryRepository(client) + failed: Final = await repository.apply_pending(apply, max_attempts=2) + assert failed.failed == 2 + assert failed.parked == () + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + assert delivery_database.execute( + 'SELECT DISTINCT state::text, attempts FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("pending", 1)] + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + isolated: Final = await repository.apply_pending(apply, max_attempts=2) + assert isolated.parked == ("bad",) + applied: Final = await repository.apply_pending(apply, max_attempts=2) + assert applied.applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + assert delivery_database.execute( + 'SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id' + ).fetchall() == [("bad", "parked"), ("good", "applied")] + assert (await repository.apply_pending(apply)).applied == 0 + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_a_locked_database_never_parks_a_valid_delivery(delivery_database, delivery_client): + _insert(delivery_database, (_row("valid"),)) + + async def apply(transaction, deliveries): + await transaction.execute_raw("SET LOCAL lock_timeout = '50ms'") + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries) + ) + + repository = AccountingDeliveryRepository(delivery_client) + with psycopg.connect( + urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + ) as locker: + locker.execute("LOCK TABLE totals IN ACCESS EXCLUSIVE MODE") + for _ in range(4): + result = await repository.apply_pending(apply, max_attempts=2) + assert result.failed == 1 and result.parked == () + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("pending", 4) + ] + assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_an_unreachable_database_reported_as_a_data_error_never_parks_a_delivery( + delivery_database, delivery_client +): + from prisma.errors import DataError + + _insert(delivery_database, (_row("valid"),)) + + async def unreachable(transaction, deliveries): + raise DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}}) + + repository = AccountingDeliveryRepository(delivery_client) + for _ in range(3): + assert (await repository.apply_pending(unreachable, max_attempts=2)).parked == () + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP') + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("pending", 3) + ] + + +@pytest_asyncio.fixture(loop_scope="function") +async def delivery_client(delivery_database: psycopg.Connection[tuple[object, ...]]) -> AsyncIterator[Prisma]: + url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}")) + client: Final = Prisma(datasource={"url": url}) + delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)") + delivery_database.execute("INSERT INTO totals VALUES (0)") + await client.connect() + try: + yield client + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_two_workers_claim_disjoint_rows_and_apply_once( + delivery_database: psycopg.Connection[tuple[object, ...]], + delivery_client: Prisma, +) -> None: + _insert(delivery_database, (_row("first"), _row("second"))) + entered: Final[asyncio.Queue[str]] = asyncio.Queue() + release: Final = asyncio.Event() + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + assert len(rows) == 1 + await entered.put(str(rows[0].row["request_id"])) + await release.wait() + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + repository: Final = AccountingDeliveryRepository(delivery_client) + first: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1)) + first_id: Final = await asyncio.wait_for(entered.get(), timeout=10) + second: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1)) + try: + second_id: Final = await asyncio.wait_for(entered.get(), timeout=10) + assert first_id != second_id + finally: + release.set() + results: Final = await asyncio.gather(first, second) + assert sum(result.applied for result in results) == 2 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (1.0,) + assert delivery_database.execute( + "SELECT COUNT(*) FROM \"LiteLLM_SpendLogDelivery\" WHERE state = 'applied'" + ).fetchone() == (2,) + + +@pytest.mark.asyncio +async def test_cancelled_worker_rolls_back_and_restart_applies_once( + delivery_database: psycopg.Connection[tuple[object, ...]], + delivery_client: Prisma, +) -> None: + _insert(delivery_database, (_row("restart"),)) + wrote: Final = asyncio.Event() + hold: Final = asyncio.Event() + + async def interrupted(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + wrote.set() + await hold.wait() + + repository: Final = AccountingDeliveryRepository(delivery_client) + task: Final = asyncio.create_task(repository.apply_pending(interrupted)) + await asyncio.wait_for(wrote.wait(), timeout=10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchone() == ( + "pending", + 0, + ) + + async def complete(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + assert (await repository.apply_pending(complete)).applied == 1 + assert (await repository.apply_pending(complete)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_worker_applies_accounting_after_spend_logs_are_deleted( + delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma +) -> None: + _insert(delivery_database, (_row("removed"),)) + delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"') + + async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None: + await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"]))) + + repository: Final = AccountingDeliveryRepository(delivery_client) + assert (await repository.apply_pending(apply)).applied == 1 + assert (await repository.apply_pending(apply)).applied == 0 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_receipt_cleanup_keeps_pending_and_parked_rows_and_reports_lag( + delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma +) -> None: + _insert(delivery_database, tuple(_row(state) for state in ("pending", "parked", "applied"))) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" ' + 'SET state = request_id::"AccountingDeliveryState", ' + "created_at = CURRENT_TIMESTAMP - INTERVAL '40 days', " + "applied_at = CURRENT_TIMESTAMP - INTERVAL '40 days'" + ) + repository = AccountingDeliveryRepository(delivery_client) + assert await repository.cleanup_applied() == 1 + metrics = await repository.metrics() + assert metrics.pending_count == 1 + assert metrics.parked_count == 1 + assert metrics.oldest_pending_seconds >= 40 * 86400 + assert delivery_database.execute( + 'SELECT request_id FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id' + ).fetchall() == [("parked",), ("pending",)] + + +@pytest.mark.parametrize("delivery_database", ["public", "ledger_tenant"], indirect=True) +@pytest.mark.parametrize("state", [None, "pending", "parked", "applied"]) +def test_delivery_migration_replay_preserves_ledger_receipts(delivery_database, state): + if state is not None: + _insert(delivery_database, (_row("replay"),)) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET state=%s, attempts=3, last_error=%s', + (state, "retained failure"), + ) + before = delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall() + migration = (_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text() + delivery_database.execute(migration) + delivery_database.execute(migration) + assert delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall() == before + assert delivery_database.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(state is not None),) + + +@pytest.mark.parametrize("state", ["pending", "parked"]) +def test_delivery_migration_preserves_unused_old_receipts(state): + with scratch_database("spendlogs_old_delivery") as url, psycopg.connect(url, autocommit=True) as connection: + connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text()) + connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id text PRIMARY KEY)') + connection.execute( + 'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state) VALUES (%s, %s, 1, %s)', + ("old-event", "LiteLLM_UserTable", state), + ) + connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text()) + assert connection.execute('SELECT event_id, state::text FROM "LiteLLM_AccountingDelivery"').fetchall() == [ + ("old-event", state) + ] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + + +@pytest.mark.asyncio +async def test_malformed_payload_is_parked_without_losing_other_deliveries(delivery_database, delivery_client): + _insert(delivery_database, (_row("bad"), _row("good"))) + delivery_database.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET accounting_payload=$$ [] $$::jsonb WHERE request_id=$$bad$$' + ) + + async def apply(transaction, deliveries): + await transaction.execute_raw( + "UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries) + ) + + repository = AccountingDeliveryRepository(delivery_client) + assert (await repository.apply_pending(apply, max_attempts=2)).failed == 2 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,) + delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at=CURRENT_TIMESTAMP') + assert (await repository.apply_pending(apply, max_attempts=2)).parked == ("bad",) + assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1 + assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,) + + +@pytest.mark.asyncio +async def test_retention_keeps_batch_receipts_indefinitely(delivery_database, delivery_client): + _insert(delivery_database, (_row("normal"), {**_row("batch"), "call_type": "aretrieve_batch"})) + delivery_database.execute( + "UPDATE \"LiteLLM_SpendLogDelivery\" SET state='applied', applied_at=CURRENT_TIMESTAMP - INTERVAL '400 days'" + ) + repository = AccountingDeliveryRepository(delivery_client) + assert await repository.cleanup_applied() == 1 + assert await repository.cleanup_applied() == 0 + assert delivery_database.execute('SELECT request_id FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("batch",)] + metrics = await repository.metrics() + assert metrics.batch_count == 1 + assert metrics.table_bytes > 0 + + +@pytest.mark.parametrize("disabled", [False, True]) +@pytest.mark.asyncio +async def test_ledger_batch_retrieves_charge_once_in_every_logging_mode( + delivery_database, delivery_client, monkeypatch, disabled +): + from types import SimpleNamespace + + from litellm.proxy import proxy_server + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.utils import jsonify_object + + completed = { + **_row("batch_completed"), "call_type": "aretrieve_batch", "status": "success", "spend": 0.5, + "metadata": '{"status":"success","private":"request detail"}', + "accounting_payload": { + "request_id": "batch_completed", "call_type": "aretrieve_batch", "status": "success", "spend": 0.5, + "metadata": {"accounting_savings": {"compression": 0.2}}, + }, + } + monkeypatch.setattr(proxy_server, "general_settings", {"spend_logs_metadata_fields": {"include": ["status"]}}) + client = SimpleNamespace(db=delivery_client, writer_db=delivery_client, jsonify_object=jsonify_object) + writer = DBSpendUpdateWriter() + first = await writer._claim_settle_once_cost_row(payload=completed, prisma_client=client, disable_spend_logs=disabled) + again = await writer._claim_settle_once_cost_row( + payload={**completed, "startTime": "2026-11-05T12:00:00+00:00"}, prisma_client=client, disable_spend_logs=disabled + ) + assert (first, again) == (True, False) + assert delivery_database.execute('SELECT spend, metadata FROM "LiteLLM_SpendLogs"').fetchall() == ( + [] if disabled else [(0.5, {"status": "success"})] + ) + assert delivery_database.execute( + 'SELECT delivery_id, accounting_payload FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("batch_completed", completed["accounting_payload"])] + + +@pytest.mark.parametrize("setting", ["retention_days", "batch_size"]) +@pytest.mark.asyncio +async def test_invalid_cleanup_settings_preserve_receipts(delivery_database, delivery_client, setting): + _insert(delivery_database, (_row("protected"),)) + with pytest.raises(ValueError, match="positive"): + await AccountingDeliveryRepository(delivery_client).cleanup_applied(**{setting: 0}) + assert delivery_database.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("protected", "pending") + ] + + +@pytest.mark.parametrize("setting", ["max_attempts", "batch_size"]) +@pytest.mark.asyncio +async def test_invalid_worker_settings_preserve_pending_work(delivery_database, delivery_client, setting): + from unittest.mock import AsyncMock + + _insert(delivery_database, (_row("protected"),)) + apply = AsyncMock() + with pytest.raises(ValueError, match="positive"): + await AccountingDeliveryRepository(delivery_client).apply_pending(apply, **{setting: 0}) + apply.assert_not_awaited() + assert delivery_database.execute( + 'SELECT request_id, state::text, attempts FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("protected", "pending", 0)] diff --git a/tests/integration/database/test_spendlogs_ledger_parity.py b/tests/integration/database/test_spendlogs_ledger_parity.py new file mode 100644 index 00000000000..e1894aa66bf --- /dev/null +++ b/tests/integration/database/test_spendlogs_ledger_parity.py @@ -0,0 +1,1437 @@ +import contextlib +import asyncio +import json +import os +import uuid +import subprocess +import sys +import time +from collections.abc import Iterator +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import psycopg +from psycopg import sql +import pytest +import litellm + +from litellm.accounting.application import apply_deliveries +from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.proxy.spend_tracking.spend_tracking_utils import spend_log_row_with_retained_metadata +from litellm.proxy import proxy_server +from litellm.proxy._types import SpendLogsMetadataFields +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.model_usage_rollup import flush_model_usage_transactions +from litellm.proxy.guardrails.usage_tracking import PendingRollups, process_spend_logs_guardrail_usage +from litellm.proxy.utils import PrismaClient +from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows +from litellm.proxy.hooks.model_max_budget_limiter import PROXY_VirtualKeyModelMaxBudgetLimiter +from litellm.repositories.accounting_delivery_repository import AccountingDeliveryRepository +from litellm.repositories.table_repositories import SpendLogsRepository +from litellm.responses.streaming_iterator import drain_logging_tasks +from litellm.types.utils import ModelResponse, Usage +from tests.integration._support.database import scratch_database + +_ROOT: Final = Path(__file__).resolve().parents[3] +_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) +_ENTITY_TABLES: Final = ( + "LiteLLM_VerificationToken", + "LiteLLM_UserTable", + "LiteLLM_TeamTable", + "LiteLLM_TeamMembership", + "LiteLLM_OrganizationTable", + "LiteLLM_OrganizationMembership", + "LiteLLM_EndUserTable", + "LiteLLM_AgentsTable", + "LiteLLM_TagTable", + "LiteLLM_ProjectTable", + "LiteLLM_ModelAccessGroupBudgetTable", +) +_DAILY_TABLES: Final = ( + "LiteLLM_DailyUserSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyEndUserSpend", + "LiteLLM_DailyAgentSpend", + "LiteLLM_DailyTagSpend", + "LiteLLM_DailyModelUsage", + "LiteLLM_DailyGuardrailUsageUnits", + "LiteLLM_BudgetWindowSpend", +) + + +@pytest.mark.parametrize("batch_size,row_count,expected,distinct_models", [(None, 251, 251, False), ("200", 251, 200, False), (None, 10001, 10000, False), (None, 3001, 3001, True)]) +@pytest.mark.asyncio +async def test_worker_applies_the_configured_backlog_in_one_transaction( + parity_database: str, monkeypatch: pytest.MonkeyPatch, batch_size: str | None, row_count: int, expected: int, distinct_models: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + if batch_size is None: + monkeypatch.delenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", raising=False) + else: + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", batch_size) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + stored = await client.writer_db.query_raw('SELECT to_jsonb(s) AS row FROM "LiteLLM_SpendLogs" s') + delivery = (await client.writer_db.query_raw('SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"'))[0]["accounting_payload"] + assert await SpendLogsRepository(client).insert_with_deliveries(tuple( + {**stored[0]["row"], "request_id": f"backlog-{index:05}", "model": f"backlog-model-{index}" if distinct_models else stored[0]["row"]["model"], "accounting_payload": {**delivery, "request_id": f"backlog-{index:05}", "model": f"backlog-model-{index}" if distinct_models else delivery["model"]}} for index in range(row_count - 1) + )) == row_count - 1 + started = time.monotonic() + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + print(f"Ledger batch applied={expected} queued={row_count} seconds={time.monotonic() - started:.3f}") + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'applied\'').fetchone() == (expected,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'pending\'').fetchone() == (row_count - expected,) + assert connection.execute('SELECT COUNT(DISTINCT applied_at) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (1,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (expected * 0.25,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend"').fetchone() == (expected if distinct_models else 1,) + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("ledger", [False, True]) +@pytest.mark.parametrize("exclude_messages", [False, True]) +@pytest.mark.asyncio +async def test_registered_proxy_callbacks_charge_model_windows_before_persisting_spend( + parity_database: str, monkeypatch: pytest.MonkeyPatch, ledger: bool, exclude_messages: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.utils import Rules, function_setup + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "false") + for name in ("callbacks", "input_callback", "success_callback", "failure_callback", "_async_success_callback", "_async_failure_callback"): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages"] if exclude_messages else []) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server.proxy_logging_obj, "db_spend_update_writer", writer) + limiter = PROXY_VirtualKeyModelMaxBudgetLimiter( + dual_cache=DualCache(), clock=lambda: _NOW.timestamp(), + shared_windows=DatabaseSharedWindows(prisma_client_source=lambda: client), + ) + monkeypatch.setattr(proxy_server, "model_max_budget_limiter", limiter) + litellm.logging_callback_manager.add_litellm_callback(limiter) + proxy_server.cost_tracking() + logging, _ = function_setup("acompletion", Rules(), _NOW, model="ledger-test", messages=[], litellm_call_id="registered-window") + logging.update_environment_variables( + optional_params={}, custom_llm_provider="openai", + litellm_params={"metadata": {"user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", "user_api_key_model_max_budget": { + "ledger-test": {"max_budget": 100, "budget_duration": "1d"} + }}}, + ) + logging.model_call_details["response_cost"] = 0.25 + response = ModelResponse(id="registered-window", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5)) + await client.db.connect() + try: + await logging.async_success_handler(response, _NOW, _NOW, standard_logging_object={ + "model": "ledger-test", "model_group": "ledger-test", "response_cost": 0.25, + "metadata": {"user_api_key_hash": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}, + "model_map_information": {}, "hidden_params": {}, "request_tags": [], + }) + await drain_logging_tasks() + await SpendLogsRepository(client).insert_with_deliveries(tuple(client.spend_log_transactions)) + if ledger: + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_BudgetWindowSpend" WHERE entity_type=\'model_max_budget\'').fetchall() == [(0.25,)] + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [(0.25,)] + finally: + await client.db.disconnect() + + +@pytest.fixture(scope="module") +def ledger_schema() -> str: + return subprocess.run( + [ + sys.executable, + "-I", + "-m", + "prisma", + "migrate", + "diff", + "--from-empty", + "--to-schema-datamodel", + str(_ROOT / "schema.prisma"), + "--script", + ], + check=True, + capture_output=True, + text=True, + cwd=_ROOT, + ).stdout + + +@pytest.fixture +def parity_database(ledger_schema: str) -> Iterator[str]: + with scratch_database("spendlogs_parity") as url, psycopg.connect(url, autocommit=True) as connection: + connection.execute(ledger_schema) + connection.execute( + "INSERT INTO \"LiteLLM_BudgetTable\" (budget_id, created_by, updated_by) VALUES ('budget', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_UserTable\" (user_id) VALUES ('user')") + connection.execute("INSERT INTO \"LiteLLM_TeamTable\" (team_id) VALUES ('team')") + connection.execute( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by) " + "VALUES ('org', 'org', 'budget', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_TeamMembership\" (team_id, user_id) VALUES ('team', 'user')") + connection.execute( + "INSERT INTO \"LiteLLM_OrganizationMembership\" (organization_id, user_id) VALUES ('org', 'user')" + ) + connection.execute("INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683')") + connection.execute("INSERT INTO \"LiteLLM_EndUserTable\" (user_id) VALUES ('end-user')") + connection.execute( + 'INSERT INTO "LiteLLM_AgentsTable" ' + "(agent_id, agent_name, agent_card_params, created_by, updated_by) " + "VALUES ('agent', 'agent', '{}', 'test', 'test')" + ) + connection.execute( + 'INSERT INTO "LiteLLM_ProjectTable" (project_id, created_by, updated_by) ' + "VALUES ('project', 'test', 'test')" + ) + connection.execute("INSERT INTO \"LiteLLM_TagTable\" (tag_name) VALUES ('tag')") + connection.execute("INSERT INTO \"LiteLLM_ModelAccessGroupBudgetTable\" (access_group_name) VALUES ('group')") + yield url + + +def _snapshot(url: str) -> dict[str, list[object]]: + with psycopg.connect(url) as connection: + return { + table: [ + row[0] + for row in connection.execute( + sql.SQL( + "SELECT to_jsonb(t) - 'id' - 'created_at' - 'updated_at' - 'request_id' - 'last_active' FROM {} t " + "ORDER BY (to_jsonb(t) - 'id' - 'created_at' - 'updated_at' - 'request_id' - 'last_active')::text" + ).format(sql.Identifier(table)) + ).fetchall() + ] + for table in (*_ENTITY_TABLES, *_DAILY_TABLES) + } + + +def _reset(url: str) -> None: + with psycopg.connect(url, autocommit=True) as connection: + for table in _ENTITY_TABLES: + connection.execute(sql.SQL("UPDATE {} SET spend=0").format(sql.Identifier(table))) + for table in (*_DAILY_TABLES, "LiteLLM_SpendLogs", "LiteLLM_SpendLogDelivery"): + connection.execute(sql.SQL("TRUNCATE {}").format(sql.Identifier(table))) + connection.execute('UPDATE "LiteLLM_VerificationToken" SET total_spend=0') + connection.execute('UPDATE "LiteLLM_TeamMembership" SET total_spend=0') + + +async def _assert_legacy_database_parity( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, + status: str, + origin: str | None, + cost: float, + retention: str, + reset_kind: str | None = None, + correction_order: str | None = None, + toggle_after_capture: bool = False, + fallback_narrowing: bool = False, + failing_table: str | None = None, + organization_id: str | None = "org", + export_url: str | None = None, + export_failure: bool = False, +): + monkeypatch.setitem( + litellm.model_cost, + "ledger-test", + { + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.004, + "cache_read_input_token_cost": 0.0005, + "cache_creation_input_token_cost": 0.003, + "litellm_provider": "openai", + "mode": "chat", + "max_tokens": 4096, + }, + ) + if fallback_narrowing: + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=[{ + "model_name": "ledger-test", + "litellm_params": {"model": "openai/ledger-test", "api_key": "test"}, + "model_info": {"id": "deployment", "access_groups": ["group"]}, + }])) + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + "INSERT INTO \"LiteLLM_ModelAccessGroupBudgetTable\" (access_group_name) VALUES ('unserved-group')" + ) + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + await client.db.connect() + results = [] + try: + for path in ("legacy", "ledger"): + _reset(parity_database) + if reset_kind is not None: + _prime_reset(parity_database, reset_kind) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": path == "ledger"}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", path == "ledger" and retention == "lean") + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server.proxy_logging_obj, "db_spend_update_writer", writer) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + monkeypatch.setattr(proxy_server, "spend_counter_cache", DualCache()) + for cache_key in ("2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", "team_id:team"): + await proxy_server.user_api_key_cache.async_set_cache( + key=cache_key, + value={ + "budget_limits": [ + {"budget_duration": "1d", "max_budget": 100, "reset_at": "2026-11-04T00:00:00+00:00"} + ] + }, + ) + request_kwargs = { + "model": "ledger-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "request", + "standard_logging_object": { + "model": "ledger-test", + "model_group": "ledger-test", + "metadata": { + "user_api_key_hash": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_read_input_tokens": 4, + "cache_creation_input_tokens": 1, + }, + }, + "model_map_information": {}, + "hidden_params": {}, + "request_tags": ["tag"], + "model_id": "deployment", + "response_cost": cost, + "autorouter_savings": 0.125, + "guardrail_information": [ + { + "guardrail_id": "guard", + "guardrail_name": "guard", + "guardrail_status": "success", + "guardrail_usage": {"text": 2}, + "guardrail_cost_by_unit": {"text": 0.02}, + "guardrail_cost_in_spend": True, + } + ], + }, + "agent_id": "agent", + "user": "end-user", + "litellm_params": { + "metadata": { + "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + "user_api_key_model_max_budget": {"ledger-test": {"max_budget": 100, "budget_duration": "1d"}}, + "user_api_key_user_id": "user", + "user_api_key_team_id": "team", + "user_api_key_org_id": organization_id, + "user_api_key_project_id": "project", + "user_api_key_end_user_id": "end-user", + "tags": ["tag"], + "user_api_key_matched_model_access_groups": ["group", "unserved-group"] if fallback_narrowing else ["group"], + "model_info": {"id": "deployment"}, + "litellm_gateway_injected_cache": "deployment", + "compression_savings": {"tokens_before": 20, "tokens_after": 10, "tokens_saved": 10}, + "status": status, + **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if origin == "billed_retry" else {}), + **({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin} if origin == "auto_router" else {}), + }, + "user": "end-user", + }, + } + limiter = PROXY_VirtualKeyModelMaxBudgetLimiter( + dual_cache=proxy_server.spend_counter_cache, + clock=lambda: _NOW.timestamp(), + shared_windows=DatabaseSharedWindows(prisma_client_source=lambda: client), + ) + if status == "success": + await limiter.async_log_success_event(request_kwargs, None, _NOW, _NOW) + charged = await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + user_id="user", + end_user_id="end-user", + team_id="team", + org_id=organization_id, + project_id="project", + kwargs=request_kwargs, + completion_response=ModelResponse( + id="request", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + response_cost=cost, + ) + assert charged + await proxy_server.increment_spend_counters( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", + team_id="team", + user_id="user", + response_cost=cost, + request_started_at=_NOW, + **({"windows_from_ledger": True} if path == "ledger" else {}), + ) + await drain_logging_tasks() + rows = tuple(client.spend_log_transactions) + client.spend_log_transactions = [] + assert len(rows) == 1 + if path == "ledger" and retention == "lean": + assert rows[0]["accounting_log_disabled"] is True + frozen = rows[0]["accounting_payload"] + if isinstance(frozen, str): + frozen = json.loads(frozen) + assert not {"messages", "response", "proxy_server_request"}.intersection(frozen) + stored_rows = ( + tuple(spend_log_row_with_retained_metadata(row, SpendLogsMetadataFields(include=())) for row in rows) + if retention == "stripped" + else rows + ) + if toggle_after_capture: + for enabled in (True, False, True): + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": enabled}) + await SpendLogsRepository(client).insert_with_deliveries(stored_rows) + elif path == "ledger" and export_url is not None: + import httpx + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("SPEND_LOGS_URL", export_url) + handler = AsyncHTTPHandler(timeout=10) + wire_rows = [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in stored_rows + ] + try: + if export_failure: + with pytest.raises(httpx.HTTPStatusError): + await ProxyUpdateSpend.update_spend_logs(0, client, handler, proxy_server.proxy_logging_obj, wire_rows) + else: + await ProxyUpdateSpend.update_spend_logs(0, client, handler, proxy_server.proxy_logging_obj, wire_rows) + finally: + await handler.close() + else: + await SpendLogsRepository(client).insert_with_deliveries(stored_rows) + if path == "ledger" and (retention == "lean" or export_url is not None): + assert await client.writer_db.query_raw('SELECT COUNT(*) AS count FROM "LiteLLM_SpendLogs"') == [{"count": 0}] + if reset_kind is not None: + await _apply_reset(reset_kind, client) + if correction_order == "before": + await _publish_savings_correction(client, stored_rows[0]) + if path != "ledger": + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + await writer.commit_daily_tag_spend_to_db(client, 0, proxy_server.proxy_logging_obj) + await flush_model_usage_transactions(prisma_client=client, transactions=client.model_usage_transactions) + client.model_usage_transactions = [] + await process_spend_logs_guardrail_usage(client, rows, pending=PendingRollups()) + else: + + async def apply(transaction, deliveries, writer: DBSpendUpdateWriter = writer): + await apply_deliveries( + transaction, + deliveries, + prisma_client=client, + writer=writer, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + + if failing_table is not None: + before_failure = _snapshot(parity_database) + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute(sql.SQL('ALTER TABLE {} ADD CONSTRAINT reject_spend CHECK (spend = 0)').format(sql.Identifier(failing_table))) + failed = await AccountingDeliveryRepository(client.writer_db).apply_pending(apply) + assert (failed.applied, failed.failed) == (0, 1) + assert _snapshot(parity_database) == before_failure + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("pending", 1)] + connection.execute(sql.SQL('ALTER TABLE {} DROP CONSTRAINT reject_spend').format(sql.Identifier(failing_table))) + connection.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at=CURRENT_TIMESTAMP') + result = await AccountingDeliveryRepository(client.writer_db).apply_pending(apply) + assert result.applied == 1, result + await writer._commit_spend_updates_to_db_without_redis_buffer(client, 0, proxy_server.proxy_logging_obj) + if correction_order == "after": + await _publish_savings_correction(client, stored_rows[0]) + results.append(_snapshot(parity_database)) + differences = { + (index, table): (results[0][table], result[table]) + for index, result in enumerate(results[1:], 1) + for table in results[0] + if results[0][table] != result[table] + } + assert not differences, differences + prior_spend = 5 if reset_kind is not None and reset_kind != "scheduled" else 0 + assert results[-1]["LiteLLM_UserTable"][0]["spend"] == cost + prior_spend + daily = results[-1]["LiteLLM_DailyUserSpend"][0] + assert daily["compression_savings_spend"] > 0, results[-1]["LiteLLM_DailyUserSpend"] + assert daily["prompt_caching_savings_spend"] != 0 + assert daily["gateway_injected_caching_savings_spend"] != 0 + assert daily["autorouter_savings_spend"] == pytest.approx( + (0 if origin == "auto_router" else 0.125) + (0.2 if correction_order else 0) + ) + group_spend = {row["access_group_name"]: row["spend"] for row in results[-1]["LiteLLM_ModelAccessGroupBudgetTable"]} + assert group_spend == {"group": cost + prior_spend, **({"unserved-group": prior_spend} if fallback_narrowing else {})} + assert results[-1]["LiteLLM_DailyGuardrailUsageUnits"] + windows = results[-1]["LiteLLM_BudgetWindowSpend"] + assert len(windows) == (2 if cost else 0) + (1 if status == "success" else 0) + assert all(row["spend"] == cost for row in windows) + if reset_kind == "window_rollover": + model_window = next(row for row in windows if row["entity_type"] == "model_max_budget") + assert model_window["window_start"].startswith("2026-11-04") + if reset_kind in ("key", "global", "scheduled"): + assert results[-1]["LiteLLM_VerificationToken"][0]["spend"] == cost + if reset_kind in ("global", "scheduled"): + assert results[-1]["LiteLLM_TeamTable"][0]["spend"] == cost + if reset_kind == "team_member": + assert results[-1]["LiteLLM_TeamMembership"][0]["spend"] == cost + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize( + "status,origin,cost", + [ + ("success", None, 0.25), + ("failure", None, 0.25), + ("success", None, 0.0), + ("success", "auto_router", 0.25), + ("failure", "billed_retry", 0.25), + ], +) +@pytest.mark.parametrize("retention", ["full", "stripped", "lean"]) +@pytest.mark.asyncio +async def test_stored_ledger_matches_real_legacy_database_updates( + parity_database: str, monkeypatch: pytest.MonkeyPatch, status: str, origin: str | None, cost: float, retention: str +): + await _assert_legacy_database_parity(parity_database, monkeypatch, status, origin, cost, retention) + + +@pytest.mark.parametrize("retention", ["full", "stripped", "lean"]) +@pytest.mark.asyncio +async def test_missing_organization_matches_legacy_daily_rollups(parity_database, monkeypatch, retention): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, retention, organization_id=None + ) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT count(*) FROM "LiteLLM_DailyOrganizationSpend"').fetchone() == (0,) + + +@pytest.mark.parametrize("reset_kind", ["scheduled", "key", "team_member", "global", "window_rollover"]) +@pytest.mark.asyncio +async def test_pending_delivery_after_reset_matches_queued_legacy_spend( + parity_database: str, monkeypatch: pytest.MonkeyPatch, reset_kind: str +): + await _assert_legacy_database_parity(parity_database, monkeypatch, "success", None, 0.25, "full", reset_kind) + + +def _prime_reset(url: str, reset_kind: str) -> None: + with psycopg.connect(url, autocommit=True) as connection: + for table in _ENTITY_TABLES: + connection.execute(sql.SQL("UPDATE {} SET spend=5").format(sql.Identifier(table))) + if reset_kind == "scheduled": + for table in ("LiteLLM_VerificationToken", "LiteLLM_UserTable", "LiteLLM_TeamTable", "LiteLLM_BudgetTable"): + connection.execute( + sql.SQL("UPDATE {} SET budget_duration='1d', budget_reset_at='2026-11-01'").format( + sql.Identifier(table) + ) + ) + connection.execute("UPDATE \"LiteLLM_ModelAccessGroupBudgetTable\" SET budget_id='budget'") + + +async def _apply_reset(reset_kind: str, client: PrismaClient) -> None: + from litellm.proxy._types import ResetSpendRequest, UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob + from litellm.proxy.management_endpoints.key_management_endpoints import reset_key_spend_fn + from litellm.proxy.management_endpoints.team_endpoints import reset_team_member_spend_fn + from litellm.proxy.spend_tracking.spend_management_endpoints import global_spend_reset + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + if reset_kind == "window_rollover": + from litellm.proxy.db.budget_window_spend_writer import roll_window_spend_row + + windows = await client.writer_db.query_raw('SELECT * FROM "LiteLLM_BudgetWindowSpend"') + assert windows + for window in windows: + await roll_window_spend_row( + client, window["entity_type"], window["entity_id"], window["window_duration"], _NOW + timedelta(days=2) + ) + elif reset_kind == "global": + await global_spend_reset() + elif reset_kind == "key": + await reset_key_spend_fn( + key="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", data=ResetSpendRequest(reset_to=0), user_api_key_dict=admin, litellm_changed_by=None + ) + elif reset_kind == "team_member": + await reset_team_member_spend_fn( + team_id="team", user_id="user", data=ResetSpendRequest(reset_to=0), user_api_key_dict=admin + ) + else: + await ResetBudgetJob(proxy_server.proxy_logging_obj, client).reset_budget() + + +@pytest.mark.parametrize("correction_order", ["before", "after"]) +@pytest.mark.asyncio +async def test_baseline_correction_order_preserves_frozen_request_savings( + parity_database: str, monkeypatch: pytest.MonkeyPatch, correction_order: str +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "full", correction_order=correction_order + ) + + +async def _publish_savings_correction(client: PrismaClient, row) -> None: + from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING + from litellm.proxy.db.baseline_accounting import ( + BaselinePublication, + DailyBaselineAttribution, + DailyBaselineTarget, + _Change, + _publish, + ) + + metadata_before = ( + await client.writer_db.query_raw( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', row["request_id"] + ) + )[0]["metadata"] + if isinstance(metadata_before, str): + metadata_before = json.loads(metadata_before) + change = _Change( + request_id=row["request_id"], + publication=BaselinePublication( + comparison_id="autorouter-baseline:v3:" + "a" * 64, + comparison_started_at=_NOW.timestamp(), + status="estimated", + reason="modeled", + provenance="modeled", + actual_spend=0.25, + baseline_spend=0.575, + ), + api_key=row["api_key"], + session_id="session", + router_name="router", + baseline_model="ledger-test", + covered_delta=0, + actual_delta=0.0, + savings_delta=0.2, + daily=DailyBaselineAttribution( + date=_NOW.date().isoformat(), + api_key=row["api_key"], + model=row["model"], + custom_llm_provider=row["custom_llm_provider"], + model_group=row["model_group"], + endpoint=ROUTE_ENDPOINT_MAPPING.get(row["call_type"]), + targets=tuple( + DailyBaselineTarget(entity=entity, entity_id=identity) + for entity, identity in ( + ("user", "user"), + ("team", "team"), + ("org", "org"), + ("end_user", "end-user"), + ("agent", "agent"), + ("tag", "tag"), + ) + ), + ), + ) + async with client.writer_db.tx() as transaction: + await _publish(transaction, (change,)) + metadata_after = ( + await client.writer_db.query_raw( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', row["request_id"] + ) + )[0]["metadata"] + if isinstance(metadata_after, str): + metadata_after = json.loads(metadata_after) + assert metadata_after["autorouter_savings"] == pytest.approx(0.325) + if "accounting_payload" in row: + delivery = (await client.writer_db.query_raw( + 'SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery" WHERE request_id=$1', row["request_id"] + ))[0]["accounting_payload"] + assert delivery["metadata"]["accounting_savings"]["autorouter"] == pytest.approx(0.125) + + +@pytest.mark.asyncio +async def test_flag_changes_do_not_reclassify_or_reapply_buffered_rows( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, +): + await _assert_legacy_database_parity( + parity_database, + monkeypatch, + "success", + None, + 0.25, + "stripped", + toggle_after_capture=True, + ) + + +@pytest.mark.parametrize("entity_type", ["key", "team"]) +@pytest.mark.asyncio +async def test_window_seed_excludes_pending_and_parked_but_preserves_legacy_cutoff( + parity_database: str, + monkeypatch: pytest.MonkeyPatch, + entity_type: str, +): + from datetime import timedelta + from litellm.proxy.db.budget_window_spend_writer import spend_logs_seed_totals, commit_window_spend_updates + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + await client.db.connect() + try: + for request_id, cost, offset in ( + ("old-legacy", 2.0, -1), + ("current-legacy", 5.0, 0), + ("applied", 3.0, 1), + ("pending", 7.0, 0), + ("parked", 11.0, 0), + ): + await client.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogs" ' + '(request_id,call_type,api_key,team_id,spend,"startTime","endTime") ' + "VALUES ($1,'acompletion','key','team',$2::float8,$3::timestamp,$3::timestamp)", + request_id, + cost, + _NOW + timedelta(seconds=offset), + ) + for request_id, state in (("applied", "applied"), ("pending", "pending"), ("parked", "parked")): + await client.writer_db.execute_raw( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id,state,accounting_payload,is_batch) VALUES ($1,$2::"AccountingDeliveryState",$$ {} $$::jsonb,false)', + request_id, + state, + ) + start = _NOW - timedelta(hours=1) + totals = await spend_logs_seed_totals(client, entity_type, entity_type, start, _NOW) + assert totals is not None and totals.total == 10.0 and totals.before_batch == 5.0 + unbounded = await spend_logs_seed_totals(client, entity_type, entity_type, start, None) + assert unbounded is not None and unbounded.total == unbounded.before_batch == 10.0 + await commit_window_spend_updates( + client, + ( + { + "entity_type": entity_type, + "entity_id": entity_type, + "window_duration": "1d", + "window_start": start.isoformat(), + "started_at": _NOW.isoformat(), + "spend": 5.0, + }, + ), + ) + rows = await client.writer_db.query_raw('SELECT spend FROM "LiteLLM_BudgetWindowSpend"') + assert rows[0]["spend"] == 10.0 + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("disable_logs", [False, True]) +@pytest.mark.asyncio +async def test_batch_claim_failure_queues_an_atomic_retry_and_repeated_retrieve_does_not_charge( + parity_database: str, monkeypatch: pytest.MonkeyPatch, disable_logs: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", disable_logs) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + + + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'ALTER TABLE "LiteLLM_SpendLogDelivery" ADD CONSTRAINT reject_batch ' + "CHECK (request_id <> 'batch_batch_cost')" + ) + assert await _retrieve_batch(writer, 0.25) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,) + assert len(client.spend_log_transactions) == 1 + queued = tuple(client.spend_log_transactions) + assert "accounting_payload" in queued[0] + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" DROP CONSTRAINT reject_batch') + assert await SpendLogsRepository(client).insert_with_deliveries(queued) == 1 + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0.25,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(not disable_logs),) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_failed_claim_over_a_legacy_zero_batch_row_still_charges_it_once( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + await drain_logging_tasks() + await SpendLogsRepository(client).insert_with_deliveries(tuple(client.spend_log_transactions)) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'batch_batch_cost\'' + ).fetchall() == [(0.0,)] + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" RENAME TO "LiteLLM_SpendLogDelivery_unavailable"') + assert await _retrieve_batch(writer, 0.25) + queued = tuple(client.spend_log_transactions) + assert ["accounting_payload" in row for row in queued] == [True] + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery_unavailable" RENAME TO "LiteLLM_SpendLogDelivery"') + assert await SpendLogsRepository(client).insert_with_deliveries(queued) == 1 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'batch_batch_cost\'' + ).fetchall() == [(0.0,)] + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_worker_exposes_invalid_accounting_as_delayed_then_parked_work( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from prometheus_client import REGISTRY + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, created_at, accounting_payload, is_batch) ' + "VALUES ('invalid-accounting', CURRENT_TIMESTAMP - INTERVAL '1 hour', " + '$$ {"metadata": {"accounting_windows": [{}]}} $$::jsonb, false)' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert REGISTRY.get_sample_value("litellm_accounting_pending") == 1 + assert REGISTRY.get_sample_value("litellm_accounting_parked") == 0 + with psycopg.connect(parity_database, autocommit=True) as connection: + assert connection.execute( + 'SELECT state::text, attempts, last_error FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("pending", 1, "KeyError")] + connection.execute( + 'UPDATE "LiteLLM_SpendLogDelivery" SET attempts=4, next_attempt_at=CURRENT_TIMESTAMP' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert REGISTRY.get_sample_value("litellm_accounting_pending") == 0 + assert REGISTRY.get_sample_value("litellm_accounting_parked") == 1 + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT state::text, attempts, last_error FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("parked", 5, "KeyError")] + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0,) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_fallback_charges_only_groups_declared_by_the_served_deployment( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "stripped", fallback_narrowing=True + ) + + +@pytest.mark.asyncio +async def test_worker_skips_another_leader_and_releases_its_redis_lock_after_retry( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager + + monkeypatch.setenv("DATABASE_URL", parity_database) + cache = RedisCache(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), namespace=uuid.uuid4().hex) + other_leader = PodLockManager(cache) + writer = DBSpendUpdateWriter(redis_cache=cache) + writer.pod_lock_manager.redis_cache = cache + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch) VALUES ($$invalid-row$$, $$ {"metadata": {"accounting_windows": [{}]}} $$::jsonb, false)') + assert await other_leader.acquire_lock(cronjob_id="accounting_delivery", allow_reentrant=False) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(0,)] + await other_leader.release_lock(cronjob_id="accounting_delivery") + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(1,)] + assert await other_leader.acquire_lock(cronjob_id="accounting_delivery", allow_reentrant=False) + finally: + await other_leader.release_lock(cronjob_id="accounting_delivery") + await client.db.disconnect() + await cache.disconnect() + + +async def _retrieve_batch(writer: DBSpendUpdateWriter, cost: float) -> bool: + return await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user_id="user", end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "aretrieve_batch", "litellm_call_id": "batch", + "litellm_params": {"metadata": {"status": "success", "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}}, + }, + completion_response={"id": "batch", "status": "completed"}, + start_time=_NOW, end_time=_NOW, response_cost=cost, + ) + + +@pytest.mark.parametrize("ledger_after_zero_row,delivery_unavailable", [(False, False), (True, False), (True, True)]) +@pytest.mark.asyncio +async def test_zero_cost_legacy_batch_row_is_charged_once_across_a_flag_change( + parity_database: str, monkeypatch: pytest.MonkeyPatch, ledger_after_zero_row: bool, delivery_unavailable: bool +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + await drain_logging_tasks() + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger_after_zero_row}) + assert await _retrieve_batch(writer, 0.25) + await drain_logging_tasks() + assert await _retrieve_batch(writer, 0.25) is False + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0,) + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [ + (0.0 if ledger_after_zero_row else 0.25,) + ] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == ( + int(ledger_after_zero_row), + ) + if delivery_unavailable: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE "LiteLLM_SpendLogDelivery" RENAME TO temporarily_unavailable_delivery') + try: + await writer.db_update_spend_transaction_handler(client, 0, proxy_server.proxy_logging_obj) + finally: + if delivery_unavailable: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute('ALTER TABLE temporarily_unavailable_delivery RENAME TO "LiteLLM_SpendLogDelivery"') + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (0.25,) + assert connection.execute('SELECT spend, api_requests FROM "LiteLLM_DailyUserSpend"').fetchall() == [(0.25, 2)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("failing_table", [ + "LiteLLM_UserTable", "LiteLLM_VerificationToken", "LiteLLM_TeamTable", "LiteLLM_TeamMembership", + "LiteLLM_OrganizationMembership", "LiteLLM_OrganizationTable", "LiteLLM_ProjectTable", "LiteLLM_AgentsTable", "LiteLLM_EndUserTable", +]) +@pytest.mark.asyncio +async def test_destination_failure_rolls_back_all_earlier_updates_before_retry( + parity_database: str, monkeypatch: pytest.MonkeyPatch, failing_table: str +): + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "full", failing_table=failing_table + ) + + +@pytest.mark.asyncio +async def test_spend_logs_cleanup_does_not_remove_pending_accounting(parity_database, monkeypatch): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + cleanup = SpendLogCleanup(clock=lambda: 0) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + delete_sql = 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id IN (SELECT request_id FROM "LiteLLM_SpendLogs" WHERE "startTime" < $1::timestamp LIMIT $2::integer)' + assert await cleanup._execute_delete_batch(client, delete_sql, _NOW + timedelta(days=1), "LiteLLM_SpendLogs", 60) == 1 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert await _retrieve_batch(writer, 0.25) is False + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,) + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=$$user$$').fetchone() == (0.25,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("mode", ["local", "url", "disabled"]) +@pytest.mark.parametrize("receipt_state", ["pending", "parked", "applied", "retained"]) +@pytest.mark.asyncio +async def test_ledger_charged_batch_is_not_charged_again_after_flag_disable( + parity_database: str, monkeypatch: pytest.MonkeyPatch, receipt_state: str, mode: str +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", mode == "disabled") + if mode == "url": + monkeypatch.setenv("SPEND_LOGS_URL", "http://127.0.0.1:1") + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0) + assert await _retrieve_batch(writer, 0.25) + if receipt_state in ("applied", "retained"): + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database, autocommit=True) as connection: + if receipt_state == "retained": + connection.execute("UPDATE \"LiteLLM_SpendLogDelivery\" SET applied_at = CURRENT_TIMESTAMP - INTERVAL '400 days'") + if receipt_state == "parked": + connection.execute('UPDATE "LiteLLM_SpendLogDelivery" SET state=\'parked\'') + assert await AccountingDeliveryRepository(client.writer_db).cleanup_applied() == 0 + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": False}) + assert await _retrieve_batch(writer, 0.25) is False + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == ( + [(0.25,)] if mode != "disabled" else [] + ) + expected_spend = 0 if receipt_state == "parked" else 0.25 + expected_state = "parked" if receipt_state == "parked" else "applied" + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (expected_spend,) + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [(expected_state,)] + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("disabled", [False, True]) +@pytest.mark.parametrize("export_status", [200, 500]) +@pytest.mark.asyncio +async def test_export_failure_does_not_change_committed_accounting(parity_database, monkeypatch, export_status, disabled): + requests = asyncio.Queue() + + async def receive(reader, writer): + header = await reader.readuntil(b"\r\n\r\n") + length = next(int(line.split(b":", 1)[1]) for line in header.split(b"\r\n") if line.lower().startswith(b"content-length:")) + body = await reader.readexactly(length) + await requests.put(json.loads(body)) + writer.write(f"HTTP/1.1 {export_status} Result\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{{}}".encode()) + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(receive, "127.0.0.1", 0) + try: + port = server.sockets[0].getsockname()[1] + await _assert_legacy_database_parity( + parity_database, monkeypatch, "success", None, 0.25, "lean" if disabled else "full", + export_url=f"http://127.0.0.1:{port}", export_failure=export_status == 500 and not disabled, + ) + assert requests.qsize() == int(not disabled) + if disabled: + return + exported = await asyncio.wait_for(requests.get(), timeout=5) + assert len(exported) == 1 + assert exported[0]["spend"] == 0.25 + assert "accounting_payload" not in exported[0] + assert "accounting_log_disabled" not in exported[0] + finally: + server.close() + await server.wait_closed() + + +@pytest.mark.asyncio +async def test_spend_logs_url_without_an_export_client_keeps_local_logs_and_charges_once( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import ProxyUpdateSpend + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("SPEND_LOGS_URL", "http://127.0.0.1:1") + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + assert await writer.update_database( + token="2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user_id="user", end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "acompletion", "litellm_call_id": "url-request", + "litellm_params": {"metadata": {"status": "success", "user_api_key": "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"}}, + }, + completion_response=ModelResponse( + id="url-request", model="ledger-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, end_time=_NOW, response_cost=0.25, + ) + await drain_logging_tasks() + rows = [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in client.spend_log_transactions + ] + client.spend_log_transactions = [] + await ProxyUpdateSpend.update_spend_logs(0, client, None, proxy_server.proxy_logging_obj, rows) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=\'url-request\'' + ).fetchall() == [(0.25,)] + assert connection.execute('SELECT state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("applied",)] + assert connection.execute('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=\'2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683\'').fetchone() == (0.25,) + finally: + await client.db.disconnect() + + +@contextlib.asynccontextmanager +async def _ledger_client(parity_database: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("DATABASE_URL", parity_database) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + client.model_usage_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + await client.db.connect() + try: + yield client + finally: + await client.db.disconnect() + + +async def _record_call( + writer: DBSpendUpdateWriter, call_id: str, *, token: str = "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683", user: str = "user", tags: object = None +) -> None: + metadata = {"status": "success", "user_api_key": token, **({"tags": tags} if tags is not None else {})} + assert await writer.update_database( + token=token, user_id=user, end_user_id=None, team_id="team", org_id=None, + kwargs={ + "model": "ledger-test", "call_type": "aembedding", "litellm_call_id": call_id, + "litellm_params": {"metadata": metadata}, + }, + completion_response=None, start_time=_NOW, end_time=_NOW, response_cost=0.25, + ) + await drain_logging_tasks() + + +def _wire(rows) -> list[dict[str, object]]: + return [ + {key: value.isoformat() if isinstance(value, datetime) else value for key, value in row.items()} + for row in rows + ] + + +async def _flush(writer: DBSpendUpdateWriter, client: PrismaClient) -> None: + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import ProxyUpdateSpend + + rows = _wire(client.spend_log_transactions) + client.spend_log_transactions = [] + await ProxyUpdateSpend.update_spend_logs(0, client, None, proxy_server.proxy_logging_obj, rows) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + + +def _key_spend(parity_database: str) -> object: + with psycopg.connect(parity_database) as connection: + return connection.execute( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', ("2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683",) + ).fetchone() + + +@pytest.mark.asyncio +async def test_requests_sharing_a_client_call_id_are_each_charged_but_a_replay_is_not( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "shared-call-id") + await _record_call(writer, "shared-call-id") + buffered = _wire(client.spend_log_transactions) + await _flush(writer, client) + assert await SpendLogsRepository(client).insert_with_deliveries(buffered) == 0 + await _flush(writer, client) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"' + ).fetchall() == [("shared-call-id", "applied"), ("shared-call-id", "applied")] + assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (1,) + + +@pytest.mark.asyncio +async def test_an_opaque_bearer_token_is_never_stored_in_a_delivery( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "opaque-call", token="opaque-bearer-secret") + await _flush(writer, client) + with psycopg.connect(parity_database) as connection: + stored = connection.execute('SELECT accounting_payload::text, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() + assert [state for _, state in stored] == ["applied"] + assert "opaque-bearer-secret" not in stored[0][0] + + +@pytest.mark.asyncio +async def test_an_evicted_spend_log_still_charges_its_request(parity_database: str, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.utils import enqueue_spend_logs + + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "evicted-call") + await _record_call(writer, "kept-call") + queued = tuple(client.spend_log_transactions) + client.spend_log_transactions = [] + await enqueue_spend_logs(client, queued, max_bytes=1) + assert [row.get("accounting_log_disabled") for row in client.spend_log_transactions] == [True, False] + await _flush(writer, client) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id FROM "LiteLLM_SpendLogs"').fetchall() == [("kept-call",)] + + +@pytest.mark.parametrize("isolation_budget", [5, 0], ids=["isolated", "budget_exhausted"]) +@pytest.mark.asyncio +async def test_a_spend_log_postgres_rejects_still_charges_its_request_and_the_rest_of_the_batch( + parity_database: str, monkeypatch: pytest.MonkeyPatch, isolation_budget: int +): + from prisma.errors import DataError + + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + class RejectsOneLog: + def __init__(self, repository: SpendLogsRepository) -> None: + self.repository = repository + + async def insert_with_deliveries(self, rows, *, write_spend_logs: bool = True) -> int: + if any(row["request_id"] == "poison-call" and not row.get("accounting_log_disabled") for row in rows): + raise DataError({"user_facing_error": {"message": 'invalid byte sequence for encoding "UTF8": 0x00'}}) + return await self.repository.insert_with_deliveries(rows, write_spend_logs=write_spend_logs) + + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "poison-call") + await _record_call(writer, "clean-call") + rows = tuple(client.jsonify_object(row) for row in client.spend_log_transactions) + client.spend_log_transactions = [] + await _create_spend_logs_with_poison_isolation(RejectsOneLog(SpendLogsRepository(client)), rows, isolation_budget) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + assert _key_spend(parity_database) == (0.5,) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id FROM "LiteLLM_SpendLogs"').fetchall() == ( + [("clean-call",)] if isolation_budget else [] + ) + assert connection.execute( + 'SELECT state::text, COUNT(*) FROM "LiteLLM_SpendLogDelivery" GROUP BY 1' + ).fetchall() == [("applied", 2)] + + +@pytest.mark.asyncio +async def test_a_request_with_a_non_text_tag_is_still_charged(parity_database: str, monkeypatch: pytest.MonkeyPatch): + async with _ledger_client(parity_database, monkeypatch) as client: + writer = DBSpendUpdateWriter() + await _record_call(writer, "tagged-call", tags=[None, "real-tag"]) + await _flush(writer, client) + assert _key_spend(parity_database) == (0.25,) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT tag, spend FROM "LiteLLM_DailyTagSpend" WHERE tag=\'real-tag\'' + ).fetchall() == [("real-tag", 0.25)] + + +@pytest.mark.parametrize("threshold", ["count", "bytes"]) +@pytest.mark.asyncio +async def test_retained_batch_storage_threshold_alerts_without_deleting_receipts( + parity_database: str, monkeypatch: pytest.MonkeyPatch, threshold: str +): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.accounting.worker import flush_accounting_deliveries + from litellm.types.integrations.slack_alerting import AlertType + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1" if threshold == "count" else "1000000") + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1" if threshold == "bytes" else "1073741824") + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + alerts = SimpleNamespace(alerting_handler=AsyncMock()) + await client.db.connect() + try: + with psycopg.connect(parity_database, autocommit=True) as connection: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, applied_at, accounting_payload, is_batch) ' + "VALUES ('old-batch', 'applied', CURRENT_TIMESTAMP - INTERVAL '400 days', '{}', true)" + ) + await flush_accounting_deliveries(DBSpendUpdateWriter(), client, alerts) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ + ("old-batch", "applied") + ] + table_bytes = connection.execute( + 'SELECT pg_total_relation_size(\'"LiteLLM_SpendLogDelivery"\')' + ).fetchone()[0] + alerts.alerting_handler.assert_awaited_once_with( + message=f"Accounting delivery storage requires review. Retained batches=1, table bytes={table_bytes}.", + level="High", alert_type=AlertType.failed_tracking_spend, + ) + finally: + await client.db.disconnect() + + +@pytest.mark.parametrize("backlog", ["retries", "expired"]) +@pytest.mark.parametrize("batch_size", [1, 1100]) +@pytest.mark.asyncio +async def test_worker_drains_recovered_and_expired_work_at_its_configured_capacity( + parity_database, monkeypatch, backlog, batch_size +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setenv("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", str(batch_size)) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + writer = DBSpendUpdateWriter() + monkeypatch.setattr(proxy_server, "prisma_client", client) + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + with psycopg.connect(parity_database, autocommit=True) as connection: + if backlog == "retries": + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, attempts) ' + "SELECT 'retry-' || i, accounting_payload, false, 1 " + 'FROM "LiteLLM_SpendLogDelivery" CROSS JOIN generate_series(1, 3) i' + ) + else: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, state, applied_at) ' + "SELECT 'expired-' || i, '{}', false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days' " + 'FROM generate_series(1, 1100) i' + ) + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == ( + 0.25 * (1 + min(batch_size, 3)) if backlog == "retries" else 0.25, + ) + assert connection.execute( + 'SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery" WHERE state=\'pending\' OR request_id LIKE \'expired-%\'' + ).fetchone() == ((3 - min(batch_size, 3)) if backlog == "retries" else max(0, 1100 - batch_size * 2),) + finally: + await client.db.disconnect() + + +@pytest.mark.asyncio +async def test_analytics_overflow_across_requests_still_applies_every_charge( + parity_database: str, monkeypatch: pytest.MonkeyPatch +): + from litellm.accounting.worker import flush_accounting_deliveries + + monkeypatch.setenv("DATABASE_URL", parity_database) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": True}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) + client.spend_log_transactions = [] + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + writer = DBSpendUpdateWriter() + await client.db.connect() + try: + assert await _retrieve_batch(writer, 0.25) + stored = await client.writer_db.query_raw('SELECT to_jsonb(s) AS row FROM "LiteLLM_SpendLogs" s') + delivery = (await client.writer_db.query_raw('SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"'))[0][ + "accounting_payload" + ] + oversized_savings = {**delivery["metadata"], "compression_savings": {"tokens_saved": 2**62}} + assert await SpendLogsRepository(client).insert_with_deliveries( + tuple( + { + **stored[0]["row"], + "request_id": f"overflow-{index}", + "accounting_payload": {**delivery, "request_id": f"overflow-{index}", "metadata": oversized_savings}, + } + for index in range(3) + ) + ) == 3 + await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) + with psycopg.connect(parity_database) as connection: + assert connection.execute( + 'SELECT state::text, COUNT(*) FROM "LiteLLM_SpendLogDelivery" GROUP BY state' + ).fetchall() == [("applied", 4)] + assert connection.execute('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id=\'user\'').fetchone() == (1.0,) + assert connection.execute( + 'SELECT api_requests, compression_saved_tokens FROM "LiteLLM_DailyUserSpend"' + ).fetchone() == (4, 3 * 2**40) + finally: + await client.db.disconnect() diff --git a/tests/integration/management/test_credential_display_name.py b/tests/integration/management/test_credential_display_name.py index 0e72dcf8b2f..f605be1d059 100644 --- a/tests/integration/management/test_credential_display_name.py +++ b/tests/integration/management/test_credential_display_name.py @@ -122,6 +122,9 @@ def test_patch_rename_is_rejected_and_model_keeps_working(gateway: Gateway) -> N "PATCH", f"/credentials/{name}", {"credential_name": name, "credential_info": {"description": "y"}} ) assert same_name.status_code == 200, same_name.text + assert read_rows( + 'SELECT credential_info FROM "LiteLLM_CredentialsTable" WHERE credential_name = %s', (name,) + ) == [{"credential_info": {"custom_llm_provider": "openai", "description": "y"}}] assert _served_credential(gateway, name)["credential_info"] == { "custom_llm_provider": "openai", "description": "y", diff --git a/tests/integration/mcp/test_mcp_transports.py b/tests/integration/mcp/test_mcp_transports.py index 92fcf37994e..4c3648a3381 100644 --- a/tests/integration/mcp/test_mcp_transports.py +++ b/tests/integration/mcp/test_mcp_transports.py @@ -669,7 +669,7 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( import httpx2 import yaml from integration._support.asgi import asgi_server - from integration._support.mcp import _drain_sse_streams + from integration._support.mcp import _acknowledge_notifications_after_dispatch, _drain_sse_streams from integration._support.process import owned_proxy from mcp import ClientSession, MCPError from mcp.client.streamable_http import streamable_http_client @@ -769,7 +769,9 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( watchers = tuple(task for task in asyncio.all_tasks() if "_shutdown_watcher" in repr(task.get_coro())) await asyncio.wait_for(asyncio.gather(*watchers), 2) - hosted: Final = Starlette(routes=[Mount("/", app=app)], lifespan=lifespan) + hosted: Final = Starlette( + routes=[Mount("/", app=_acknowledge_notifications_after_dispatch(app))], lifespan=lifespan + ) with asgi_server(hosted) as upstream_url, httpx.Client() as client: config: Final = tmp_path / "proxy.yaml" config.write_text( diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py new file mode 100644 index 00000000000..9d0e31058e4 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py @@ -0,0 +1,746 @@ +import json +import os +import threading +import time +import uuid +from collections.abc import Callable, Generator, Mapping +from contextlib import contextmanager +from typing import Final, Literal +from urllib.parse import unquote + +import anthropic +import httpx +import openai +import pytest +from integration._support.anthropic_sse import ( + LIFECYCLE, + error_frame, + event_type, + event_types, + message_id, + message_json, + message_stream, + parse_sse, + status_reply, + stream_reply, + user_prompt, +) +from integration._support.client import ( + JSON_OBJECT, + Gateway, + JsonValue, + Scenario, + eventually, + object_value, + string_value, +) +from integration._support.database import read_rows +from integration._support.openai_wire import openai_error +from integration._support.upstream import aws_event_stream_frame +from integration._support.wire import Reply, Request, Wire, wire_server +from redis import Redis + +_MODEL: Final = "claude-sonnet-4-5-20250929" +_API_KEY: Final = "synthetic-anthropic-key" +_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" +_BEDROCK_TOKEN: Final = "synthetic-bedrock-bearer" +_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}' +_CONVERSE_MODEL: Final = f"bedrock/converse/{_BEDROCK_MODEL}" +_CONVERSE_STREAM_TARGET: Final = f"/model/{_BEDROCK_MODEL}/converse-stream" +_OPENAI_BACKEND: Final = "gpt-4o-mini" +_CHAT_ONLY_BACKEND: Final = "audit-chat-only" +_CODEX_BACKEND: Final = "gpt-5.3-codex" +_TEXT: Final = "Hello" +_KEEPALIVE_SECONDS: Final = 0.2 +_LATE_ANSWER_SECONDS: Final = 1.0 +_OVERLOAD_CLASS: Final = "503_PROVIDER_OVERLOADED" + +Shape = Literal["stream", "non-stream"] + + +def _prompt() -> str: + return "overloaded-" + uuid.uuid4().hex + + +def _always(reply: Reply) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST", request + return reply + + return respond + + +def _late(reply: Reply, seconds: float) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST", request + time.sleep(seconds) + return reply + + return respond + + +def _anthropic_sdk(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60) + + +def _async_anthropic_sdk(gateway: Gateway) -> anthropic.AsyncAnthropic: + return anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60 + ) + + +def _openai_sdk(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60) + + +def _async_openai_sdk(gateway: Gateway) -> openai.AsyncOpenAI: + return openai.AsyncOpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60) + + +def _messages_body(model: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]: + return { + "model": model, + "max_tokens": 16, + "stream": stream, + "messages": [{"role": "user", "content": prompt}], + "num_retries": 0, + } + + +def _anthropic_error(response: httpx.Response) -> dict[str, JsonValue]: + assert response.headers["content-type"].startswith("application/json"), dict(response.headers) + payload: Final = object_value(json.loads(response.content)) + assert payload["type"] == "error", response.text + return object_value(payload["error"]) + + +def _openai_error(response: httpx.Response) -> dict[str, JsonValue]: + return object_value(object_value(json.loads(response.content))["error"]) + + +def _assert_overloaded(response: httpx.Response, *, message: str) -> None: + assert response.status_code == 529, response.text + error: Final = _anthropic_error(response) + assert error["type"] == "overloaded_error", response.text + assert message in string_value(error["message"]), response.text + assert "content_block_delta" not in response.text, response.text + + +def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None: + assert response.status_code == status, response.text + error: Final = _anthropic_error(response) + assert error["type"] == "api_error", response.text + assert message in string_value(error["message"]), response.text + + +def _assert_internal_server_error(response: httpx.Response) -> None: + assert response.status_code == 500, response.text + error: Final = _openai_error(response) + assert error["code"] == "500", response.text + assert "InternalServerError" in string_value(error["message"]), response.text + + +def _assert_openai_overloaded(response: httpx.Response) -> None: + assert response.status_code == 503, response.text + error: Final = _openai_error(response) + assert set(error) == {"message", "type", "param", "code"}, response.text + assert string_value(error["message"]), response.text + assert error["type"] == "service_unavailable_error", response.text + assert error["param"] is None, response.text + assert error["code"] == "server_is_overloaded", response.text + + +def _one_post(wire: Wire, target: str) -> None: + assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)] + + +def _spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + "SELECT status, metadata->'error_information'->>'error_code' AS code, " + "metadata->'error_information'->>'error_class' AS error_class, " + "metadata->'error_information'->>'normalized_error' AS normalized_error " + 'FROM "LiteLLM_SpendLogs" WHERE model_group=%s', + (model,), + ), + lambda rows: len(rows) >= expected, + seconds=70, + ) + + +def _assert_logged_failure(model: str, *, rows: int, error_class: str, code: str = "529") -> None: + assert ( + _spend_rows(model, rows) + == [{"status": "failure", "code": code, "error_class": error_class, "normalized_error": _OVERLOAD_CLASS}] * rows + ) + + +def test_messages_529_answers_overloaded_error_to_the_sync_sdk_and_logs_each_call_once(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + sdk: Final = _anthropic_sdk(gateway) + prompt: Final = _prompt() + for _ in range(2): + with pytest.raises(anthropic.APIStatusError) as caught: + sdk.messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": prompt}], + extra_body={"num_retries": 0}, + ) + assert caught.value.status_code == 529, caught.value.response.text + _assert_overloaded(caught.value.response, message="scripted 529") + assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2 + _assert_logged_failure(model, rows=2, error_class="InternalServerError") + + +async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + with pytest.raises(anthropic.APIStatusError) as caught: + await _async_anthropic_sdk(gateway).messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": _prompt()}], + extra_body={"num_retries": 0}, + ) + assert caught.value.status_code == 529, caught.value.response.text + _assert_overloaded(caught.value.response, message="scripted 529") + _one_post(wire, "/v1/messages") + + +@pytest.mark.parametrize( + ("upstream_status", "error_class", "error_type"), + ((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")), +) +def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json( + gateway: Gateway, upstream_status: int, error_class: str, error_type: str +) -> None: + with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + assert response.status_code == upstream_status, response.text + error: Final = _anthropic_error(response) + assert error["type"] == error_type, response.text + assert f"scripted {upstream_status}" in string_value(error["message"]), response.text + _one_post(wire, "/v1/messages") + _assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status)) + + +def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + with pytest.raises(anthropic.APIStatusError) as caught: + with _anthropic_sdk(gateway).messages.stream( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": _prompt()}], + extra_body={"num_retries": 0}, + ) as stream: + stream.get_final_message() + assert caught.value.status_code == 529, caught.value.response.text + _assert_overloaded(caught.value.response, message="scripted 529") + _one_post(wire, "/v1/messages") + + +def _error_frames(text: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + object_value(json.loads(line.removeprefix("data:").strip())) + for line in text.splitlines() + if line.startswith("data:") and '"error"' in line + ) + + +def test_messages_529_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event( + gateway: Gateway, +) -> None: + with ( + wire_server(_late(status_reply(529), _LATE_ANSWER_SECONDS)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, keepalive_seconds=_KEEPALIVE_SECONDS + ) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream"), dict(response.headers) + assert response.text.startswith(":"), response.text + (failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping") + assert failure.event == "error", response.text + assert object_value(failure.data["error"])["type"] == "overloaded_error", response.text + assert "[DONE]" not in response.text, response.text + _one_post(wire, "/v1/messages") + + +def _late_bedrock_deployment(scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model=f"bedrock/{_BEDROCK_MODEL}", + api_key=_BEDROCK_TOKEN, + aws_region_name="us-east-1", + api_base=wire.url, + keepalive_seconds=_KEEPALIVE_SECONDS, + ) + + +def _one_bedrock_post(wire: Wire) -> None: + (request,) = wire.drain() + assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request + + +def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event( + gateway: Gateway, +) -> None: + with ( + wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire, + gateway.scenario() as scenario, + ): + model: Final = _late_bedrock_deployment(scenario, wire) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + assert response.status_code == 200, response.text + assert response.text.startswith(":"), response.text + assert "[DONE]" not in response.text, response.text + (failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping") + assert failure.event == "error", response.text + error: Final = object_value(failure.data["error"]) + assert error["type"] == "api_error", response.text + assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text + _one_bedrock_post(wire) + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") + + +def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame( + gateway: Gateway, +) -> None: + with ( + wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire, + gateway.scenario() as scenario, + ): + model: Final = _late_bedrock_deployment(scenario, wire) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0}, + ) + assert response.status_code == 200, response.text + assert response.text.startswith(":"), response.text + assert "event: error" not in response.text, response.text + (frame,) = _error_frames(response.text) + assert object_value(frame["error"])["code"] == "503", response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + _one_bedrock_post(wire) + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") + + +def test_chat_completions_overloaded_frame_after_content_ends_with_a_500_error_frame(gateway: Gateway) -> None: + prompt: Final = _prompt() + upstream: Final = (*message_stream(prompt, _MODEL, _TEXT)[:3], error_frame(529, "Overloaded")) + with wire_server(_always(stream_reply(upstream))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0}, + ) + assert response.status_code == 200, response.text + assert _TEXT in response.text, response.text + (frame,) = _error_frames(response.text) + error: Final = object_value(frame["error"]) + assert error["code"] == "500", response.text + _one_post(wire, "/v1/messages") + + +def _overloaded_before_content(prompt: str) -> Reply: + return stream_reply((*message_stream(prompt, _MODEL, _TEXT)[:1], error_frame(529, "Overloaded"))) + + +def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_relayed_as_an_overloaded_event( + gateway: Gateway, +) -> None: + prompt: Final = _prompt() + with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=True)) + assert response.status_code == 200, response.text + events: Final = parse_sse(response.text) + assert event_types(events) == ("message_start", "error"), response.text + assert message_id(events) == prompt, response.text + assert events[1].data == {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + _one_post(wire, "/v1/messages") + assert _spend_rows(model, 1) == [ + {"status": "success", "code": None, "error_class": None, "normalized_error": None} + ] + + +def test_chat_completions_overloaded_frame_before_content_answers_503(gateway: Gateway) -> None: + prompt: Final = _prompt() + with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0}, + ) + _assert_openai_overloaded(response) + _one_post(wire, "/v1/messages") + + +def test_chat_completions_529_answers_503_to_the_sync_sdk(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + with pytest.raises(openai.APIStatusError) as caught: + _openai_sdk(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0} + ) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) + _one_post(wire, "/v1/messages") + _assert_logged_failure(model, rows=1, error_class="InternalServerError", code="503") + + +async def test_chat_completions_stream_529_answers_503_to_the_async_sdk(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + with pytest.raises(openai.APIStatusError) as caught: + await _async_openai_sdk(gateway).chat.completions.create( + model=model, + messages=[{"role": "user", "content": _prompt()}], + stream=True, + extra_body={"num_retries": 0}, + ) + assert caught.value.status_code == 503, caught.value.response.text + _assert_openai_overloaded(caught.value.response) + _one_post(wire, "/v1/messages") + + +def test_completions_529_answers_503(gateway: Gateway) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/completions", + {"model": model, "prompt": _prompt(), "max_tokens": 16, "num_retries": 0}, + ) + _assert_openai_overloaded(response) + _one_post(wire, "/v1/messages") + + +@pytest.mark.parametrize("shape", ("non-stream", "stream")) +def test_responses_529_answers_503(gateway: Gateway, shape: Shape) -> None: + with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0} + ) + _assert_openai_overloaded(response) + _one_post(wire, "/v1/messages") + + +@pytest.mark.parametrize("shape", ("non-stream", "stream")) +def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None: + with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url + ) + response: Final = gateway.request( + "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") + ) + _assert_api_error(response, status=503, message="Bedrock is unable to process your request.") + (request,) = wire.drain() + assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") + + +def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes: + return aws_event_stream_frame( + {":event-type": event, ":content-type": "application/json", ":message-type": "event"}, + json.dumps(payload).encode(), + ) + + +def _converse_503_after_text() -> Reply: + unavailable: Final = aws_event_stream_frame( + { + ":exception-type": "serviceUnavailableException", + ":content-type": "application/json", + ":message-type": "exception", + }, + _BEDROCK_503, + ) + frames: Final = ( + _converse_event("messageStart", {"role": "assistant"}), + _converse_event("contentBlockDelta", {"delta": {"text": _TEXT}, "contentBlockIndex": 0}), + unavailable, + ) + return Reply(body=b"".join(frames), content_type="application/vnd.amazon.eventstream") + + +def _converse_deployment(scenario: Scenario, wire: Wire) -> str: + return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url) + + +def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None: + with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario: + model: Final = _converse_deployment(scenario, wire) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + assert response.status_code == 200, response.text + events: Final = parse_sse(response.text) + types: Final = event_types(events) + assert "message_stop" not in types, response.text + assert types.index("content_block_delta") < types.index("error"), response.text + assert _TEXT in response.text, response.text + (failure,) = (event for event in events if event_type(event) == "error") + error: Final = object_value(failure.data["error"]) + assert error["type"] == "api_error", response.text + assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text + _one_post(wire, _CONVERSE_STREAM_TARGET) + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") + + +def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None: + with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario: + model: Final = _converse_deployment(scenario, wire) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0}, + ) + assert response.status_code == 200, response.text + assert _TEXT in response.text, response.text + (frame,) = _error_frames(response.text) + error: Final = object_value(frame["error"]) + assert error["code"] == "503", response.text + _one_post(wire, _CONVERSE_STREAM_TARGET) + _assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503") + + +@pytest.mark.parametrize("shape", ("non-stream", "stream")) +def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None: + with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key" + ) + response: Final = gateway.request( + "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") + ) + _assert_api_error(response, status=503, message="scripted 503") + _one_post(wire, "/v1/chat/completions") + + +@pytest.mark.parametrize("shape", ("non-stream", "stream")) +def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None: + with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key" + ) + response: Final = gateway.request( + "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") + ) + _assert_api_error(response, status=503, message="scripted 503") + _one_post(wire, "/v1/responses") + + +_SETTLED_ANSWERS: Final = 4 + + +def _messages_to(gateway: Gateway, model: str) -> tuple[tuple[str, httpx.Response], ...]: + prompts: Final = tuple(_prompt() for _ in range(_SETTLED_ANSWERS)) + return tuple( + (prompt, gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False))) + for prompt in prompts + ) + + +def _failure_rows_of(models: tuple[str, str], expected: int) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + "SELECT metadata->'error_information'->>'error_code' AS code, " + "metadata->'error_information'->>'error_class' AS error_class, " + "metadata->'error_information'->>'normalized_error' AS normalized_error " + 'FROM "LiteLLM_SpendLogs" WHERE status = %s AND model_group IN (%s, %s)', + ("failure", *models), + ), + lambda rows: len(rows) >= expected, + seconds=70, + ) + + +@contextmanager +def _paused_deployment(gateway: Gateway) -> Generator[tuple[Wire, str, str]]: + unpaused: Final = Reply(body=message_json("msg_unpaused", _MODEL, _TEXT)) + with wire_server(_always(unpaused)) as wire, gateway.scenario() as scenario: + identity: Final = f"audit-paused-{uuid.uuid4().hex}" + name: Final = scenario.model( + model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, model_info={"id": identity} + ) + paused: Final = gateway.request("PATCH", f"/model/{identity}/update", {"blocked": True}) + assert paused.status_code == 200, paused.text + yield wire, identity, name + + +def _assert_refused_without_an_upstream_call( + gateway: Gateway, wire: Wire, target: str, *, status: int, error_type: str, message: str +) -> None: + answers: Final = eventually( + lambda: _messages_to(gateway, target), + lambda settled: all(response.status_code != 200 for _, response in settled), + seconds=90, + ) + texts: Final = [response.text for _, response in answers] + assert [response.status_code for _, response in answers] == [status] * _SETTLED_ANSWERS, texts + errors: Final = [_anthropic_error(response) for _, response in answers] + assert [error["type"] for error in errors] == [error_type] * _SETTLED_ANSWERS, texts + assert all(message in string_value(error["message"]) for error in errors), texts + reached: Final = {user_prompt(JSON_OBJECT.validate_json(request.body)) for request in wire.drain()} + assert reached.isdisjoint(prompt for prompt, _ in answers), reached + + +def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None: + with _paused_deployment(gateway) as (wire, identity, name): + _assert_refused_without_an_upstream_call( + gateway, wire, identity, status=503, error_type="api_error", message="is currently paused" + ) + rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS) + assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == { + ("503", "ServiceUnavailableError", _OVERLOAD_CLASS) + }, rows + + +def test_messages_to_a_paused_model_group_named_by_its_name_answer_403_permission_error(gateway: Gateway) -> None: + with _paused_deployment(gateway) as (wire, _, name): + _assert_refused_without_an_upstream_call( + gateway, wire, name, status=403, error_type="permission_error", message="Model is blocked" + ) + + +_MALFORMED_529_BODIES: Final = { + "empty": b"", + "text-5k": b"x" * 5000, + "html": b"

529

", + "api_error-type": b'{"type": "error", "error": {"type": "api_error", "message": "capacity"}}', +} + + +def _overloaded_for_sad_prompts(body: bytes) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + prompt: Final = user_prompt(object_value(json.loads(request.body))) + if prompt.startswith("sad:"): + return Reply(status=529, body=body, content_type="text/plain" if body[:1] != b"{" else "application/json") + return Reply(body=message_json(f"msg_{prompt}", _MODEL, _TEXT)) + + return respond + + +@pytest.mark.parametrize("body", _MALFORMED_529_BODIES.keys()) +def test_messages_529_with_a_malformed_body_still_answers_overloaded_error_and_unrelated_calls_keep_working( + gateway: Gateway, body: str +) -> None: + with wire_server(_overloaded_for_sad_prompts(_MALFORMED_529_BODIES[body])) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + sad: Final = gateway.request("POST", "/v1/messages", _messages_body(model, "sad:" + _prompt(), stream=False)) + assert sad.status_code == 529, sad.text + assert _anthropic_error(sad)["type"] == "overloaded_error", sad.text + prompt: Final = _prompt() + healthy: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False)) + assert healthy.status_code == 200, healthy.text + assert object_value(json.loads(healthy.content))["id"] == f"msg_{prompt}", healthy.text + assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2 + assert sorted(_spend_rows(model, 2), key=lambda row: str(row["status"])) == [ + { + "status": "failure", + "code": "529", + "error_class": "InternalServerError", + "normalized_error": _OVERLOAD_CLASS, + }, + {"status": "success", "code": None, "error_class": None, "normalized_error": None}, + ] + + +@pytest.mark.parametrize(("upstream_status", "error_type"), ((500, "api_error"), (429, "rate_limit_error"))) +def test_messages_other_upstream_statuses_keep_their_status_and_type( + gateway: Gateway, upstream_status: int, error_type: str +) -> None: + with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + assert response.status_code == upstream_status, response.text + assert _anthropic_error(response)["type"] == error_type, response.text + _one_post(wire, "/v1/messages") + + +def _anthropic_rejection(status: int, error_type: str, message: str) -> Reply: + return Reply( + status=status, body=json.dumps({"type": "error", "error": {"type": error_type, "message": message}}).encode() + ) + + +_REJECTIONS_SAYING_OVERLOADED: Final = { + "400-tool-name": _anthropic_rejection( + 400, "invalid_request_error", "tools.0.name: 'Overloaded' is not a valid tool name" + ), + "404-model": _anthropic_rejection(404, "not_found_error", "model: claude-Overloaded"), +} + + +@pytest.mark.parametrize("shape", ("non-stream", "stream")) +@pytest.mark.parametrize("rejection", _REJECTIONS_SAYING_OVERLOADED.keys()) +def test_messages_4xx_whose_text_says_overloaded_is_not_answered_as_an_overload( + gateway: Gateway, rejection: str, shape: Shape +) -> None: + with wire_server(_always(_REJECTIONS_SAYING_OVERLOADED[rejection])) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream") + ) + assert response.status_code == 500, response.text + error: Final = _anthropic_error(response) + assert error["type"] == "api_error", response.text + assert "Overloaded" in string_value(error["message"]), response.text + _one_post(wire, "/v1/messages") + assert _spend_rows(model, 1) == [ + { + "status": "failure", + "code": "500", + "error_class": "InternalServerError", + "normalized_error": _OVERLOAD_CLASS, + } + ] + + +def _string_values(cache: Redis) -> tuple[bytes, ...]: + keys: Final = tuple(key for key in cache.scan_iter(count=1000) if cache.type(key) == b"string") + return tuple(value for value in cache.mget(keys) if value is not None) if keys else () + + +def _cached_somewhere(served_id: str) -> bool: + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + return any(served_id.encode() in value for value in _string_values(cache)) + + +def _healthy_until(overloaded: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if overloaded.is_set(): + return status_reply(529) + prompt: Final = user_prompt(object_value(json.loads(request.body))) + return stream_reply(message_stream(f"msg_{prompt}", _MODEL, _TEXT)) + + return respond + + +def test_messages_cache_hit_is_served_while_the_upstream_is_overloaded(gateway: Gateway) -> None: + overloaded: Final = threading.Event() + with wire_server(_healthy_until(overloaded)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + prompt: Final = _prompt() + body: Final = _messages_body(model, prompt, stream=True) + first: Final = gateway.request("POST", "/v1/messages", body) + assert first.status_code == 200, first.text + assert event_types(parse_sse(first.text)) == LIFECYCLE, first.text + assert message_id(parse_sse(first.text)) == f"msg_{prompt}", first.text + eventually(lambda: _cached_somewhere(f"msg_{prompt}"), bool, seconds=30) + overloaded.set() + replay: Final = gateway.request("POST", "/v1/messages", body) + assert replay.status_code == 200, replay.text + assert message_id(parse_sse(replay.text)) == f"msg_{prompt}", replay.text + fresh: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True)) + _assert_overloaded(fresh, message="scripted 529") + assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2 diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_chaos.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_chaos.py index 9dd1a2d2c18..99239efe5ee 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_chaos.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_chaos.py @@ -135,6 +135,13 @@ def _drop_or_500(request: Request, marker: str) -> Reply: return openai_error(500) +def _overloaded_while(outage: threading.Event) -> Callable[[Request, str], Reply]: + def first_attempt(request: Request, marker: str) -> Reply: + return status_reply(529) if outage.is_set() else _served(request, marker, 1) + + return first_attempt + + def _outage(statuses: Mapping[str, int]) -> Callable[[Request, str], Reply]: def first_attempt(request: Request, marker: str) -> Reply: if request.target == "/v1/messages": @@ -165,7 +172,7 @@ class _Upstream: return self.first_attempt(request, marker) -def _config(wire: Wire, directory: Path, models: _Models) -> Path: +def _config(wire: Wire, directory: Path, models: _Models, *, retries: int = 1) -> Path: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["model_list"] = [ { @@ -174,7 +181,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path: "model": f"anthropic/{_ANTHROPIC_BACKEND}", "api_base": wire.url, "api_key": _ANTHROPIC_KEY, - "num_retries": 1, + "num_retries": retries, }, }, { @@ -183,7 +190,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path: "model": f"openai/{_OPENAI_BACKEND}", "api_base": wire.url + "/v1", "api_key": _OPENAI_KEY, - "num_retries": 1, + "num_retries": retries, }, }, ] @@ -408,3 +415,68 @@ async def test_outage_on_every_first_attempt_is_absorbed_by_the_deployment_budge _assert_completed_on_the_second_attempt(item) _assert_each_served_id_landed_exactly_once(models, served) assert all(upstream.attempts.count(call.marker) == 2 for call in calls) + + +def _messages_rows(model: str, expected: int) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + "SELECT request_id, status, metadata->'error_information'->>'error_code' AS code " + 'FROM "LiteLLM_SpendLogs" WHERE model_group=%s', + (model,), + ), + lambda found: len(found) >= expected, + seconds=90, + ) + + +@pytest.mark.timeout(300) +async def test_overloaded_outage_during_a_burst_with_a_worker_sigkill_answers_529_and_lands_every_call_once( + gateway: Gateway, tmp_path: Path +) -> None: + models: Final = _models() + calls: Final = _calls((("messages", True, 12), ("messages", False, 6))) + release: Final = threading.Event() + held: Final[SimpleQueue[str]] = SimpleQueue() + outage: Final = threading.Event() + outage.set() + upstream: Final = _Upstream(Attempts(), _overloaded_while(outage), held, release) + with wire_server(upstream) as wire: + config: Final = _config(wire, tmp_path, models, retries=0) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: + candidate: Final = owned.gateway + base_url: Final = str(candidate.client.base_url) + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + burst: Final = asyncio.create_task( + _burst(base_url, candidate.key, models, calls, tolerate_transport_errors=True) + ) + await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(calls), 90) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers}) + assert sum(held_by.values()) == len(calls), held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + with owned.replacing_worker(victim_pid) as kill: + kill.freeze() + await asyncio.to_thread(kill) + release.set() + served: Final = await burst + assert held_by[survivor_pid] >= 5, held_by + assert len(served) == held_by[survivor_pid], (held_by, len(served)) + for item in served: + assert item.status == 529, (item.call, item.text) + assert "overloaded_error" in item.text, (item.call, item.text) + assert "content_block_delta" not in item.text, (item.call, item.text) + outage.clear() + follow_up: Final = _Call(endpoint="messages", stream=True, marker=uuid.uuid4().hex) + (answered,) = await _burst(base_url, candidate.key, models, (follow_up,)) + assert answered.status == 200, answered.text + assert message_id(parse_sse(answered.text)) == f"msg_{follow_up.marker}_a1", answered.text + rows: Final = _messages_rows(models.messages, len(served) + 1) + assert sorted(string_value(row["status"]) for row in rows) == ["failure"] * len(served) + ["success"], rows + assert {row["code"] for row in rows if row["status"] == "failure"} == {"529"}, rows + assert [row["request_id"] for row in rows if row["status"] == "success"] == [ + f"msg_{follow_up.marker}_a1" + ], rows + assert all(upstream.attempts.count(item.call.marker) == 1 for item in (*served, answered)) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py index 041a0a72fdd..83a9ba3d14c 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py @@ -397,14 +397,14 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway: with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) response: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "deployment")) - _assert_rejected_before_content(response, wire, status=500, attempts=2, error="error") + _assert_rejected_before_content(response, wire, status=529, attempts=2, error="overloaded_error") rows: Final = eventually( lambda: _request_rows(gateway, response.headers["x-litellm-call-id"]), lambda values: len(values) >= 1, seconds=70, ) rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0)) - _assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error") + _assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error") rejection_rows: Final = eventually( lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]), lambda values: len(values) >= 1, diff --git a/tests/integration/observability/test_anthropic_overloaded_failure_signals.py b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py new file mode 100644 index 00000000000..e86ed98fae7 --- /dev/null +++ b/tests/integration/observability/test_anthropic_overloaded_failure_signals.py @@ -0,0 +1,404 @@ +import gzip +import json +import time +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from urllib.parse import unquote + +import httpx +import pytest +import yaml +from integration._support.anthropic_sse import error_body +from integration._support.client import ( + JSON_OBJECT, + Gateway, + JsonValue, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.otlp_sink import Span, SpanSinks, recorded_spans, spans_for_trace +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.prometheus_series import Sample, scrape +from integration._support.wire import Reply, Request, Wire, wire_server + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_MODEL: Final = "claude-sonnet-4-5-20250929" +_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" +_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}' +_MESSAGES_ANTHROPIC: Final = "audit-signals-messages-anthropic" +_MESSAGES_BEDROCK: Final = "audit-signals-messages-bedrock" +_CHAT_ANTHROPIC: Final = "audit-signals-chat-anthropic" +_TRACED_MESSAGES: Final = "audit-signals-traced-messages" +_TRACED_CHAT: Final = "audit-signals-traced-chat" +_DATADOG_MESSAGES_ANTHROPIC: Final = "audit-signals-datadog-messages-anthropic" +_DATADOG_MESSAGES_BEDROCK: Final = "audit-signals-datadog-messages-bedrock" +_DATADOG_CHAT_ANTHROPIC: Final = "audit-signals-datadog-chat-anthropic" +_GUARDED_MESSAGES: Final = "audit-signals-guarded-messages" +_GUARDED_CHAT: Final = "audit-signals-guarded-chat" +_AKTO_DOWN: Final = "audit-signals-akto-down" +_AKTO_UNREACHABLE: Final = "Akto guardrail service unreachable" +_LATE_PASS_THROUGH_BACKEND: Final = "claude-late-pass-through" +_KEEPALIVE_SECONDS: Final = 0.2 +_LATE_ANSWER_SECONDS: Final = 1.0 +_OVERLOADED: Final = Reply(status=529, body=error_body(529, "Overloaded")) +_SERVER: Final = 2 + +AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path] + + +@dataclass(frozen=True, slots=True) +class _Rig: + proxy: Gateway + wire: Wire + spans: str + datadog: Wire + + +def _respond(request: Request) -> Reply: + assert request.method == "POST", request + if unquote(request.target).startswith("/model/"): + return Reply(status=503, body=_BEDROCK_503) + if json.loads(request.body).get("model") == _LATE_PASS_THROUGH_BACKEND: + time.sleep(_LATE_ANSWER_SECONDS) + return _OVERLOADED + + +def _datadog_intake(request: Request) -> Reply: + assert request.target == "/api/v2/logs", request.target + return Reply(status=202, body=b"{}") + + +def _config(directory: Path, tracing: Path, wire: Wire) -> Path: + configuration: Final = JSON_OBJECT.validate_python(yaml.safe_load(tracing.read_text())) + anthropic: Final = {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": "synthetic-anthropic-key"} + bedrock: Final = { + "model": f"bedrock/{_BEDROCK_MODEL}", + "api_base": wire.url, + "api_key": "synthetic-bedrock-bearer", + "aws_region_name": "us-east-1", + } + deployments: Final = ( + (_MESSAGES_ANTHROPIC, anthropic), + (_MESSAGES_BEDROCK, bedrock), + (_CHAT_ANTHROPIC, anthropic), + (_TRACED_MESSAGES, anthropic), + (_TRACED_CHAT, anthropic), + (_DATADOG_MESSAGES_ANTHROPIC, anthropic), + (_DATADOG_MESSAGES_BEDROCK, bedrock), + (_DATADOG_CHAT_ANTHROPIC, anthropic), + (_GUARDED_MESSAGES, anthropic), + (_GUARDED_CHAT, anthropic), + ) + akto_down: Final = { + "guardrail_name": _AKTO_DOWN, + "litellm_params": { + "guardrail": "akto", + "mode": "pre_call", + "akto_base_url": "http://127.0.0.1:9", + "akto_api_key": "synthetic-akto-key", + "unreachable_fallback": "fail_closed", + }, + } + path: Final = directory / "anthropic-overloaded-failure-signals.yaml" + path.write_text( + yaml.safe_dump( + { + **configuration, + "model_list": [{"model_name": name, "litellm_params": params} for name, params in deployments], + "router_settings": {**object_value(configuration.get("router_settings", {})), "num_retries": 0}, + "litellm_settings": { + **object_value(configuration["litellm_settings"]), + "sse_keepalive_ping_interval_seconds": _KEEPALIVE_SECONDS, + }, + "guardrails": [akto_down], + } + ) + ) + return path + + +@pytest.fixture(scope="module") +def rig( + tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter +) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals") + multiprocess: Final = directory / "prometheus" + multiprocess.mkdir() + tracing: Final = otel_audit_config(directory, {"callbacks": ["otel", "prometheus", "datadog"]}) + with ( + wire_server(_respond) as wire, + wire_server(_datadog_intake) as datadog, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + { + "OTEL_EXPORTER": "http/json", + "OTEL_ENDPOINT": audit_sinks.operator, + "PROMETHEUS_MULTIPROC_DIR": str(multiprocess), + "DD_API_KEY": "synthetic-datadog-key", + "DD_SITE": "datadog.invalid", + "DD_BASE_URL": datadog.url, + "DD_BATCH_SIZE": "1", + "ANTHROPIC_API_BASE": wire.url, + "ANTHROPIC_API_KEY": "synthetic-anthropic-key", + }, + config=_config(directory, tracing, wire), + workers=2, + ) as owned, + ): + yield _Rig(owned, wire, audit_sinks.operator, datadog) + + +@pytest.fixture(scope="module") +def v2_rig( + tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter +) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals-v2") + tracing: Final = otel_audit_config(directory, {"callbacks": ["otel"]}) + with ( + wire_server(_respond) as wire, + wire_server(_datadog_intake) as datadog, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + {"OTEL_EXPORTER": "http/json", "OTEL_ENDPOINT": audit_sinks.operator, "LITELLM_OTEL_V2": "1"}, + config=_config(directory, tracing, wire), + workers=2, + ) as owned, + ): + yield _Rig(owned, wire, audit_sinks.operator, datadog) + + +def _send( + rig: _Rig, + route: str, + model: str, + trace_id: str | None = None, + *, + key: str | None = None, + extra: Mapping[str, JsonValue] | None = None, +) -> httpx.Response: + traceparent: Final = {} if trace_id is None else {"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"} + return rig.proxy.request( + "POST", + route, + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}], + "num_retries": 0, + **(extra or {}), + }, + key=key, + headers={"anthropic-version": "2023-06-01", **traceparent}, + ) + + +def _series(samples: Sequence[Sample], name: str, model: str) -> tuple[Sample, ...]: + return tuple(sample for sample in samples if sample.name == name and sample.labels.get("requested_model") == model) + + +def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]: + return eventually(lambda: _series(scrape(rig.proxy), name, model), bool, seconds=30) + + +@pytest.mark.parametrize( + ("route", "model", "answered", "deployment_status", "deployment_class"), + ( + pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"), + pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"), + pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 503, "500", "InternalServerError", id="chat-anthropic"), + ), +) +def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status( + rig: _Rig, route: str, model: str, answered: int, deployment_status: str, deployment_class: str +) -> None: + rig.wire.drain() + response: Final = _send(rig, route, model) + assert response.status_code == answered, response.text + assert len(rig.wire.drain()) == 1 + failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model) + assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [ + (str(answered), route, 1.0) + ], failed + deployment: Final = _until(rig, "litellm_deployment_failure_responses_total", model) + assert [(sample.labels["exception_status"], sample.value) for sample in deployment] == [(deployment_status, 1.0)], ( + deployment + ) + assert all(deployment_class in sample.labels["exception_class"] for sample in deployment), deployment + + +def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]: + traced: Final = spans_for_trace(recorded_spans(rig.spans, since)[1], trace_id) + return tuple(span for span in traced if span["kind"] == _SERVER) + + +@pytest.mark.parametrize( + ("route", "model", "answered"), + ( + pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat"), + ), +) +def test_the_server_span_carries_the_status_the_route_answered( + rig: _Rig, route: str, model: str, answered: int +) -> None: + rig.wire.drain() + since, _ = recorded_spans(rig.spans) + trace_id: Final = uuid.uuid4().hex + response: Final = _send(rig, route, model, trace_id) + assert response.status_code == answered, response.text + assert len(rig.wire.drain()) == 1 + server: Final = eventually( + lambda: _server_spans(rig, since, trace_id), + lambda spans: any("error.code" in span["attributes"] for span in spans), + seconds=40, + return_last_on_timeout=True, + ) + assert [ + (str(span["attributes"].get("http.response.status_code")), str(span["attributes"].get("error.code"))) + for span in server + ] == [(str(answered), str(answered))], server + + +def _inflated(request: Request) -> bytes: + return gzip.decompress(request.body) if request.headers.get("content-encoding") == "gzip" else request.body + + +def _datadog_events(batches: Sequence[Request]) -> Iterator[JsonValue]: + for batch in batches: + yield from json.loads(_inflated(batch)) + + +def _datadog_messages(batches: Sequence[Request]) -> tuple[dict[str, JsonValue], ...]: + return tuple( + JSON_OBJECT.validate_json(string_value(object_value(event)["message"])) for event in _datadog_events(batches) + ) + + +def _keyed_by(message: Mapping[str, JsonValue], alias: str) -> bool: + proxy_side: Final = message.get("user_api_key_dict") + llm_side: Final = message.get("metadata") + return any(isinstance(side, dict) and side.get("user_api_key_alias") == alias for side in (proxy_side, llm_side)) + + +def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[str, str], ...]: + batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches + + def delivered() -> tuple[tuple[str, str], ...]: + batches.extend(rig.datadog.drain()) + return tuple( + sorted( + ("proxy", str(message["status_code"])) + if "user_api_key_dict" in message + else ("llm", str(object_value(message["error_information"])["error_code"])) + for message in _datadog_messages(batches) + if _keyed_by(message, alias) + ) + ) + + return eventually(delivered, lambda codes: len(codes) >= expected, seconds=40, return_last_on_timeout=True) + + +@pytest.mark.parametrize( + ("route", "model", "answered", "deployment_status"), + ( + pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"), + pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"), + pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 503, "500", id="chat-anthropic"), + ), +) +def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status( + rig: _Rig, route: str, model: str, answered: int, deployment_status: str +) -> None: + rig.wire.drain() + rig.datadog.drain() + alias: Final = f"audit-signals-datadog-{uuid.uuid4().hex}" + with rig.proxy.scenario() as scenario: + response: Final = _send(rig, route, model, key=scenario.key(key_alias=alias, models=[model])) + assert response.status_code == answered, response.text + assert len(rig.wire.drain()) == 1 + assert _datadog_status_codes(rig, alias, 2) == (("llm", deployment_status), ("proxy", str(answered))) + + +@pytest.mark.parametrize( + ("route", "model", "answered"), + ( + pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"), + pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"), + ), +) +def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes_status( + rig: _Rig, route: str, model: str, answered: int +) -> None: + rig.wire.drain() + rig.datadog.drain() + alias: Final = f"audit-signals-guarded-{uuid.uuid4().hex}" + with rig.proxy.scenario() as scenario: + response: Final = _send( + rig, route, model, key=scenario.key(key_alias=alias, models=[model]), extra={"guardrails": [_AKTO_DOWN]} + ) + assert response.status_code == answered, response.text + assert _AKTO_UNREACHABLE in response.text, response.text + assert rig.wire.drain() == () + failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model) + assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [ + (str(answered), route, 1.0) + ], failed + assert _datadog_status_codes(rig, alias, 2) == (("llm", str(answered)), ("proxy", str(answered))) + + +@pytest.mark.parametrize( + ("route", "model", "answered"), + ( + pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"), + pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"), + pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat-anthropic"), + ), +) +def test_the_otel_v2_server_span_carries_the_status_the_route_answered( + v2_rig: _Rig, route: str, model: str, answered: int +) -> None: + v2_rig.wire.drain() + since, _ = recorded_spans(v2_rig.spans) + trace_id: Final = uuid.uuid4().hex + response: Final = _send(v2_rig, route, model, trace_id) + assert response.status_code == answered, response.text + assert len(v2_rig.wire.drain()) == 1 + server: Final = eventually( + lambda: _server_spans(v2_rig, since, trace_id), + lambda spans: any("litellm.provider.error.code" in span["attributes"] for span in spans), + seconds=40, + return_last_on_timeout=True, + ) + assert [str(span["attributes"].get("litellm.provider.error.code")) for span in server] == [str(answered)], server + + +def test_an_anthropic_pass_through_stream_that_fails_after_the_keepalive_opened_relays_the_upstream_body( + rig: _Rig, +) -> None: + rig.wire.drain() + response: Final = rig.proxy.request( + "POST", + "/anthropic/v1/messages", + { + "model": _LATE_PASS_THROUGH_BACKEND, + "max_tokens": 16, + "stream": True, + "messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}], + }, + headers={"anthropic-version": "2023-06-01"}, + ) + assert response.status_code == 200, response.text + pings, _, relayed = response.text.rpartition("\n\n") + assert pings and set(pings.split("\n")) <= {"", ": ping"}, response.text + assert relayed == _OVERLOADED.body.decode(), response.text + assert [unquote(request.target) for request in rig.wire.drain()] == ["/v1/messages"] diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_chaos.py b/tests/integration/observability/test_arize_otel_v2_openinference_chaos.py index 79b0d9084aa..0095080d944 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_chaos.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_chaos.py @@ -1,12 +1,13 @@ from __future__ import annotations +import socket import threading import uuid from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from pathlib import Path +from queue import SimpleQueue from typing import Final -from urllib.parse import urlsplit import httpx import pytest @@ -44,7 +45,7 @@ from _openinference_support import ( ) from integration._support.client import Gateway, eventually from integration._support.process import graceful_stop_seconds -from integration._support.wire import Reply, Request, wire_server +from integration._support.wire import Reply, Request, Wire, wire_server def _call( @@ -249,11 +250,12 @@ def test_arize_otel_v2_f1_sink_outage_and_recovery(gateway: Gateway, tmp_path: P _assert_response(response, marker, surface, stream, model_name) with ExitStack() as servers: - initial_stack: Final = servers.enter_context(ExitStack()) - stopped_destination: Final = initial_stack.enter_context(wire_server(_owned_sink_handler(sink))) - sink_port: Final = urlsplit(stopped_destination.url).port - assert sink_port is not None, stopped_destination.url - initial_stack.close() + outage_socket: Final = servers.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + outage_socket.bind(("127.0.0.1", 0)) # test-quality-ok: socket stays bound throughout the outage + sink_port: Final = outage_socket.getsockname()[1] + stopped_destination: Final = Wire( + f"http://127.0.0.1:{sink_port}", SimpleQueue(), SimpleQueue(), SimpleQueue(), SimpleQueue() + ) with _rig( gateway, tmp_path, @@ -272,6 +274,7 @@ def test_arize_otel_v2_f1_sink_outage_and_recovery(gateway: Gateway, tmp_path: P with httpx.Client(trust_env=False) as client, pytest.raises(httpx.ConnectError): client.get(stopped_destination.url + "/health", timeout=2) recovered_stack: Final = servers.enter_context(ExitStack()) + outage_socket.close() recovered_destination: Final = recovered_stack.enter_context( wire_server(_owned_sink_handler(sink), port=sink_port) ) diff --git a/tests/integration/pricing/test_cost_margin_discount.py b/tests/integration/pricing/test_cost_margin_discount.py index b6cda8f125f..613fe72ed70 100644 --- a/tests/integration/pricing/test_cost_margin_discount.py +++ b/tests/integration/pricing/test_cost_margin_discount.py @@ -7,7 +7,7 @@ import httpx import pytest from integration._support.client import Gateway, eventually, object_value, string_value from integration._support.database import read_rows -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue class CostEstimate(BaseModel): @@ -110,6 +110,25 @@ def _assert_logged_spends(expectations: tuple[tuple[str, float], ...]) -> None: assert actual == pytest.approx(dict(expectations)), rows +class LoggedBreakdown(BaseModel): + model_config = ConfigDict(frozen=True) + + original_cost: float + discount_percent: float | None = None + discount_amount: float | None = None + margin_percent: float | None = None + margin_fixed_amount: float | None = None + margin_total_amount: float | None = None + + +def _logged_breakdown(request_id: str) -> LoggedBreakdown: + rows: Final = read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (request_id,)) + assert len(rows) == 1, rows + metadata: Final = rows[0]["metadata"] + parsed: Final = object_value(json.loads(metadata) if isinstance(metadata, str) else metadata) + return LoggedBreakdown.model_validate(parsed["cost_breakdown"]) + + def _estimated(gateway: Gateway, model: str) -> CostEstimate: response: Final = gateway.request( "POST", "/cost/estimate", {"model": model, "input_tokens": 20, "output_tokens": 20} @@ -236,6 +255,17 @@ def test_discount_and_margin_reprice_a_request_and_survive_rejection(gateway: Ga (streamed_global_request_id, 0.12), ) ) + discounted: Final = _logged_breakdown(discounted_request_id) + assert discounted.original_cost == pytest.approx(0.06), discounted + assert discounted.discount_percent == pytest.approx(0.05), discounted + assert discounted.discount_amount == pytest.approx(0.06 * 0.05), discounted + margined: Final = _logged_breakdown(provider_margin_request_id) + assert margined.original_cost == pytest.approx(0.06), margined + assert margined.discount_percent == pytest.approx(0.05), margined + assert margined.discount_amount == pytest.approx(0.06 * 0.05), margined + assert margined.margin_percent == pytest.approx(0.1), margined + assert margined.margin_fixed_amount == pytest.approx(0.001), margined + assert margined.margin_total_amount == pytest.approx(0.057 * 0.1 + 0.001), margined rejections: Final = ( ( diff --git a/tests/integration/pricing/test_model_listing_token_limits.py b/tests/integration/pricing/test_model_listing_token_limits.py index 5f75b5bb38a..5473d802f7f 100644 --- a/tests/integration/pricing/test_model_listing_token_limits.py +++ b/tests/integration/pricing/test_model_listing_token_limits.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Final import pytest +from litellm import get_model_info from integration._support.client import Gateway, object_value from integration._support.process import owned_proxy from pydantic import JsonValue @@ -104,6 +105,21 @@ def test_v1_models_carries_cost_map_context_window_for_a_known_model(gateway: Ga assert listed["max_output_tokens"] == 16384, listed +def test_v1_models_keeps_a_sibling_override_off_a_plain_deployment_of_the_same_model(gateway: Gateway) -> None: + catalog_info: Final = get_model_info("openai/gpt-4o-mini") + with gateway.scenario() as scenario: + overridden: Final = scenario.model( + model="openai/gpt-4o-mini", + model_info={"max_input_tokens": 4321, "max_output_tokens": 987}, + ) + plain: Final = scenario.model(model="openai/gpt-4o-mini") + listed_override: Final = _listed_model(gateway, overridden) + listed_plain: Final = _listed_model(gateway, plain) + + assert _limits(listed_plain) == (catalog_info["max_input_tokens"], catalog_info["max_output_tokens"]) + assert _limits(listed_override) == (4321, 987) + + def test_v1_models_carries_deployment_model_info_limits_for_an_unknown_model(gateway: Gateway) -> None: unknown: Final = f"openai/custom-{uuid.uuid4().hex}" with gateway.scenario() as scenario: diff --git a/tests/integration/pricing/test_price_data_reload_prices.py b/tests/integration/pricing/test_price_data_reload_prices.py index a91f5cba1ad..aa0addb7673 100644 --- a/tests/integration/pricing/test_price_data_reload_prices.py +++ b/tests/integration/pricing/test_price_data_reload_prices.py @@ -25,9 +25,16 @@ _GEO: Final = f"au.{_ROW}" _FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01" _CLAUDE: Final = "claude-audit-reload-sonnet" _MINI: Final = "gpt-audit-reload-mini" +_FREE: Final = "gpt-audit-reload-free" +_RULE_FREE: Final = "gpt-audit-rule-free" +_FILL_FREE: Final = "gpt-audit-fill-free" +_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced" +_REPORTED_FREE: Final = "gpt-audit-reported-free" _USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)") _CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06}) -_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}) +_COST_MAP_LIMITS: Final = MappingProxyType( + {"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"} +) _PUBLISHED_SCALE: Final = 2.0 _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType( { @@ -60,6 +67,13 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType( "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, }, + _FILL_FREE: {"litellm_provider": "openai", "mode": "chat"}, + _FREE: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, _MINI: { "litellm_provider": "openai", "mode": "chat", @@ -99,6 +113,8 @@ class _Deployment: prices: Mapping[str, float] usage: _Usage rates: Callable[[float], _Rates | None] + reported_cost: float | None = None + listed_rates: Callable[[float], _Rates | None] | None = None @dataclass(frozen=True, slots=True) @@ -107,10 +123,23 @@ class _Charge: expected: float -def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]: +def _catalog(scale: float) -> dict[str, JsonValue]: return { - model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()} - for model, row in _CATALOG.items() + **{ + model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()} + for model, row in _CATALOG.items() + }, + "fallback_generalizations": { + "rules": [ + { + "name": "free-pricing", + "pattern": f"({_RULE_FREE}|{_FILL_FREE})", + "fill_missing_for_providers": ["openai"], + "model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + }, + {"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}}, + ] + }, } @@ -240,6 +269,7 @@ def _reply(request: Request) -> Reply: "completion_tokens": usage.completion, "total_tokens": usage.prompt + usage.completion, "prompt_tokens_details": {"cached_tokens": usage.cached}, + **({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}), }, } ).encode() @@ -281,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge: rates: Final = deployment.rates(scale) - assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name + assert _listed_rates(proxy, deployment.name) == _expected_listing( + deployment.listed_rates(scale) if deployment.listed_rates is not None else rates + ), deployment.name request_id, charged = _billed(proxy.client, proxy.key, deployment) - expected: Final = _expected_cost(rates, deployment.usage) - assert charged == pytest.approx(expected, rel=1e-6), deployment.name + expected: Final = ( + deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage) + ) + assert charged == ( + None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6) + ), deployment.name return _Charge(request_id=request_id, expected=expected) @@ -352,8 +388,31 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices( _Usage(2000, 1000, 1000), _catalog_rates(_FINETUNE), ), - _Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates), + _Deployment( + "finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates + ), _Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced), + _Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)), + _Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)), + _Deployment( + "fill-rule-free", + f"openai/{_FILL_FREE}", + {}, + _Usage(2000, 1000, 1000), + _catalog_rates(_FREE), + listed_rates=_unpriced, + ), + _Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced), + _Deployment( + "provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0 + ), + _Deployment( + "cross-provider-free", + f"azure/{_CLAUDE}", + {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + _Usage(2000, 1000, 1000), + _catalog_rates(_FREE), + ), _Deployment( "region-custom-above-threshold", f"bedrock/ap-northeast-1.{_ROW}", diff --git a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py index 8f497fea74f..96261bab8b2 100644 --- a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py +++ b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py @@ -475,10 +475,6 @@ def row_key(request_id: str) -> str: return upstream_id_of(request_id.split("_cache_hit", 1)[0]) -def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool: - return (endpoint, stream) != ("messages", True) - - def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: usage: Final = payload.get("usage") return JSON.validate_python(usage) if isinstance(usage, dict) else {} @@ -733,10 +729,9 @@ def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None: assert_answered(outcome, marker) assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream) assert_usage(endpoint, outcome.usage) - row: Final = success_row(name, marker) + row: Final = success_row(name, marker, outcome.response_id) assert_priced_row(row, GPT) - if caller_sees_upstream_id(endpoint, stream=stream): - assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) + assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) if not stream: assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), ( outcome.headers @@ -901,7 +896,7 @@ def assert_marked_sdk_cell( only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream ) assert_usage(endpoint, outcome.usage) - assert_priced_row(success_row(name, marker), GPT) + assert_priced_row(success_row(name, marker, outcome.response_id), GPT) HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = { diff --git a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py index 6879d1f8acd..d4c0e2651d8 100644 --- a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py +++ b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py @@ -482,15 +482,20 @@ def test_duplicated_search_result_blocks_rebuild_the_call_once(gateway: Gateway, @pytest.mark.parametrize( - ("status", "kind", "caller_status"), + ("status", "kind", "caller_status", "caller_error_type", "caller_error_code"), ( - pytest.param(400, "invalid_request_error", 400, id="400"), - pytest.param(401, "authentication_error", 401, id="401"), - pytest.param(529, "overloaded_error", 500, id="529"), + pytest.param(400, "invalid_request_error", 400, None, None, id="400"), + pytest.param(401, "authentication_error", 401, None, None, id="401"), + pytest.param(529, "overloaded_error", 503, "service_unavailable_error", "server_is_overloaded", id="529"), ), ) def test_provider_error_on_the_replay_reaches_the_caller( - gateway: Gateway, status: int, kind: str, caller_status: int + gateway: Gateway, + status: int, + kind: str, + caller_status: int, + caller_error_type: str | None, + caller_error_code: str | None, ) -> None: marker: Final = uuid.uuid4().hex @@ -508,6 +513,11 @@ def test_provider_error_on_the_replay_reaches_the_caller( received: Final = wire.drain() assert second.status_code == caller_status, second.text assert f"scripted {kind} {marker}" in second.text, second.text + if caller_error_type is not None: + payload: Final = JSON_OBJECT.validate_json(second.content) + error: Final = JSON_OBJECT.validate_python(payload["error"]) + assert error["type"] == caller_error_type, second.text + assert error["code"] == caller_error_code, second.text assert len(received) >= 2, received assert wire_assistant(received[1]) == list(SEARCH_TURN), received[1].body assert gateway.client.get("/health/liveliness").status_code == 200 diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index a6714979c8e..05e08c5fc27 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -1251,17 +1251,19 @@ def test_s01_to_s06_an_odd_typed_flag_is_read_as_false_never_as_true( @pytest.mark.timeout(600) -def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_it_in_place(gateway: Gateway) -> None: +def test_s10_s11_two_deployments_of_one_arn_keep_their_own_flags_and_a_delete_changes_neither( + gateway: Gateway, +) -> None: arn: Final = _arn("shared") with wire_server(_peer()) as wire, gateway.scenario() as scenario: flagged: Final = _deployment( gateway, scenario, wire, f"bedrock/{arn}", {"supports_prompt_cache_breakpoint": False}, cleanup=False ) plain: Final = _deployment(gateway, scenario, wire, f"bedrock/{arn}") - for name in (flagged, plain): + for name, cache_points in ((flagged, 0), (plain, 2)): outcome, received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER)) _assert_answered(outcome) - assert received.cache_points == 0, (name, received.body) + assert received.cache_points == cache_points, (name, received.body) _success_row(outcome.response_id) flagged_identity: Final = _model_id(flagged) gateway.post("/model/delete", {"id": flagged_identity}) @@ -1276,7 +1278,7 @@ def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_i wire.drain() after, after_received = _observe_at(gateway, wire, "chat", _chat_body(plain, _prompt(), _SYSTEM_AND_USER)) _assert_answered(after) - assert after_received.cache_points == 0, after_received.body + assert after_received.cache_points == 2, after_received.body _success_row(after.response_id) diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 32b6c4f1690..3652e49fe8a 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -1211,6 +1211,6 @@ def test_non_claude_mantle_id_with_a_bare_bedrock_twin_keeps_its_route_and_spend cost: Final = _mantle_cost(gateway, _LUNA) assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(cost), response.text _assert_success_row( - _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "responses", cost + _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "acompletion", cost ) assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)] diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 67edcf7f3e4..2502bd90bd6 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -622,10 +622,15 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gateway: Gateway, route: str) -> None: +@pytest.mark.parametrize( + ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"] +) +def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default( + gateway: Gateway, route: str, backend: str, thinks_by_default: bool +) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, route, _HAIKU) + model: Final = _deployment(scenario, wire, route, backend) response: Final = gateway.request( "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA)) ) @@ -634,6 +639,9 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa assert not message.get("tool_calls"), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" + if thinks_by_default: + _assert_native_without_tools(body, invoke, _SCHEMA) + return assert _native_schema(body, invoke) is None, body assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body diff --git a/tests/integration/providers/test_custom_video_provider_wire.py b/tests/integration/providers/test_custom_video_provider_wire.py new file mode 100644 index 00000000000..a40ecae3ed4 --- /dev/null +++ b/tests/integration/providers/test_custom_video_provider_wire.py @@ -0,0 +1,113 @@ +"""A provider registered in `custom_provider_map` with a `BaseVideoConfig` serves /v1/videos create, status and content.""" + +import json +import shutil +from contextlib import ExitStack +from pathlib import Path +from typing import Final + +from integration._support.client import Gateway +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server + +_SUPPORT_MODULE: Final = Path(__file__).resolve().parents[1] / "_support" / "custom_video_provider.py" +_PROMPT: Final = "a cat walking across a bridge" +_MP4: Final = b"\x00\x00\x00\x18ftypmp42" + b"queue-video-bytes" + + +def _queue_backend(request: Request) -> Reply: + if (request.method, request.target) == ("POST", "/jobs"): + return Reply(body=json.dumps({"job_id": "job-7", "state": "queued"}).encode()) + if (request.method, request.target) == ("GET", "/jobs/job-7"): + return Reply(body=json.dumps({"job_id": "job-7", "state": "done", "seconds": 8}).encode()) + if (request.method, request.target) == ("GET", "/jobs/job-7/file"): + return Reply(body=_MP4, content_type="video/mp4") + return Reply(status=404, body=json.dumps({"error": f"no route {request.method} {request.target}"}).encode()) + + +def _write_config(path: Path, wire: Wire) -> Path: + shutil.copy(_SUPPORT_MODULE, path.parent / "custom_video_provider.py") + path.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "queue-video", + "litellm_params": { + "model": "queue-video/seedance-1", + "api_base": wire.url, + "api_key": "queue-secret", + }, + }, + { + "model_name": "image-only-video", + "litellm_params": { + "model": "image-only/seedance-1", + "api_base": wire.url, + "api_key": "unused", + }, + }, + ], + "litellm_settings": { + "custom_provider_map": [ + {"provider": "queue-video", "custom_handler": "custom_video_provider.VIDEO_CONFIG"}, + {"provider": "image-only", "custom_handler": "custom_video_provider.IMAGE_ONLY_HANDLER"}, + ] + }, + } + ), + encoding="utf-8", + ) + return path + + +def _lower_headers(request: Request) -> dict[str, str]: + return {name.lower(): value for name, value in request.headers.items()} + + +def test_custom_provider_map_base_video_config_serves_create_status_and_content( + gateway: Gateway, tmp_path: Path +) -> None: + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_queue_backend)) + config: Final = _write_config(tmp_path / "custom-video.yaml", wire) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config)) + + created: Final = owned.gateway.request( + "POST", "/v1/videos", {"model": "queue-video", "prompt": _PROMPT, "seconds": "8"} + ) + assert created.status_code == 200, created.text + created_body: Final = created.json() + assert created_body["status"] == "queued", created_body + video_id: Final = created_body["id"] + assert isinstance(video_id, str) and video_id.startswith("video_"), created_body + + submits: Final = wire.drain() + assert [(r.method, r.target) for r in submits] == [("POST", "/jobs")], submits + assert json.loads(submits[0].body) == {"model": "seedance-1", "prompt": _PROMPT, "seconds": "8"} + assert _lower_headers(submits[0])["x-api-key"] == "queue-secret" + + status: Final = owned.gateway.request("GET", f"/v1/videos/{video_id}") + assert status.status_code == 200, status.text + assert status.json()["status"] == "completed", status.text + assert status.json()["id"] == video_id, status.text + + content: Final = owned.gateway.request("GET", f"/v1/videos/{video_id}/content") + assert content.status_code == 200, content.text + assert content.content == _MP4 + + polls: Final = wire.drain() + assert [(r.method, r.target) for r in polls] == [("GET", "/jobs/job-7"), ("GET", "/jobs/job-7/file")], polls + assert all(_lower_headers(r)["x-api-key"] == "queue-secret" for r in polls) + + +def test_custom_provider_without_a_video_config_is_refused_with_400(gateway: Gateway, tmp_path: Path) -> None: + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_queue_backend)) + config: Final = _write_config(tmp_path / "custom-video.yaml", wire) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config)) + + refused: Final = owned.gateway.request("POST", "/v1/videos", {"model": "image-only-video", "prompt": _PROMPT}) + assert refused.status_code == 400, refused.text + assert "video generation is not supported for image-only" in refused.text + assert wire.drain() == () diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 4e091460667..bb4973d6f2f 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -139,6 +139,7 @@ _PROVIDERS: Final = ( False, "databricks/databricks-openjev-qwen35-4b", ), + _Provider("empiriolabs", "empiriolabs/aplomb-test", "/v1/decisions", "aplomb-test", _API_KEY, False, None), ) _LOCAL_PROVIDERS: Final = ( pytest.param( diff --git a/tests/integration/providers/test_nvidia_riva_wire.py b/tests/integration/providers/test_nvidia_riva_wire.py index ef36c449761..56d54a87600 100644 --- a/tests/integration/providers/test_nvidia_riva_wire.py +++ b/tests/integration/providers/test_nvidia_riva_wire.py @@ -135,3 +135,12 @@ def test_a_stalled_offline_recognize_answers_408_at_the_deployment_timeout(gatew assert timed_out.status_code == 408, timed_out.text calls: Final = eventually(server.drain, lambda observed: len(observed) > 0, seconds=30) assert all(call.rpc is Rpc.RECOGNIZE and call.cancelled for call in calls), calls + + +@pytest.mark.parametrize("riva_offline", (False, True), ids=("streaming", "offline")) +def test_the_deployment_api_key_reaches_riva_as_exactly_one_bearer_token(gateway: Gateway, riva_offline: bool) -> None: + with riva_server(_WORDS) as server, gateway.scenario() as scenario: + model: Final = _riva_model(scenario, server, api_key="riva-deployment-key", riva_offline=riva_offline) + served: Final = _transcribe(gateway, scenario.key(), model) + assert served.status_code == 200, served.text + assert [call.authorization for call in server.drain()] == [("Bearer riva-deployment-key",)] diff --git a/tests/integration/providers/test_openai_chat_wire.py b/tests/integration/providers/test_openai_chat_wire.py index 69295485380..5ba631ac776 100644 --- a/tests/integration/providers/test_openai_chat_wire.py +++ b/tests/integration/providers/test_openai_chat_wire.py @@ -1,4 +1,3 @@ -import base64 import json import uuid from collections.abc import Callable @@ -547,14 +546,8 @@ def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_ lambda found: len(found) == 2, seconds=70, ) - by_response_id: Final = { - (decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode()) - .rsplit("response_id:", 1)[1]: (decoded, row) - for row in rows - } - for response_id in response_ids: - decoded, row = by_response_id[response_id] - assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows + assert sorted(string_value(row["request_id"]) for row in rows) == sorted(response_ids), rows + for row in rows: assert ( string_value(row["status"]), string_value(row["cache_hit"]), @@ -601,11 +594,7 @@ def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cac seconds=70, ) priced, cached = rows - priced_request: Final = base64.b64decode( - string_value(priced["request_id"]).removeprefix("resp_") - ).decode() - assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows - assert priced_request.endswith(";response_id:resp_cached"), rows + assert string_value(priced["request_id"]) == "resp_cached", rows assert ( priced["status"], priced["cache_hit"], diff --git a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py index 4b25275285e..2e1bd73a4b4 100644 --- a/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py +++ b/tests/integration/providers/test_responses_bridge_message_cache_control_wire.py @@ -809,7 +809,10 @@ def test_provider_error_on_a_marked_request_reaches_the_caller(gateway: Gateway, model: Final = _anthropic_deployment(scenario, wire) answer: Final = _send(gateway, _marked_system(model, marker)) labels: Final = anthropic_labels(_only_request(wire)) - assert answer.status == 500, answer.text + assert answer.status == 503, answer.text + error: Final = object_value(_JSON_OBJECT.validate_json(answer.text)["error"]) + assert error["type"] == "service_unavailable_error", answer.text + assert error["code"] == "server_is_overloaded", answer.text assert "overloaded_error" in answer.text, answer.text assert labels == [SYSTEM_LABEL] spend.landed(model, answer.call_id, None, status="failure") diff --git a/tests/integration/providers/test_responses_lifecycle_wire.py b/tests/integration/providers/test_responses_lifecycle_wire.py index ac5b4dc6d5c..35a555fc776 100644 --- a/tests/integration/providers/test_responses_lifecycle_wire.py +++ b/tests/integration/providers/test_responses_lifecycle_wire.py @@ -4,7 +4,7 @@ import json import os import threading import uuid -from collections.abc import Callable +from collections.abc import Callable, Iterator from types import MappingProxyType from typing import Final, Literal from urllib.parse import parse_qsl, urlsplit @@ -12,7 +12,15 @@ from urllib.parse import parse_qsl, urlsplit import httpx import openai import pytest -from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.process import owned_proxy from integration._support.responses_vendor import same_response from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.worker_readiness import wait_for_worker_readiness @@ -26,6 +34,16 @@ _PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", " _ALIASES: Final = ("/v1", "", "/openai/v1") +@pytest.fixture(scope="module") +def cancel_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("responses-cancel") + with ( + gateway_from_environment() as shared, + owned_proxy(shared, directory, {"PROXY_BATCH_POLLING_ENABLED": "false"}, workers=_PROXY_WORKERS) as owned, + ): + yield owned + + class _DeleteResponse(BaseModel): model_config = ConfigDict(extra="forbid") @@ -737,7 +755,7 @@ def test_delete_returns_the_client_held_id_not_the_provider_id( @pytest.mark.parametrize("prefix", _ALIASES) def test_cancel_forwards_provider_id_and_returns_cancelled_response( - gateway: Gateway, + cancel_gateway: Gateway, prefix: str, ) -> None: raw_id: Final = f"resp_cancel_{uuid.uuid4().hex}" @@ -756,9 +774,9 @@ def test_cancel_forwards_provider_id_and_returns_cancelled_response( assert request.body in (b"", b"{}") return Reply(body=_response(raw_id, "cancelled")) - with wire_server(respond) as wire, gateway.scenario() as scenario: + with wire_server(respond) as wire, cancel_gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) - client: Final = _client(gateway, gateway.key, prefix) + client: Final = _client(cancel_gateway, cancel_gateway.key, prefix) created: Final = client.responses.create(model=model, input="cancel lifecycle", background=True) cancelled: Final = client.responses.cancel(created.id) assert cancelled.id != raw_id and same_response(cancelled.id, created.id), cancelled.model_dump_json() diff --git a/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py b/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py index 16d37980efa..a266b20b227 100644 --- a/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py +++ b/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py @@ -419,7 +419,7 @@ class _LoginDeployment: _LOGIN_DEPLOYMENTS: Final = ( _LoginDeployment("github_copilot", f"github_copilot/{_CODEX}", None), _LoginDeployment("chatgpt", f"chatgpt/{_CODEX}", None), - _LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3", "synthetic-azure-key"), + _LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3.2", "synthetic-azure-key"), ) diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 4e31122440d..19928ecd6e0 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -34,14 +34,20 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _USER_TOOL: Final = "lookup" _STRUCTURED_OUTPUTS_BETA: Final = "structured-outputs-2025-11-13" _TOOL_MODEL: Final = "claude-sonnet-5" _EFFORT_GATED_MODEL: Final = "claude-haiku-4-5" _NATIVE_MODEL: Final = "claude-opus-5-5" -_FORCED_TOOL_NATIVE_MODEL: Final = "claude-haiku-5-5" +_FORCED_TOOL_NATIVE_MODEL: Final = "claude-opus-4-8" +_THINKING_BY_DEFAULT_NATIVE_MODEL: Final = "claude-haiku-5-5" +_THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" +_BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8" _BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" @@ -608,11 +614,15 @@ def _one_delivery(wire: Wire, marker_hex: str) -> _Delivery: return sent -def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: +def _json_tool(delivery: _Delivery) -> dict[str, JsonValue]: tools: Final = tuple(object_value(tool) for tool in list_value(delivery.body["tools"])) (json_tool,) = tuple(tool for tool in tools if tool["name"] == _JSON_TOOL) - assert json_tool.keys() == {"name", "input_schema"}, json_tool - return object_value(json_tool["input_schema"]) + assert json_tool.keys() - {"description"} == {"name", "input_schema"}, json_tool + return json_tool + + +def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]: + return object_value(_json_tool(delivery)["input_schema"]) def _folded_format(delivery: _Delivery) -> dict[str, JsonValue]: @@ -830,10 +840,10 @@ def _family_deployment(gateway: Gateway, scenario: Scenario, api_base: str, memb case "azure_ai_unmapped": return scenario.model(model=f"azure_ai/{_UNMAPPED_MODEL}", api_base=api_base, api_key=_AZURE_KEY) case "bedrock_invoke": - return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_INVOKE_MODEL) + return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_TOOL_PATH_MODEL) case "bedrock_mantle": return scenario.model( - model=f"bedrock_mantle/{_BEDROCK_INVOKE_MODEL}", + model=f"bedrock_mantle/{_BEDROCK_TOOL_PATH_MODEL}", api_base=api_base, api_key=_MANTLE_KEY, aws_region_name="us-east-1", @@ -1448,21 +1458,24 @@ def test_tool_path_stream_cut_before_the_tool_block_closes_still_delivers_the_an _one_delivery(wire, marker_hex) -@pytest.mark.parametrize("accepts_forced_tools", [True, False], ids=["accepts_forced_tools", "rejects_forced_tools"]) -def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_accepts_forced_tools( - gateway: Gateway, accepts_forced_tools: bool +@pytest.mark.parametrize( + ("tool_model", "forced"), + [(_THINKING_OFF_TOOL_MODEL, True), (_TOOL_MODEL, False), (None, False)], + ids=["accepts_forced_tools", "thinks_by_default", "rejects_forced_tools"], +) +def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_will_not_think( + gateway: Gateway, tool_model: str | None, forced: bool ) -> None: marker, marker_hex = _marker("plain") - backend: Final = ( - _TOOL_MODEL if accepts_forced_tools else _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") - ) + backend: Final = tool_model or _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude") with wire_server(_peer) as wire, gateway.scenario() as scenario: model: Final = _vertex_deployment(gateway, scenario, wire.url, backend) response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker)) assert _chat_content(response) == _payload(marker_hex), response.text sent: Final = _one_delivery(wire, marker_hex) assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body - assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if accepts_forced_tools else None), sent.body + assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if forced else None), sent.body + assert _json_tool(sent).get("description") == (None if forced else _UNFORCED_JSON_TOOL_DESCRIPTION), sent.body assert "output_config" not in sent.body and "output_format" not in sent.body, sent.body @@ -1488,7 +1501,7 @@ def test_vertex_chat_json_schema_on_a_model_rejecting_forced_tools_maps_reasonin assert _reasoning_fields(structured_sent) == _reasoning_fields(plain_sent), (structured_sent.body, plain_sent.body) -def test_bedrock_invoke_json_schema_sends_the_json_tool_unforced_when_the_model_rejects_forced_tools( +def test_bedrock_invoke_json_schema_sends_native_structured_output_when_the_model_rejects_forced_tools( gateway: Gateway, ) -> None: marker, marker_hex = _marker("plain") @@ -1499,7 +1512,8 @@ def test_bedrock_invoke_json_schema_sends_the_json_tool_unforced_when_the_model_ assert _chat_content(response) == _payload(marker_hex), response.text sent: Final = _one_delivery(wire, marker_hex) assert sent.target == f"/model/{backend}/invoke", sent.target - assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body + assert object_value(_folded_format(sent)["schema"])["properties"] == _SCHEMA["properties"], sent.body + assert "tools" not in sent.body, sent.body assert "tool_choice" not in sent.body, sent.body @@ -1565,6 +1579,25 @@ def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_ke sent: Final = _one_delivery(wire, marker_hex) assert not _structured(sent.body), sent.body assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert sent.body.get("tool_choice") == _FORCED_JSON_TOOL, sent.body + + +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"]) +def test_open_object_schema_takes_the_unforced_json_tool_on_a_model_that_thinks_by_default( + gateway: Gateway, provider: str +) -> None: + marker, marker_hex = _marker("values") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _native_deployment(gateway, scenario, wire.url, provider, _THINKING_BY_DEFAULT_NATIVE_MODEL) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT) + ) + assert _chat_content(response) == _tool_input("values", marker_hex), response.text + sent: Final = _one_delivery(wire, marker_hex) + assert not _structured(sent.body), sent.body + assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body + assert _json_tool(sent).get("description") == _UNFORCED_JSON_TOOL_DESCRIPTION, sent.body + assert "tool_choice" not in sent.body, sent.body @pytest.mark.parametrize( diff --git a/tests/integration/providers/test_vertex_passthrough_stream_wire.py b/tests/integration/providers/test_vertex_passthrough_stream_wire.py new file mode 100644 index 00000000000..1ec2f740480 --- /dev/null +++ b/tests/integration/providers/test_vertex_passthrough_stream_wire.py @@ -0,0 +1,121 @@ +import json +import os +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy +from integration._support.process import owned_proxy +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, Wire, wire_server + +_LOCATION: Final = "us-central1" +_HOST: Final = f"{_LOCATION}-aiplatform.googleapis.com" +_PROJECT: Final = "audit-vertex-pass-through" +_STREAMING_MODEL: Final = "gemini-3.8-flash" +_UNAVAILABLE_MODEL: Final = "gemini-3.8-flash-unavailable" +_VENDOR_TOKEN: Final = "Bearer synthetic-vertex-token" +_CHUNKS: Final = ( + b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "Hel"}]}}]}\r\n\r\n', + b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "lo"}]}, "finishReason": "STOP"}],' + b' "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}}\r\n\r\n', +) +_UNAVAILABLE_MESSAGE: Final = "The service is currently unavailable." +_UNAVAILABLE_BODY: Final = json.dumps( + {"error": {"code": 503, "message": _UNAVAILABLE_MESSAGE, "status": "UNAVAILABLE"}} +).encode() +_SCRUBBED: Final = frozenset( + { + "HTTPS_PROXY", + "HTTP_PROXY", + "ALL_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "DEFAULT_VERTEXAI_PROJECT", + "DEFAULT_VERTEXAI_LOCATION", + "DEFAULT_GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_APPLICATION_CREDENTIALS", + } +) + + +@dataclass(frozen=True, slots=True) +class _Vertex: + wire: Wire + tunnel: ForwardProxy + proxy: Gateway + + def stream(self, model: str, marker: str) -> httpx.Response: + return self.proxy.request( + "POST", + f"/vertex_ai/v1/{_model_path(model)}:streamGenerateContent", + {"contents": [{"role": "user", "parts": [{"text": marker}]}]}, + headers={"Authorization": _VENDOR_TOKEN, "x-litellm-api-key": f"Bearer {self.proxy.key}"}, + ) + + +def _model_path(model: str) -> str: + return f"projects/{_PROJECT}/locations/{_LOCATION}/publishers/google/models/{model}" + + +def _respond(request: Request) -> Reply: + if request.target.startswith(f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:"): + return Reply(status=503, body=_UNAVAILABLE_BODY) + if request.target.startswith(f"/v1/{_model_path(_STREAMING_MODEL)}:"): + return Reply(content_type="text/event-stream", chunks=_CHUNKS) + return Reply(status=404, body=b'{"error": "no scripted Vertex model"}') + + +@pytest.fixture(scope="module") +def vertex(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Vertex]: + directory: Final = tmp_path_factory.mktemp("vertex-pass-through-stream") + cert, key = write_self_signed_cert(directory, (_HOST,)) + with ( + wire_server(_respond, tls=server_context(cert, key)) as wire, + tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel, + gateway_from_environment() as rig, + owned_proxy( + rig, + directory, + {"HTTPS_PROXY": tunnel.url, "NO_PROXY": "127.0.0.1,localhost", "SSL_CERT_FILE": str(cert)}, + remove_environment=tuple(name for name in os.environ if name.upper() in _SCRUBBED), + workers=2, + ) as proxy, + ): + yield _Vertex(wire, tunnel, proxy) + + +@pytest.fixture(autouse=True) +def fresh_edges(vertex: _Vertex) -> None: + vertex.wire.drain() + vertex.tunnel.drain() + + +def test_vertex_pass_through_stream_relays_every_sse_chunk_byte_for_byte(vertex: _Vertex) -> None: + marker: Final = uuid.uuid4().hex + response: Final = vertex.stream(_STREAMING_MODEL, marker) + sent: Final = vertex.wire.drain() + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream"), response.headers + assert response.content == b"".join(_CHUNKS), response.text + assert [request.target for request in sent] == [ + f"/v1/{_model_path(_STREAMING_MODEL)}:streamGenerateContent?alt=sse" + ], sent + assert sent[0].headers["authorization"] == _VENDOR_TOKEN + assert json.loads(sent[0].body) == {"contents": [{"role": "user", "parts": [{"text": marker}]}]} + assert set(vertex.tunnel.targets()) == {f"{_HOST}:443"} + + +def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Vertex) -> None: + response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex) + + assert response.status_code == 503, response.text + assert response.content == _UNAVAILABLE_BODY, response.text + assert [request.target for request in vertex.wire.drain()] == [ + f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse" + ] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4e2cab56d27..c552e11c22a 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -587,6 +587,11 @@ model_list: model: perplexity/pplx-decider-v1-27b api_base: http://127.0.0.1:8191 api_key: synthetic-perplexity-key + - model_name: parallel_ai/decision + litellm_params: + model: parallel_ai/decision + api_base: http://127.0.0.1:8191 + api_key: synthetic-parallel-key - model_name: typesafe/jev-1.13.0 litellm_params: model: typesafe/jev-1.13.0 diff --git a/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py b/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py new file mode 100644 index 00000000000..e6df49a73bd --- /dev/null +++ b/tests/integration/routing/test_anthropic_overloaded_cooldown_owned_proxy.py @@ -0,0 +1,247 @@ +import json +import os +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from urllib.parse import unquote + +import httpx +import pytest +import yaml +from integration._support.anthropic_sse import ( + LIFECYCLE, + error_body, + event_types, + message_id, + message_stream, + parse_sse, + status_reply, + stream_reply, + user_prompt, +) +from integration._support.client import ( + Gateway, + JsonValue, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server + +_OVERLOADED: Final = "claude-overloaded" +_HEALTHY: Final = "claude-healthy" +_COUNTED: Final = "claude-counted" +_COUNTED_OVERLOADED: Final = "claude-counted-overloaded" +_COUNTED_RATE_LIMITED: Final = "claude-counted-rate-limited" +_CHAT_ONLY_BACKEND: Final = "audit-chat-only" +_API_KEY: Final = "synthetic-anthropic-key" +_TEXT: Final = "Hello" +_COOLDOWN_GROUP: Final = "audit-overloaded-cooldown" +_COUNT_DISABLED_GROUP: Final = "audit-count-tokens-disabled" +_COUNT_UPSTREAM_GROUP: Final = "audit-count-tokens-upstream-503" +_COUNT_OVERLOADED_GROUP: Final = "audit-count-tokens-upstream-529" +_COUNT_RATE_LIMITED_GROUP: Final = "audit-count-tokens-upstream-429" +_GATEWAY: Final = "/claude_code_gateway" +_GATEWAY_GROUPS: Final = { + "non-stream": "audit-gateway-overloaded-non-stream", + "stream": "audit-gateway-overloaded-stream", +} +_COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429} +_COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens" +_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url" + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + + +def _served_id(marker: str) -> str: + return f"msg_{_HEALTHY}_{marker}" + + +def _respond(request: Request) -> Reply: + body: Final = object_value(json.loads(request.body)) + backend: Final = str(body["model"]) + if unquote(request.target) == _COUNT_TOKENS_TARGET: + status: Final = _COUNT_STATUSES[backend] + return Reply(status=status, body=error_body(status, f"scripted {status}")) + if backend == _OVERLOADED: + return status_reply(529) + if backend == _HEALTHY: + return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT)) + return Reply(status=404, body=b'{"error": "no scripted backend"}') + + +def _deployment(name: str, model: str, api_base: str, api_key: str, **params: JsonValue) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}} + + +def _config(wire: Wire, directory: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + _deployment(_COOLDOWN_GROUP, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=1), + _deployment(_COOLDOWN_GROUP, f"anthropic/{_HEALTHY}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=2), + _deployment( + _COUNT_DISABLED_GROUP, + f"hosted_vllm/{_CHAT_ONLY_BACKEND}", + _UPSTREAM_URL_PLACEHOLDER + "/v1", + "synthetic-vllm-key", + ), + _deployment(_COUNT_UPSTREAM_GROUP, f"anthropic/{_COUNTED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY), + _deployment(_COUNT_OVERLOADED_GROUP, f"anthropic/{_COUNTED_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY), + _deployment( + _COUNT_RATE_LIMITED_GROUP, f"anthropic/{_COUNTED_RATE_LIMITED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY + ), + *( + _deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY) + for group in _GATEWAY_GROUPS.values() + ), + ] + config["router_settings"] = { + "num_retries": 0, + "allowed_fails_policy": {"InternalServerErrorAllowedFails": 0}, + "cooldown_time": 120, + "redis_host": os.environ["REDIS_HOST"], + "redis_port": int(os.environ["REDIS_PORT"]), + } + config["litellm_settings"]["disable_token_counter"] = True + config["general_settings"]["enable_claude_code_gateway"] = True + path: Final = directory / "anthropic-overloaded-cooldown.yaml" + path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url)) + return path + + +@dataclass(frozen=True, slots=True) +class _Rig: + proxy: Gateway + wire: Wire + + def messages(self, path: str, model: str, marker: str, **extra: JsonValue) -> httpx.Response: + return self.proxy.request( + "POST", + path, + { + "model": model, + "max_tokens": 16, + "stream": True, + "messages": [{"role": "user", "content": marker}], + **extra, + }, + headers={"anthropic-version": "2023-06-01"}, + ) + + def count_tokens(self, model: str, prefix: str = "") -> httpx.Response: + return self.proxy.request( + "POST", + prefix + _COUNT_TOKENS_TARGET, + {"model": model, "messages": [{"role": "user", "content": "count me"}]}, + headers={"anthropic-version": "2023-06-01"}, + ) + + def targets_called(self) -> tuple[str, ...]: + return tuple(unquote(request.target) for request in self.wire.drain()) + + def backends_called(self) -> tuple[str, ...]: + return tuple(str(object_value(json.loads(request.body))["model"]) for request in self.wire.drain()) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-cooldown") + with gateway_from_environment() as gateway, wire_server(_respond) as wire: + overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _API_KEY} + with owned_proxy(gateway, directory, overrides, config=_config(wire, directory), workers=2) as proxy: + yield _Rig(proxy, wire) + + +@pytest.fixture(autouse=True) +def _fresh_wire(rig: _Rig) -> None: + rig.wire.drain() + + +def _assert_overloaded_json(response: httpx.Response) -> None: + assert response.status_code == 529, response.text + payload: Final = object_value(json.loads(response.content)) + assert object_value(payload["error"])["type"] == "overloaded_error", response.text + + +def _count_tokens_error(response: httpx.Response, status: int) -> dict[str, JsonValue]: + assert response.status_code == status, response.text + envelope: Final = object_value(json.loads(response.content)) + assert envelope["type"] == "error", response.text + return object_value(envelope["error"]) + + +def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_allowed_fails_policy(rig: _Rig) -> None: + marker: Final = uuid.uuid4().hex + first: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True) + _assert_overloaded_json(first) + assert rig.backends_called() == (_OVERLOADED,) + second: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True) + assert second.status_code == 200, second.text + events: Final = parse_sse(second.text) + assert event_types(events) == LIFECYCLE, events + assert message_id(events) == _served_id(marker), events + assert rig.backends_called() == (_HEALTHY,) + + +def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None: + response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP) + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text + assert "Token counting is disabled" in string_value(error["message"]), response.text + assert rig.targets_called() == () + + +def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None: + response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP) + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text + assert "scripted 503" in string_value(error["message"]), response.text + assert rig.targets_called() == (_COUNT_TOKENS_TARGET,) +@pytest.mark.parametrize( + ("group", "status", "error_type"), + ( + pytest.param(_COUNT_OVERLOADED_GROUP, 529, "overloaded_error", id="529"), + pytest.param(_COUNT_RATE_LIMITED_GROUP, 429, "rate_limit_error", id="429"), + ), +) +def test_count_tokens_keeps_other_upstream_statuses_and_their_error_body( + rig: _Rig, group: str, status: int, error_type: str +) -> None: + response: Final = rig.count_tokens(group) + error: Final = _count_tokens_error(response, status) + assert error["type"] == error_type, response.text + assert f"scripted {status}" in string_value(error["message"]), response.text + assert rig.targets_called() == (_COUNT_TOKENS_TARGET,) + + +def test_pass_through_messages_stream_forwards_an_upstream_529_verbatim(rig: _Rig) -> None: + response: Final = rig.messages("/anthropic/v1/messages", _OVERLOADED, uuid.uuid4().hex) + assert response.status_code == 529, response.text + assert response.content == error_body(529, "scripted 529"), response.text + assert rig.backends_called() == (_OVERLOADED,) + + +@pytest.mark.parametrize("shape", _GATEWAY_GROUPS.keys()) +def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overloaded_error( + rig: _Rig, shape: str +) -> None: + response: Final = rig.messages( + f"{_GATEWAY}/v1/messages", + _GATEWAY_GROUPS[shape], + uuid.uuid4().hex, + stream=shape == "stream", + num_retries=0, + ) + _assert_overloaded_json(response) + assert rig.backends_called() == (_OVERLOADED,) + + +def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None: + response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY) + error: Final = _count_tokens_error(response, 503) + assert error["type"] == "api_error", response.text + assert "scripted 503" in string_value(error["message"]), response.text + assert rig.targets_called() == (_COUNT_TOKENS_TARGET,) diff --git a/tests/integration/routing/test_least_busy_concurrent_decisions.py b/tests/integration/routing/test_least_busy_concurrent_decisions.py new file mode 100644 index 00000000000..bb923534db4 --- /dev/null +++ b/tests/integration/routing/test_least_busy_concurrent_decisions.py @@ -0,0 +1,90 @@ +import json +import threading +from collections import Counter +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +pytestmark = pytest.mark.timeout(240) + +_GROUP: Final = "least-busy-decider" +_DEPLOYMENTS: Final = ("least-busy-decider-a", "least-busy-decider-b") +_BURST: Final = 4 +_REPLY: Final = json.dumps( + { + "model": "Qwen/Qwen3-0.6B", + "answers": {"billing": {"type": "noul", "noul": 0.91}}, + "usage": {"input_tokens": 367, "output_tokens": 3}, + } +).encode() +_DECISION: Final = { + "model": _GROUP, + "input": "I was charged twice for order 4417.", + "questions": [{"type": "predicate", "name": "billing", "instructions": "Is this a billing issue?"}], +} + + +def _held_until_the_burst_arrives(arrived: threading.Barrier) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + arrived.wait() + return Reply(body=_REPLY) + + return respond + + +def _write_config(path: Path, provider_url: str) -> Path: + path.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": _GROUP, + "litellm_params": {"model": "hosted_vllm/Qwen/Qwen3-0.6B", "api_base": provider_url}, + "model_info": {"id": deployment_id}, + } + for deployment_id in _DEPLOYMENTS + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + }, + "router_settings": {"routing_strategy": "least-busy", "num_retries": 0, "disable_cooldowns": True}, + } + ) + ) + return path + + +def _decide(proxy: Gateway) -> httpx.Response: + return proxy.client.post( + "/v1/decisions", json=_DECISION, headers={"Authorization": f"Bearer {proxy.key}"}, timeout=120 + ) + + +def test_least_busy_spreads_a_burst_of_decisions_calls_across_the_group(tmp_path: Path) -> None: + arrived: Final = threading.Barrier(_BURST, timeout=60) + with ( + wire_server(_held_until_the_burst_arrives(arrived)) as upstream, + gateway_from_environment() as rig, + owned_proxy( + rig, + tmp_path, + {"STORE_MODEL_IN_DB": "False"}, + config=_write_config(tmp_path / "least-busy.yaml", upstream.url), + ) as proxy, + ThreadPoolExecutor(max_workers=_BURST) as pool, + ): + responses: Final = tuple(pool.map(lambda _: _decide(proxy), range(_BURST))) + assert [response.status_code for response in responses] == [200] * _BURST, [r.text for r in responses] + assert Counter(response.headers["x-litellm-model-id"] for response in responses) == Counter( + {deployment_id: _BURST // len(_DEPLOYMENTS) for deployment_id in _DEPLOYMENTS} + ) diff --git a/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py b/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py index 0257b8372b0..e0900bc0926 100644 --- a/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py +++ b/tests/integration/routing/test_messages_stream_retry_budget_sources_owned_proxy.py @@ -45,10 +45,19 @@ _FALLBACK_GROUP: Final = "audit-fallback" _CW_FALLBACK_GROUP: Final = "audit-cw-fallback" _LONELY: Final = "audit-lonely" _POLICY: Final = "audit-policy" +_SERVICE_UNAVAILABLE_RETRY_POLICY: Final = "audit-service-unavailable-retry-policy" +_INTERNAL_RETRY_POLICY: Final = "audit-internal-retry-policy" _UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url" Behavior = Literal[ - "drop-once", "drop-always", "hold-ping", "drop-then-too-long", "drop-then-401", "overloaded-frames-fallback-503" + "drop-once", + "drop-always", + "hold-ping", + "drop-then-too-long", + "drop-then-401", + "overloaded-frames-fallback-529", + "overloaded-frames", + "overloaded-once", ] pytestmark = pytest.mark.timeout(240) @@ -81,13 +90,15 @@ def _primary_reply( return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum")) case "drop-then-401": return dropped if attempt == 1 else status_reply(401) - case "overloaded-frames-fallback-503": + case "overloaded-frames-fallback-529" | "overloaded-frames": return stream_reply((error_frame(529, "scripted overloaded"),)) + case "overloaded-once": + return status_reply(529) if attempt == 1 else stream_reply(full) raise AssertionError(behavior) def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply: - if behavior == "overloaded-frames-fallback-503": + if behavior == "overloaded-frames-fallback-529": return status_reply(503) return stream_reply(full) @@ -141,13 +152,19 @@ def _config(wire: Wire, directory: Path) -> Path: _deployment(_CW_FALLBACK_GROUP, _CONTEXT_WINDOW), _deployment(_LONELY, _PRIMARY, num_retries=1), _deployment(_POLICY, _PRIMARY), + _deployment(_SERVICE_UNAVAILABLE_RETRY_POLICY, _PRIMARY), + _deployment(_INTERNAL_RETRY_POLICY, _PRIMARY), ] config["router_settings"] = { "num_retries": 1, "disable_cooldowns": True, "fallbacks": [{_WITH_FALLBACKS: [_FALLBACK_GROUP]}], "context_window_fallbacks": [{_WITH_FALLBACKS: [_CW_FALLBACK_GROUP]}], - "model_group_retry_policy": {_POLICY: {"DefaultRetries": 2}}, + "model_group_retry_policy": { + _POLICY: {"DefaultRetries": 2}, + _SERVICE_UNAVAILABLE_RETRY_POLICY: {"ServiceUnavailableErrorRetries": 1, "InternalServerErrorRetries": 0}, + _INTERNAL_RETRY_POLICY: {"InternalServerErrorRetries": 1, "ServiceUnavailableErrorRetries": 0}, + }, } path: Final = directory / "messages-retry-budget-sources.yaml" path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url)) @@ -290,10 +307,34 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha assert rig.attempts_on(_PRIMARY, marker) == 2 -def test_overloaded_frames_whose_fallback_fails_answer_the_mapped_internal_server_error(rig: _Rig) -> None: +def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None: marker: Final = uuid.uuid4().hex - response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker) - assert response.status_code == 500, response.text + response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker) + assert response.status_code == 529, response.text + assert "overloaded_error" in response.text, response.text assert "content_block_delta" not in response.text, response.text assert rig.attempts_on(_PRIMARY, marker) == 2 assert rig.attempts_on(_FALLBACK, marker) == 2 + + +def test_overloaded_frames_that_spend_the_budget_without_a_fallback_relay_the_last_attempts_frame(rig: _Rig) -> None: + marker: Final = uuid.uuid4().hex + response: Final = rig.stream(_LONELY, "overloaded-frames", marker) + assert response.status_code == 200, response.text + assert response.text == error_frame(529, "scripted overloaded").decode(), response.text + assert rig.attempts_on(_PRIMARY, marker) == 2 + + +def test_a_529_is_retried_under_the_internal_server_error_retry_policy(rig: _Rig) -> None: + marker: Final = uuid.uuid4().hex + _assert_completed(rig.stream(_INTERNAL_RETRY_POLICY, "overloaded-once", marker), _served_id(_PRIMARY, marker, 2)) + assert rig.attempts_on(_PRIMARY, marker) == 2 + + +def test_a_529_is_not_retried_under_the_service_unavailable_error_retry_policy(rig: _Rig) -> None: + marker: Final = uuid.uuid4().hex + response: Final = rig.stream(_SERVICE_UNAVAILABLE_RETRY_POLICY, "overloaded-once", marker) + assert response.status_code == 529, response.text + assert "overloaded_error" in response.text, response.text + assert "content_block_delta" not in response.text, response.text + assert rig.attempts_on(_PRIMARY, marker) == 1 diff --git a/tests/integration/routing/test_prompt_cache_pin_ttl.py b/tests/integration/routing/test_prompt_cache_pin_ttl.py index f6ee96a9206..733f7859ff0 100644 --- a/tests/integration/routing/test_prompt_cache_pin_ttl.py +++ b/tests/integration/routing/test_prompt_cache_pin_ttl.py @@ -9,7 +9,7 @@ from typing import Final import anthropic import openai import pytest -from integration._support.client import eventually, gateway_from_environment, held +from integration._support.client import eventually, gateway_from_environment, held, object_value from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.redis_process import owned_redis from integration._support.wire import Wire, wire_server @@ -56,11 +56,12 @@ from integration.routing._prompt_cache_pin_support import ( vertex_peer, wire_carries_one_hour, ) -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter from redis import Redis pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 300), pytest.mark.shares_worker_state] +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _CLAUDE: Final = "pin-claude" _SLOW: Final = "pin-slow" _MIXED: Final = "pin-mixed" @@ -539,7 +540,10 @@ def test_failed_first_turn_reports_the_error_and_writes_no_pin(rig: _Rig) -> Non "/v1/chat/completions", chat_body(_CLAUDE, _first_turn(new_marker(), f"{new_marker()} {OVERLOAD_TOKEN}", ONE_HOUR_MARK)), ) - assert response.status_code == 500, response.text + assert response.status_code == 503, response.text + error: Final = object_value(_JSON_OBJECT.validate_json(response.text)["error"]) + assert error["type"] == "service_unavailable_error", response.text + assert error["code"] == "server_is_overloaded", response.text assert "overloaded_error" in response.text, response.text healthy, _ = post_chat(rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), new_marker(), None))) spend_row_lands(healthy) diff --git a/tests/integration/routing/test_reasoning_effort_levels_wire.py b/tests/integration/routing/test_reasoning_effort_levels_wire.py index d15867ac572..efca26d38d0 100644 --- a/tests/integration/routing/test_reasoning_effort_levels_wire.py +++ b/tests/integration/routing/test_reasoning_effort_levels_wire.py @@ -1977,6 +1977,9 @@ _INVOKE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = { "format": {"type": "json_schema", "schema": _NATIVE_OUTPUT_SCHEMA}, } _JSON_TOOL: Final = "json_tool_call" +_UNFORCED_JSON_TOOL_DESCRIPTION: Final = ( + "When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text." +) _CONVERSE_TOOL_SCHEMA: Final[dict[str, JsonValue]] = { key: value for key, value in _NATIVE_OUTPUT_SCHEMA.items() if key != "additionalProperties" } @@ -1994,7 +1997,7 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( { "toolSpec": { "name": _JSON_TOOL, - "description": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, "inputSchema": {"json": _CONVERSE_TOOL_SCHEMA}, } } @@ -2004,7 +2007,15 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields( ) _INVOKE_WITH_JSON_TOOL: Final = _with_fields( _without(_invoke_chat("high", _ADAPTIVE, ()), "anthropic_beta"), - {"tools": [{"name": _JSON_TOOL, "input_schema": _NATIVE_OUTPUT_SCHEMA}]}, + { + "tools": [ + { + "name": _JSON_TOOL, + "description": _UNFORCED_JSON_TOOL_DESCRIPTION, + "input_schema": _NATIVE_OUTPUT_SCHEMA, + } + ] + }, ) diff --git a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py index 54a6df02edf..294791128ab 100644 --- a/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py +++ b/tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py @@ -27,7 +27,7 @@ _RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { "json_schema": {"name": "risk", "schema": _SCHEMA}, } _ANTHROPIC_MODEL: Final = "anthropic/claude-scripted-9" -_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-sonnet-5" +_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-opus-4-8" def _payload(marker_hex: str) -> dict[str, JsonValue]: diff --git a/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py b/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py new file mode 100644 index 00000000000..358b36cd2b0 --- /dev/null +++ b/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py @@ -0,0 +1,95 @@ +import json +from collections.abc import Awaitable +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, wire_server + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.realtime_api import main as realtime_main + +UPSTREAM_PATHS: Final = MappingProxyType( + { + "client_secrets": "/v1/realtime/client_secrets", + "translation_client_secrets": "/v1/realtime/translations/client_secrets", + "calls": "/v1/realtime/calls", + "translation_calls": "/v1/realtime/translations/calls", + } +) + + +def _logging(route: str) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id=f"test-realtime-{route}", + function_id=f"test-realtime-{route}", + ) + + +def _route_call(route: str, api_base: str) -> Awaitable[httpx.Response]: + match route: + case "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-test", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + custom_llm_provider="openai", + api_key="sk-test", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case "calls": + return realtime_main.arealtime_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime", + custom_llm_provider="openai", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case _: + return realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime-translate", + custom_llm_provider="openai", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + + +def _ephemeral_key(_: Request) -> Reply: + return Reply(body=json.dumps({"value": "ek_test"}).encode()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(UPSTREAM_PATHS)) +async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify( + route: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + certificate, key = write_self_signed_cert(tmp_path) + monkeypatch.setattr(litellm, "ssl_verify", str(certificate)) + + with wire_server(_ephemeral_key, tls=server_context(certificate, key)) as upstream: + response: Final = await _route_call(route, upstream.url) + targets: Final = tuple(request.target for request in upstream.drain()) + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test"} + assert targets == (UPSTREAM_PATHS[route],) diff --git a/tests/integration/spend/test_baggage_tag_mappings.py b/tests/integration/spend/test_baggage_tag_mappings.py index e08348a074c..af68d7b242b 100644 --- a/tests/integration/spend/test_baggage_tag_mappings.py +++ b/tests/integration/spend/test_baggage_tag_mappings.py @@ -61,10 +61,10 @@ class Rig: @property def exact_tag(self) -> str: - return f"usecase-a-{self.marker}" + return f"app-a-{self.marker}" - def usecase_tag(self, value: str) -> str: - return f"usecase-{self.marker}:{value}" + def app_tag(self, value: str) -> str: + return f"app-{self.marker}:{value}" def tenant_tag(self, value: str) -> str: return f"tenant-{self.marker}:{value}" @@ -195,7 +195,7 @@ def _routed_deployment(marker: str, provider: Wire, name: str, tag: str) -> dict def _mapped_models(marker: str, provider: Wire, upstream_url: str) -> list[JsonValue]: routed: Final = ( - _routed_deployment(marker, provider, "tagged", f"usecase-a-{marker}"), + _routed_deployment(marker, provider, "tagged", f"app-a-{marker}"), _routed_deployment(marker, provider, "default", "default"), ) realtime: Final[dict[str, JsonValue]] = { @@ -221,8 +221,8 @@ def _mapped_models(marker: str, provider: Wire, upstream_url: str) -> list[JsonV def _mapped_rules(marker: str, *, tenant_required: bool) -> list[JsonValue]: return [ - {"key": "entrypoint", "value": "a", "tag": f"usecase-a-{marker}"}, - {"key": "entrypoint", "tag_prefix": f"usecase-{marker}:"}, + {"key": "service", "value": "a", "tag": f"app-a-{marker}"}, + {"key": "service", "tag_prefix": f"app-{marker}:"}, {"key": "tenant", "tag_prefix": f"tenant-{marker}:", "required": tenant_required}, {"key": "budgeted", "tag_prefix": f"budgeted-{marker}:", "require_budgeted_tag": True}, {"key": "team+id", "tag_prefix": f"team-{marker}:"}, @@ -403,7 +403,7 @@ def test_mapped_baggage_tags_reach_spend_and_callbacks_but_not_requester_metadat ROUTE_PATHS[route], _inference_body(route, mapped.chat_model, f"baggage {tenant}", caller_tag), key=key, - headers={"baggage": f"entrypoint=a, tenant={tenant}"}, + headers={"baggage": f"service=a, tenant={tenant}"}, ) assert response.status_code == 200, response.text assert len(mapped.provider.drain()) == 1 @@ -423,7 +423,7 @@ def test_json_string_metadata_gains_exactly_the_mapped_tags(optional: Rig) -> No key: Final = scenario.key() optional.provider.drain() plain: Final = _chat(optional, plain_key, None, fields=fields) - response: Final = _chat(optional, key, f"entrypoint=a,tenant={tenant}", fields=fields) + response: Final = _chat(optional, key, f"service=a,tenant={tenant}", fields=fields) assert (plain.status_code, response.status_code) == (200, 200), (plain.text, response.text) assert len(optional.provider.drain()) == 2 assert _spend_tags(optional.proxy, key) == _spend_tags(optional.proxy, plain_key) | mapped_tags @@ -494,7 +494,7 @@ def test_a2a_spend_row_carries_the_mapped_tags(mapped: Rig) -> None: }, }, key=key, - headers={"baggage": f"entrypoint=a,tenant={tenant}"}, + headers={"baggage": f"service=a,tenant={tenant}"}, ) assert sent.status_code == 200, sent.text assert {mapped.exact_tag, mapped.tenant_tag(tenant)} <= _spend_tags(mapped.proxy, key) @@ -514,8 +514,8 @@ def test_spent_mapped_tag_budget_rejects_the_next_request_before_the_provider( mapped: Rig, key_fields: Mapping[str, JsonValue], fields: Mapping[str, JsonValue], headers: Mapping[str, str] ) -> None: entry: Final = uuid.uuid4().hex - tag: Final = mapped.usecase_tag(entry) - baggage: Final = f"entrypoint={entry},tenant={entry}" + tag: Final = mapped.app_tag(entry) + baggage: Final = f"service={entry},tenant={entry}" with mapped.proxy.scenario() as scenario: key: Final = scenario.key(**key_fields) mapped.proxy.post("/tag/new", {"name": tag, "max_budget": 0.0001}) @@ -528,7 +528,7 @@ def test_spent_mapped_tag_budget_rejects_the_next_request_before_the_provider( assert blocked.status_code == 422, blocked.text assert "budget" in blocked.text.lower(), blocked.text assert mapped.provider.drain() == () - other: Final = _chat(mapped, key, f"entrypoint=other-{entry},tenant={entry}") + other: Final = _chat(mapped, key, f"service=other-{entry},tenant={entry}") assert other.status_code == 200, other.text @@ -536,10 +536,10 @@ def test_required_mapping_rejects_a_request_without_its_baggage_value_before_the with mapped.proxy.scenario() as scenario: key: Final = scenario.key() mapped.provider.drain() - rejected: Final = _chat(mapped, key, "entrypoint=a") + rejected: Final = _chat(mapped, key, "service=a") assert rejected.status_code == 400, rejected.text assert mapped.provider.drain() == () - admitted: Final = _chat(mapped, key, f"entrypoint=a,tenant={uuid.uuid4().hex}") + admitted: Final = _chat(mapped, key, f"service=a,tenant={uuid.uuid4().hex}") assert admitted.status_code == 200, admitted.text @@ -591,7 +591,7 @@ def test_mapped_tags_never_select_a_tag_routed_deployment(mapped: Rig, caller_ta _chat( mapped, key, - f"entrypoint=a,tenant={tenant}", + f"service=a,tenant={tenant}", model=f"routed-{mapped.marker}", fields=fields, **headers, @@ -604,8 +604,8 @@ def test_mapped_tags_never_select_a_tag_routed_deployment(mapped: Rig, caller_ta def test_mapped_tags_never_trigger_a_tag_scoped_policy_attachment(mapped: Rig) -> None: entry: Final = uuid.uuid4().hex - tag: Final = mapped.usecase_tag(entry) - baggage: Final = f"entrypoint={entry},tenant={entry}" + tag: Final = mapped.app_tag(entry) + baggage: Final = f"service={entry},tenant={entry}" proxy: Final = mapped.proxy with wire_server(lambda _request: Reply(body=GUARDRAIL_VERDICT)) as guardrail, proxy.scenario() as scenario: key: Final = scenario.key() @@ -662,7 +662,7 @@ def test_required_mapping_refuses_websocket_inference_before_the_upstream(mapped upstream.get("/__observations").raise_for_status() status: Final = _handshake_status( f"ws{proxy_url.removeprefix('http')}/v1/realtime?model=realtime-{mapped.marker}", - {"Authorization": f"Bearer {key}", "baggage": f"entrypoint=a,tenant={uuid.uuid4().hex}"}, + {"Authorization": f"Bearer {key}", "baggage": f"service=a,tenant={uuid.uuid4().hex}"}, ) assert status == 403, status observed: Final = list_value(JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]) @@ -706,7 +706,7 @@ def test_custom_passthrough_spend_row_carries_the_mapped_tags(mapped: Rig) -> No path, {"model": mapped.chat_model, "messages": [{"role": "user", "content": tenant}]}, key=key, - headers={"baggage": f"entrypoint=a,tenant={tenant}"}, + headers={"baggage": f"service=a,tenant={tenant}"}, ) assert response.status_code == 200, response.text assert {mapped.exact_tag, mapped.tenant_tag(tenant)} <= _spend_tags(mapped.proxy, key) @@ -777,7 +777,7 @@ def test_passthrough_forwards_the_client_body_unchanged_while_its_spend_row_carr plain: Final = optional.proxy.request("POST", path, body, key=plain_key) plain_forwarded: Final = _forwarded_bodies(optional) response: Final = optional.proxy.request( - "POST", path, body, key=key, headers={"baggage": f"entrypoint=a,tenant={tenant}"} + "POST", path, body, key=key, headers={"baggage": f"service=a,tenant={tenant}"} ) assert (plain.status_code, response.status_code) == (200, 200), (plain.text, response.text) assert plain_forwarded == (_upstream_body(entrypoint, body),) @@ -929,7 +929,7 @@ def test_reject_clientside_metadata_tags_with_baggage( ) -> None: tenant: Final = uuid.uuid4().hex fields, headers = _rejected_client_tag(client_tag, f"caller-{tenant}") - baggage: Final = f"entrypoint=a,tenant={tenant}" if baggage_maps else f"region={tenant}" + baggage: Final = f"service=a,tenant={tenant}" if baggage_maps else f"region={tenant}" mapped_tags: Final = frozenset({rejecting.exact_tag, rejecting.tenant_tag(tenant)} if baggage_maps else ()) with rejecting.proxy.scenario() as scenario: plain_key: Final = scenario.key() @@ -952,7 +952,7 @@ def test_plus_sign_rule_key_matches_either_wire_spelling_but_not_both( optional: Rig, baggage_keys: tuple[str, ...], maps_team: bool ) -> None: value: Final = uuid.uuid4().hex - baggage: Final = ",".join(("entrypoint=a", *(f"{baggage_key}={value}" for baggage_key in baggage_keys))) + baggage: Final = ",".join(("service=a", *(f"{baggage_key}={value}" for baggage_key in baggage_keys))) with optional.proxy.scenario() as scenario: key: Final = scenario.key() response: Final = _chat(optional, key, baggage) @@ -991,7 +991,7 @@ def test_baggage_mapping_a_deployment_tag_keeps_the_team_tag_guardrail_mode_firs @pytest.mark.parametrize("metadata", ["run-123", {"tags": "run-123"}], ids=["non_object", "string_tags"]) @pytest.mark.parametrize( ("header_tag", "baggage"), - [(True, "region=sg"), (True, "entrypoint=a"), (False, "region=sg"), (False, "entrypoint=a")], + [(True, "region=sg"), (True, "service=a"), (False, "region=sg"), (False, "service=a")], ids=["header_tag_unmapped_baggage", "header_tag_mapped_baggage", "unmapped_baggage", "mapped_baggage"], ) def test_malformed_metadata_keeps_its_status_under_baggage( diff --git a/tests/integration/spend/test_failing_request_stored_snapshot.py b/tests/integration/spend/test_failing_request_stored_snapshot.py new file mode 100644 index 00000000000..4f016c33f37 --- /dev/null +++ b/tests/integration/spend/test_failing_request_stored_snapshot.py @@ -0,0 +1,84 @@ +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment, object_value +from integration._support.process import owned_proxy +from pydantic import JsonValue + +UNREACHABLE_BASE: Final = "http://127.0.0.1:9/v1" +TRANSCRIPT_TURNS: Final = 40 +RETRIES: Final = 2 +STORED_REQUEST_BUDGET_KB: Final = 64 + + +def _config_storing_prompts(directory: Path) -> Path: + config: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general_settings: Final = object_value(config["general_settings"]) + path: Final = directory / "store-prompts.yaml" + path.write_text( + yaml.safe_dump({**config, "general_settings": {**general_settings, "store_prompts_in_spend_logs": True}}) + ) + return path + + +@pytest.fixture(scope="module") +def gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with gateway_from_environment() as parent: + directory: Final = tmp_path_factory.mktemp("failing_request_stored_snapshot") + with owned_proxy(parent, directory, {}, config=_config_storing_prompts(directory)) as owned: + yield owned + + +def _transcript(marker: str) -> list[JsonValue]: + turns: Final[list[JsonValue]] = [ + {"role": role, "content": f"turn {turn // 2} {role}"} + for turn, role in enumerate(("user", "assistant") * TRANSCRIPT_TURNS) + ] + return [*turns, {"role": "user", "content": marker}] + + +def _stored_request(gateway: Gateway, call_id: str) -> JsonValue: + rows: Final = eventually( + lambda: gateway.rows('SELECT proxy_server_request FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (call_id,)), + lambda values: len(values) == 1, + seconds=40, + ) + stored: Final = rows[0]["proxy_server_request"] + return json.loads(stored) if isinstance(stored, str) else stored + + +def test_failing_request_with_retries_and_fallback_stores_one_copy_of_the_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + primary: Final = f"stored-snapshot-{uuid.uuid4().hex}" + fallback: Final = f"stored-snapshot-fb-{uuid.uuid4().hex}" + for name in (primary, primary, fallback, fallback): + scenario.model(name=name, api_base=UNREACHABLE_BASE, cooldown_time=0) + key: Final = scenario.key(models=[primary, fallback]) + marker: Final = f"stored snapshot {uuid.uuid4().hex}" + + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": primary, + "max_tokens": 16, + "messages": _transcript(marker), + "router_settings_override": {"num_retries": RETRIES, "fallbacks": [{primary: [fallback]}]}, + }, + key=key, + ) + assert response.status_code != 200, response.text + assert fallback in response.text, response.text + + serialized: Final = json.dumps(_stored_request(gateway, response.headers["x-litellm-call-id"])) + stored_kb: Final = len(serialized.encode()) / 1024 + assert marker in serialized, serialized[:400] + assert stored_kb <= STORED_REQUEST_BUDGET_KB, ( + f"one failing request stored a {stored_kb:.0f} KB request snapshot for a " + f"{2 * TRANSCRIPT_TURNS + 1}-message transcript with {RETRIES} retries and a fallback" + ) diff --git a/tests/integration/spend/test_partial_stream_live_budget.py b/tests/integration/spend/test_partial_stream_live_budget.py index 89fb66c9bbf..cba51873e2d 100644 --- a/tests/integration/spend/test_partial_stream_live_budget.py +++ b/tests/integration/spend/test_partial_stream_live_budget.py @@ -346,5 +346,15 @@ def test_a_broken_messages_stream_counts_toward_the_budget(gateway: Gateway) -> _broken_messages_stream(gateway, model, key) after: Final = _messages(gateway, model, key, streamed=False) - assert_budget_refusal(after) + assert after.status_code == 422, after.text + body: Final = JSON_OBJECT.validate_json(after.content) + error: Final = JSON_OBJECT.validate_python(body["error"]) + assert body == { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": error["message"], + "litellm_error_type": "budget_exceeded", + }, + }, after.text assert _rows(gateway, key, 2) == (metered, StatusRow(status="failure", spend=0.0)) diff --git a/tests/integration/spend/test_responses_bridge_spend_once.py b/tests/integration/spend/test_responses_bridge_spend_once.py index 9eddd832ca0..cc2a0af58a7 100644 --- a/tests/integration/spend/test_responses_bridge_spend_once.py +++ b/tests/integration/spend/test_responses_bridge_spend_once.py @@ -1,5 +1,4 @@ import json -from base64 import b64decode from collections.abc import Callable from hashlib import sha256 from typing import Final @@ -100,8 +99,7 @@ def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_on call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id] assert len(call_rows) == 1, rows assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows - logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode() - assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows + assert call_rows[0]["request_id"] == upstream_id, rows key_spend: Final = eventually( lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py new file mode 100644 index 00000000000..7989cb6a7bc --- /dev/null +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -0,0 +1,467 @@ +import itertools +import json +import os +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql +from pydantic import BaseModel, ConfigDict, Field + +from litellm.proxy.spend_tracking.session_page import read_session_page +from litellm.proxy.spend_tracking.spend_management_endpoints import _fetch_session_representatives +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_URL: Final = "/spend/logs/ui" +_WINDOW_SECONDS: Final = 3600 + + +@dataclass(frozen=True, slots=True) +class _Row: + request: str + session: str | None + api_key: str + second: int + status: str = "success" + model: str = "m-a" + + +_ROWS: Final = ( + _Row("z1", "z", "key-1", -1), + _Row("a1", "a", "key-1", 10), + _Row("a2", "a", "key-1", 50), + _Row("b1", "b", "key-1", 20, model="m-b"), + _Row("b2", "b", "key-1", 60), + _Row("b3", "b", "key-1", 30), + _Row("c1", "c", "key-2", 60), + _Row("d1", "b", "key-2", 40, model="m-b"), + _Row("e1", None, "key-1", 55, model="m-b"), + _Row("f1", "", "key-1", 45), + _Row("g1", "g", "key-3", 5, status="failure"), + _Row("g2", "g", "key-3", 15, status="failure"), + *(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)), + _Row("i1", "i", "key-3", 35, status="failure"), + _Row("i2", "i", "key-3", _WINDOW_SECONDS + 1), +) +_NEWEST_FIRST: Final = ( + ("h", "key-1"), + ("c", "key-2"), + ("b", "key-1"), + ("e1", "key-1"), + ("a", "key-1"), + ("f1", "key-1"), + ("b", "key-2"), + ("i", "key-3"), + ("g", "key-3"), +) + + +class _LogRow(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + session_id: str | None + api_key: str + session_total_count: int + + +class _Page(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_LogRow, ...] + total: int + total_is_capped: bool + has_more: bool + next_session_cursor: str | None + + +@dataclass(frozen=True, slots=True) +class _Seeded: + prefix: str + window: Mapping[str, str] + + def session(self, row: _LogRow) -> tuple[str, str]: + key: Final = row.session_id or row.request_id + assert key.startswith(self.prefix), key + return key.removeprefix(self.prefix), row.api_key + + +@pytest.fixture +def seeded(gateway: Gateway) -> Iterator[_Seeded]: + prefix: Final = f"sess{uuid.uuid4().hex[:12]}-" + start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000) + rows: Final = [ + { + "request_id": f"{prefix}{row.request}", + "session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "", + "api_key": row.api_key, + "start_time": (start + timedelta(seconds=row.second)).isoformat(), + "status": row.status, + "model": row.model, + } + for row in _ROWS + ] + with gateway.scenario() as scenario: + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",) + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, ' + "session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, " + "status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, " + "start_time timestamp, status text, model text)", + (json.dumps(rows),), + ) + yield _Seeded( + prefix, + { + "start_date": f"{start:%Y-%m-%d %H:%M:%S}", + "end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}", + "group_by_session": "true", + }, + ) + + +def _page(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str]) -> _Page: + return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params})) + + +def _cursor_walk(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str], number: int = 1) -> tuple[_Page, ...]: + page: Final = _page(gateway, seeded, {**params, "page_size": "2", "page": str(number)}) + if page.next_session_cursor is None or number > len(_NEWEST_FIRST): + return (page,) + following: Final = {**params, "session_cursor": page.next_session_cursor} + return (page, *_cursor_walk(gateway, seeded, following, number + 1)) + + +def _rows(pages: tuple[_Page, ...]) -> Iterator[_LogRow]: + return itertools.chain.from_iterable(page.data for page in pages) + + +def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]: + return tuple(seeded.session(row) for row in _rows(pages)) + + +def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row( + gateway: Gateway, seeded: _Seeded +) -> None: + pages: Final = _cursor_walk(gateway, seeded, {}) + assert _sessions(seeded, pages) == _NEWEST_FIRST + assert [(page.total, page.total_is_capped) for page in pages] == [(9, False)] * 5 + assert [page.has_more for page in pages] == [True, True, True, True, False] + representatives: Final = tuple( + (row.request_id.removeprefix(seeded.prefix), row.session_total_count) for row in _rows(pages) + ) + assert representatives == ( + ("h6", 6), + ("c1", 1), + ("b2", 3), + ("e1", 1), + ("a2", 2), + ("f1", 1), + ("d1", 1), + ("i1", 2), + ("g2", 2), + ) + + +@pytest.mark.parametrize( + "sort_order, expected", + [("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))], + ids=["newest-first", "oldest-first"], +) +def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk( + gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...] +) -> None: + pages: Final = tuple( + _page(gateway, seeded, {"page_size": "2", "page": str(number), "sort_order": sort_order}) + for number in range(1, 7) + ) + assert _sessions(seeded, pages) == expected + assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0] + assert [page.has_more for page in pages] == [True, True, True, True, False, False] + assert {page.total for page in pages} == {len(expected)} + + +def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"status_filter": "failure"}) + assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3")) + assert pages[0].total == 2 + + +def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"model": "m-b"}) + assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1")) + assert tuple(row.request_id.removeprefix(seeded.prefix) for row in _rows(pages)) == ("e1", "d1", "b1") + assert pages[0].total == 3 + + +_COPIED_ROWS: Final = 20_000 +_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC) +_COPY_WINDOW: Final = ( + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')" +) +_COPY_WINDOW_PARAMS: Final = (_COPY_START, _COPY_START + timedelta(days=1)) + + +def _generated_sessions(rows_per_session: int) -> tuple[sql.SQL, tuple[object, ...]]: + """`_COPIED_ROWS` rows a second apart in sessions of `rows_per_session` rows, a one-row session keyed by request_id.""" + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT 'r' || n, 'acompletion', 'key-1', moment, moment, " + "CASE WHEN %s > 1 THEN 's' || (n - 1) / %s END FROM generate_series(1, %s) n, " + "LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds" + ), (rows_per_session, rows_per_session, _COPIED_ROWS, _COPY_START.replace(tzinfo=None)) + + +def _sessions_tied_at_the_newest_moment() -> tuple[sql.SQL, tuple[object, ...]]: + """`_generated_sessions(1)` plus thirty one-row sessions sharing a newer moment, inserted and keyed against session + order so a read cut inside the tie misses the sessions that rank first.""" + start: Final = _COPY_START.replace(tzinfo=None) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT request_id, 'acompletion', 'key-1', moment, moment, session_id FROM (" + "SELECT n AS ordinal, 'r' || n AS request_id, %s::timestamp + n * INTERVAL '1 second' AS moment, " + "NULL AS session_id FROM generate_series(1, %s) n UNION ALL " + "SELECT %s + 30 - n, 't' || lpad((29 - n)::text, 2, '0'), %s::timestamp + %s * INTERVAL '1 second', " + "'s' || lpad(n::text, 2, '0') FROM generate_series(0, 29) n) seeded ORDER BY ordinal" + ), (start, _COPIED_ROWS, _COPIED_ROWS, start, _COPIED_ROWS + 1) + + +@dataclass(frozen=True, slots=True) +class _Call: + session: str + call_type: str + status: str | None + second: int + + +def _calls(calls: tuple[_Call, ...]) -> tuple[sql.SQL, tuple[object, ...]]: + """One row per call, `second` seconds into the window, ending a second later.""" + seeded: Final = tuple( + { + "request_id": f"{call.session}-{call.second}", + "call_type": call.call_type, + "session_id": call.session, + "status": call.status, + "start_time": (_COPY_START + timedelta(seconds=call.second)).replace(tzinfo=None).isoformat(), + } + for call in calls + ) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id, status) ' + "SELECT request_id, call_type, 'key-1', start_time, start_time + INTERVAL '1 second', session_id, status " + "FROM jsonb_to_recordset(%s::jsonb) AS calls(request_id text, call_type text, session_id text, status text, " + "start_time timestamp)" + ), (json.dumps(seeded),) + + +@asynccontextmanager +async def _spend_logs_copy(insert: sql.SQL, args: tuple[object, ...]) -> AsyncIterator[Prisma]: + """A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded by `insert`.""" + schema: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs") + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table)) + setup.execute(insert.format(table=table), args) + setup.execute(sql.SQL("ANALYZE {}").format(table)) + parsed: Final = urlsplit(url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}) + database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +class _PlanNode(BaseModel): + model_config = ConfigDict(frozen=True) + + relation_name: str | None = Field(default=None, alias="Relation Name") + actual_rows: float = Field(alias="Actual Rows") + rows_removed_by_filter: float = Field(default=0, alias="Rows Removed by Filter") + actual_loops: int = Field(alias="Actual Loops") + plans: tuple["_PlanNode", ...] = Field(default=(), alias="Plans") + + def spend_log_rows_scanned(self) -> int: + own: Final = ( + (self.actual_rows + self.rows_removed_by_filter) * self.actual_loops + if self.relation_name == "LiteLLM_SpendLogs" + else 0 + ) + return round(own) + sum(child.spend_log_rows_scanned() for child in self.plans) + + +class _Explained(BaseModel): + model_config = ConfigDict(frozen=True) + + plan: _PlanNode = Field(alias="Plan") + + +class _ExplainedReads: + """Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans.""" + + def __init__(self, database: Prisma) -> None: + self.database: Final = database + self.rows_scanned: tuple[int, ...] = () + + @asynccontextmanager + async def tx(self, *, timeout: timedelta) -> AsyncIterator["_ExplainedReads"]: + async with self.database.tx(timeout=timeout) as transaction: + reads: Final = _ExplainedReads(transaction) + try: + yield reads + finally: + self.rows_scanned = (*self.rows_scanned, *reads.rows_scanned) + + async def execute_raw(self, query: str, *args: object) -> int: + return await self.database.execute_raw(query, *args) + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args) + plan: Final = _Explained.model_validate(explained[0]["QUERY PLAN"][0]).plan + self.rows_scanned = (*self.rows_scanned, plan.spend_log_rows_scanned()) + return await self.database.query_raw(query, *args) + + +@pytest.mark.parametrize( + "rows_per_session, page, newest_session, rows_scanned_per_read, total", + [ + pytest.param(1, 1, ("r", 20_000), (102,), (100, True), id="first-page"), + pytest.param(1, 3, ("r", 19_980), (102,), (100, True), id="third-page"), + pytest.param(10, 1, ("s", 1_999), (102, 1_171), (100, True), id="second-read-sized-by-the-first-reads-density"), + pytest.param(100, 1, ("s", 199), (102, 6_439, 12_878), (100, True), id="third-read-at-least-doubled"), + pytest.param(1_000, 1, ("s", 19), (102, 12_878, 20_000), (20, False), id="last-read-runs-out-of-rows"), + ], +) +async def test_a_newest_first_page_scans_only_the_rows_its_reads_ask_for( + rows_per_session: int, + page: int, + newest_session: tuple[str, int], + rows_scanned_per_read: tuple[int, ...], + total: tuple[int, bool], +) -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session)) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=page, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + prefix, newest = newest_session + assert tuple(session.key for session in session_page.sessions) == tuple( + f"{prefix}{newest - n}" for n in range(10) + ) + assert (session_page.total, session_page.total_is_capped) == total + assert reads.rows_scanned == rows_scanned_per_read + + +async def test_a_page_cut_inside_a_tie_reads_on_until_the_tie_is_whole() -> None: + async with _spend_logs_copy(*_sessions_tied_at_the_newest_moment()) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=1, + page_size=5, + status=None, + sort_desc=True, + session_cursor=None, + cap=10, + ) + assert tuple(session.key for session in session_page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.rows_scanned == (12, 24, 48) + + +@dataclass(frozen=True, slots=True) +class _ExplainedPrisma: + read_db: _ExplainedReads + + +async def test_a_pages_representatives_are_found_by_session_not_by_scanning_the_window() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=10)) as database: + reads: Final = _ExplainedReads(database) + representatives: Final = await _fetch_session_representatives( + prisma_client=_ExplainedPrisma(reads), # pyright: ignore[reportArgumentType] # only .read_db is used + where_clause=_COPY_WINDOW, + sql_params=_COPY_WINDOW_PARAMS, + next_param_index=len(_COPY_WINDOW_PARAMS) + 1, + session_keys=(("s1999", "key-1"), ("s1998", "key-1")), + ) + assert tuple(row["request_id"] for row in representatives) == ("r20000", "r19990") + assert reads.rows_scanned == (20,), "only the two sessions' rows, not the window's 20,000" + + +async def test_a_page_number_far_past_the_cap_shows_no_session() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=1)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=9_300_000_000_000_000, + page_size=1000, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert (session_page.sessions, session_page.total, session_page.total_is_capped) == ((), 100, True) + + +_OUTCOME_CALLS: Final = ( + _Call("agent", "asend_message", "failure", 10), + _Call("agent", "acompletion", "success", 20), + _Call("mcp", "acompletion", "failure", 5), + _Call("mcp", "call_mcp_tool", "success", 30), + _Call("unknown", "acompletion", None, 40), + _Call("failed", "acompletion", "success", 1), + _Call("failed", "acompletion", "failure", 50), + _Call("recovered", "acompletion", "failure", 2), + _Call("recovered", "acompletion", "success", 45), +) + + +@pytest.mark.parametrize( + "status, expected", + [("failure", ("failed", "mcp", "agent")), ("success", ("recovered", "unknown"))], +) +async def test_a_status_filter_keeps_sessions_whose_representative_call_has_that_status( + status: str, expected: tuple[str, ...] +) -> None: + """An agent message stands for its session, then the newest call that is not an MCP tool call, and a missing status + counts as success. The page stays ordered by each session's newest call.""" + async with _spend_logs_copy(*_calls(_OUTCOME_CALLS)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + status=status, + page=1, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(session.key for session in session_page.sessions) == expected + assert (session_page.total, session_page.total_is_capped) == (len(expected), False) diff --git a/tests/integration/streaming/test_interception_converted_stream_options.py b/tests/integration/streaming/test_interception_converted_stream_options.py index 0ca62d2f014..6fc38dfb440 100644 --- a/tests/integration/streaming/test_interception_converted_stream_options.py +++ b/tests/integration/streaming/test_interception_converted_stream_options.py @@ -5,6 +5,7 @@ import uuid from collections.abc import Callable, Generator, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass +from hashlib import sha256 from pathlib import Path from queue import SimpleQueue from types import MappingProxyType @@ -216,10 +217,13 @@ def scenario(rig: _Rig) -> Iterator[Scenario]: yield opened -def _deploy(rig: _Rig, scenario: Scenario, mode: iv.Mode, *, api_key: str = iv.AZURE_KEY) -> _Deployment: +def _deploy( + rig: _Rig, scenario: Scenario, mode: iv.Mode, *, api_key: str = iv.AZURE_KEY, priced: bool = False +) -> _Deployment: name: Final = iv.deployment(mode) + pricing: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} if priced else {} model: Final = scenario.model( - model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION + model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION, **pricing ) return _Deployment(model, name) @@ -325,6 +329,46 @@ def _assert_response_landed_once(model: str, identity: str) -> None: assert row["status"] == "success" and rv.same_response(string_value(row["request_id"]), identity), row +def _assert_single_model_spend_for_key(key: str, call_type: str, *, expected_searches: int = 1) -> None: + digest: Final = sha256(key.encode()).hexdigest() + observed: Final = eventually( + lambda: ( + read_rows( + "SELECT status, spend, request_id, call_type, model_group, session_id " + 'FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND status = %s', + (digest, "success"), + ), + read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (digest,)), + ), + lambda values: ( + bool(values[0]) + and ( + len(values[0]) > expected_searches + 1 + or ( + len(values[0]) == expected_searches + 1 + and len(values[1]) == 1 + and values[1][0]["spend"] is not None + and float(str(values[1][0]["spend"])) + == pytest.approx(sum(float(str(row["spend"])) for row in values[0])) + ) + ) + ), + seconds=70, + ) + rows, balance = observed + model_rows: Final = [row for row in rows if row["call_type"] == call_type] + search_rows: Final = [row for row in rows if row["call_type"] == "asearch"] + spend: Final = sum(float(str(row["spend"])) for row in rows) + assert ( + len(rows) == expected_searches + 1 + and len(model_rows) == 1 + and len(search_rows) == expected_searches + and float(str(model_rows[0]["spend"])) > 0 + and all(float(str(row["spend"])) > 0 for row in search_rows) + and float(str(balance[0]["spend"])) == pytest.approx(spend) + ), (rows, balance) + + def _openai(rig: _Rig) -> openai.OpenAI: return openai.OpenAI(base_url=f"{rig.base_url}/v1", api_key=rig.gateway.key, max_retries=0) @@ -402,6 +446,31 @@ def test_raw_chat_stream_without_stream_options_answers_without_a_usage_chunk(ri _assert_search_turn(rig, deployment, identity) +def test_chat_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenario: Scenario) -> None: + deployment: Final = _deploy(rig, scenario, "search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key) + identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None) + _assert_search_turn(rig, deployment, identity) + _assert_single_model_spend_for_key(key, "acompletion") + + +def test_multi_round_chat_stream_through_web_search_bills_each_search( + rig: _Rig, scenario: Scenario +) -> None: + deployment: Final = _deploy(rig, scenario, "multi-search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key) + identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None) + received: Final = _upstream(rig, deployment) + _assert_upstream(received, model_calls=3, searches=2) + assert [search["query"] for search in received.searches] == [ + iv.query(deployment.name), + iv.query(deployment.name, turn=2), + ], received.searches + _assert_single_model_spend_for_key(key, "acompletion", expected_searches=2) + + def test_openai_sdk_responses_stream_through_web_search_keeps_stream_options_off_azure( rig: _Rig, scenario: Scenario ) -> None: @@ -454,6 +523,31 @@ def test_raw_responses_stream_without_stream_options_answers(rig: _Rig, scenario _assert_response_landed_once(deployment.model, identity) +def test_responses_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenario: Scenario) -> None: + deployment: Final = _deploy(rig, scenario, "search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key) + _assert_responses_restreamed(stream, iv.answer(deployment.name)) + _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) + _assert_single_model_spend_for_key(key, "aresponses") + + +def test_multi_round_responses_stream_through_web_search_bills_each_search( + rig: _Rig, scenario: Scenario +) -> None: + deployment: Final = _deploy(rig, scenario, "multi-search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key) + _assert_responses_restreamed(stream, iv.answer(deployment.name)) + received: Final = _upstream(rig, deployment) + _assert_upstream(received, model_calls=3, searches=2) + assert [search["query"] for search in received.searches] == [ + iv.query(deployment.name), + iv.query(deployment.name, turn=2), + ], received.searches + _assert_single_model_spend_for_key(key, "aresponses", expected_searches=2) + + def test_openai_sdk_chat_without_streaming_through_web_search_answers_in_json(rig: _Rig, scenario: Scenario) -> None: deployment: Final = _deploy(rig, scenario, "search") completion: Final = _openai(rig).chat.completions.create( @@ -754,60 +848,36 @@ def test_wrong_azure_key_surfaces_azures_401(rig: _Rig, scenario: Scenario) -> N assert len(_upstream(rig, deployment).model_calls) == 1 -def _tool_call(call: Mapping[str, JsonValue]) -> tuple[str, Mapping[str, JsonValue]]: - function: Final = object_value(call["function"]) - return string_value(function["name"]), iv.JSON_OBJECT.validate_json(string_value(function["arguments"])) +def _assert_failed_once(call_id: str, *, spend: float) -> None: + rows: Final = eventually( + lambda: read_rows('SELECT status, spend FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)), + lambda found: len(found) >= 1, + seconds=70, + ) + assert [row["status"] for row in rows] == ["failure"], rows + assert float(rows[0]["spend"]) == pytest.approx(spend), rows -def _stream_choices(stream: _Stream) -> Iterator[Mapping[str, JsonValue]]: - for frame in stream.frames: - yield from rv.ITEMS.validate_python(frame.get("choices") or []) - - -def _streamed_tool_calls(stream: _Stream) -> Iterator[tuple[str, Mapping[str, JsonValue]]]: - for choice in _stream_choices(stream): - yield from map( - _tool_call, rv.ITEMS.validate_python(object_value(choice.get("delta") or {}).get("tool_calls") or []) - ) - - -def _handed_back(deployment: _Deployment) -> list[tuple[str, Mapping[str, JsonValue]]]: - return [(iv.SEARCH_TOOL, {"query": iv.query(deployment.name)})] - - -def _assert_succeeded_once(model: str, identity: str) -> None: - rows: Final = _spend_rows(model, 1) - assert [string_value(row["request_id"]) for row in rows if row["status"] == "success"] == [identity], rows - - -def test_failed_follow_up_call_hands_the_search_tool_call_back_with_its_usage(rig: _Rig, scenario: Scenario) -> None: - deployment: Final = _deploy(rig, scenario, "followup-fails") - stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)})) - assert stream.status == 200 and stream.content == "", stream.text - assert list(_streamed_tool_calls(stream)) == _handed_back(deployment), stream.text - finished: Final = [choice["finish_reason"] for choice in _stream_choices(stream) if choice.get("finish_reason")] - assert finished == ["tool_calls"], stream.text - assert [_totals(usage) for usage in stream.usages] == [iv.FIRST_USAGE["total_tokens"]], stream.text - (identity,) = stream.ids +@pytest.mark.parametrize("stream", [True, False], ids=["stream", "json"]) +def test_failed_follow_up_call_answers_with_the_providers_error_instead_of_the_search_tool_call( + rig: _Rig, scenario: Scenario, stream: bool +) -> None: + deployment: Final = _deploy( + rig, scenario, "followup-fails", priced=True + ) + extra: Final = {**_NO_CACHE, "stream_options": dict(_USAGE)} if stream else _NO_CACHE + body: Final = _chat_body(deployment, stream=stream, extra=extra) + response: Final = rig.gateway.request("POST", "/v1/chat/completions", body) + assert response.status_code == 500 and iv.FOLLOWUP_FAILURE in response.text, response.text + assert iv.SEARCH_TOOL not in response.text, response.text _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) - _assert_succeeded_once(deployment.model, identity) + _assert_failed_once( + response.headers["x-litellm-call-id"], + spend=iv.FIRST_USAGE["prompt_tokens"] * 0.001 + iv.FIRST_USAGE["completion_tokens"] * 0.002, + ) assert rig.gateway.chat(_deploy(rig, scenario, "plain").model)["object"] == "chat.completion" -def test_failed_follow_up_call_on_a_json_request_hands_the_search_tool_call_back(rig: _Rig, scenario: Scenario) -> None: - deployment: Final = _deploy(rig, scenario, "followup-fails") - response: Final = rig.gateway.request("POST", "/v1/chat/completions", _chat_body(deployment, stream=False)) - assert response.status_code == 200, response.text - body: Final = iv.JSON_OBJECT.validate_json(response.content) - (choice,) = rv.ITEMS.validate_python(body["choices"]) - calls: Final = rv.ITEMS.validate_python(object_value(choice["message"])["tool_calls"]) - assert [_tool_call(call) for call in calls] == _handed_back(deployment), body - assert choice["finish_reason"] == "tool_calls", body - assert _totals(object_value(body["usage"])) == iv.FIRST_USAGE["total_tokens"], body - _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) - _assert_succeeded_once(deployment.model, string_value(body["id"])) - - def test_search_outage_still_restreams_the_follow_up_answer(rig: _Rig, scenario: Scenario) -> None: deployment: Final = _deploy(rig, scenario, "search-down") stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)})) diff --git a/tests/integration/translation/chat_completions/bases/bedrock_converse.py b/tests/integration/translation/chat_completions/bases/bedrock_converse.py index 6540bd09d64..777e0d04db5 100644 --- a/tests/integration/translation/chat_completions/bases/bedrock_converse.py +++ b/tests/integration/translation/chat_completions/bases/bedrock_converse.py @@ -1407,6 +1407,7 @@ CLAUDE_SONNET_5_TEST_CASE: Final = TranslationTestCase( expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5/converse", expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, expected_provider_request={ + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], "messages": [{"role": "user", "content": [{"text": "Say hello."}]}], "inferenceConfig": {"maxTokens": 64}, "system": [{"text": "You are a terse assistant."}], diff --git a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py index 07e4358ff97..5cb7a7eee8e 100644 --- a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py +++ b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py @@ -6,7 +6,7 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.chat_completions.bases.anthropic import CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.chat_completions.bases.anthropic import CLAUDE_OPUS_4_8_TEST_CASE from integration.translation.runner import assert_translation PLACE_SCHEMA: Final = { @@ -24,25 +24,25 @@ PLACE_TOOL_USE: Final = { "caller": {"type": "direct"}, } -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_TEST_CASE, scenario="json_schema", litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, }, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, "tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}], }, mock_provider_response={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP2UR3vcM5ZWmEBd8D", "type": "message", "role": "assistant", @@ -68,7 +68,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( "id": ANY, "created": ANY, "object": "chat.completion", - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "finish_reason": "stop", @@ -97,11 +97,11 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( }, ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( - CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, +CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( + CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, scenario="json_schema_tool_named_envelope", mock_provider_response={ - **CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE.mock_provider_response, + **CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE.mock_provider_response, "content": [ { **PLACE_TOOL_USE, @@ -114,7 +114,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( @pytest.mark.parametrize( "case", - [CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], + [CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_response_format_anthropic( diff --git a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py index fca8de69f00..a41752a96cf 100644 --- a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py +++ b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py @@ -6,7 +6,7 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import SSEFrame, StreamingTranslationTestCase from integration.translation.chat_completions.bases.anthropic import ( - CLAUDE_HAIKU_5_5_TEST_CASE, + CLAUDE_OPUS_4_8_TEST_CASE, CLAUDE_SONNET_5_5_TEST_CASE, ) from integration.translation.runner import assert_streaming_translation @@ -158,27 +158,27 @@ def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame: "id": ANY, "object": "chat.completion.chunk", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [{"index": 0, "delta": delta}], } ) -CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( +CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( scenario="json_schema_cut_stream", - litellm_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.litellm_endpoint, + litellm_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.litellm_endpoint, litellm_request={ - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "max_tokens": 128, "messages": [{"role": "user", "content": PLACE_PROMPT}], "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, "cache": {"no-cache": True}, "stream": True, }, - expected_provider_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_endpoint, - expected_provider_headers=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_headers, + expected_provider_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_endpoint, + expected_provider_headers=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_headers, expected_provider_request={ - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], "max_tokens": 128, "tool_choice": {"name": "json_tool_call", "type": "tool"}, @@ -191,7 +191,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT data={ "type": "message_start", "message": { - "model": "claude-haiku-5-5", + "model": "claude-opus-4-8", "id": "msg_011CgCoP5RkexwE6N7PRZEca", "type": "message", "role": "assistant", @@ -263,7 +263,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT "id": ANY, "object": "chat.completion", "created": ANY, - "model": "anthropic/claude-haiku-5-5", + "model": "anthropic/claude-opus-4-8", "choices": [ { "index": 0, @@ -276,7 +276,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT @pytest.mark.parametrize( "case", - [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE], + [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE], ids=lambda case: case.id, ) def test_chat_completions_streaming_anthropic( diff --git a/tests/integration/translation/decisions/bases/parallel_ai.py b/tests/integration/translation/decisions/bases/parallel_ai.py new file mode 100644 index 00000000000..14e9cd6fc43 --- /dev/null +++ b/tests/integration/translation/decisions/bases/parallel_ai.py @@ -0,0 +1,257 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase +from pydantic import JsonValue + +"""Provider request and reply shape from https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions. Mock reply captured live on 2026-10-31. +""" +_CITATIONS: Final[list[JsonValue]] = [ + { + "url": "https://access.redhat.com/solutions/7108129", + "title": 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg. - Red Hat Customer Portal', + "excerpts": [ + 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg.\n\nSolution Verified - Updated 2026-06-18T01:07:40+00:00 -\nEnglish\n* No translations currently exist.\n\nIssue\n\n* NFS client hangs while returning delegations.\n* Below delegation recall errors may be reported in dmesg:\nNFS: nfs4_handle_delegation_recall_error: unhandled error -512\nNFS: nfs4_handle_delegation_recall_error: unhandled error -10024\n* We have a hanging process, it\'s using data ontop of a mounted NFS export. The fcntl() system call issued by a user process has not returned, The user process processing does not finish - usually it finishes within 2 minutes.\n\nEnvironment\n\n* Red Hat Enterprise Linux (RHEL) 8, all minor releases\n* kernel:\n* rhel8.4: kernels before 4.18.0-305.194.1.el8_4 affected\n* rhel8.10: kernels before 4.18.0-553.37.1.el8_10 affected\n* NFSv4' + ], + }, + { + "url": "https://github.com/moby/buildkit/issues/2804", + "title": "GitHub cache exporter (`type=gha`) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. · Issue #2804 · moby/buildkit · GitHub", + "excerpts": [ + 'Copy link\nOpen\nOpen\nGitHub cache exporter ( type=gha ) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. #2804\nCopy link\n\nDescription\n\n@bendavies\nbendavies\nopened on Apr 13, 2022\nIssue body actions\n\nBehaviour\n\nCache operations using type=gha will start hanging after a relavatively low number of cache reads/writes.\nWe are a small team (5 devs) and are hitting these limits constantly on our main repository. Not exactly high throughput.\n\n...\n\nCache reads and writes go towards saturating the limit, and both are affected by this throttling.\n@dhadka mentions there is a limit of about "150 cache requests in 5 minutes", and seems about right: #2276 (comment)\n\nExpected behaviour\n\nI don\'t know, but the current behaviour of hanging is not great. there is no feedback as to what is hanging resulting in the user just being confused.' + ], + }, + { + "url": "https://github.com/huginn/huginn/issues/2963", + "title": "WebsiteAgent hangs on a specific url, never finishes job or times out · Issue #2963 · huginn/huginn · GitHub", + "excerpts": [ + 'WebsiteAgent hangs on a specific url, never finishes job or times out\n\n* Page: GitHub issue\n* URL: https://github.com/huginn/huginn/issues/2963\n* State: open\n* Author: BoKKeR\n* Created: 2021-03-14T16:42:01Z\n* Updated: 2021-03-16T19:09:24Z\n* Repository: huginn/huginn\n* Number: #2963\n* Comments: 1\nI know it might be controversial but I am trying to get a new GPU for a while. Wanted to setup a monitor for when the stock on amd.com changes. But the site just won\'t load. The WebsiteAgent gets stuck in\nFetching https://www.amd.com/de/direct-buy/se\nThe only way to stop it is by stopping the container. As I am running the service in a container I cant find any of the debug log locations. Can anyone else try if they are encountering the same error?\n{\n "debug": "true",\n "expected_update_period_in_days": "2",\n "url": "https://www.amd.com/de/direct-buy/se",\n "type": "html",' + ], + }, + { + "url": "https://github.com/mozilla-firefox/firefox-crb/blob/main/docs/bug-mgmt/guides/severity.md", + "title": "firefox-crb/docs/bug-mgmt/guides/severity.md at main · mozilla-firefox/firefox-crb · GitHub", + "excerpts": [ + "We use the severity field in Bugzilla to indicate the scope of a bug's effect on Firefox.\nThe field is display alongside the bug's priority.\n\nValues\n\nSeverity levels and their definitions are enumerated at https://wiki.mozilla.org/BMO/UserGuide/BugFields#bug_severity.\nBy default, new bugs have a severity of --.\n\nExamples of S1 bugs\n\n* WebExtensions disabled for all users\n* Web search not working from URL bar\n* Crashes with data loss\n\nExamples of S2 bugs\n\nBugs that could reasonably be expected to cause a Firefox user to switch browsers, either because the severity is bad enough, or the frequency of occurrence is high enough.\n* Bug 1640441 - Slack hangs indefinitely in a onResize loop" + ], + }, + { + "url": "https://github.com/n0-computer/iroh/issues/4604", + "title": "Endpoint::close hangs forever if the tokio runtime gets dropped while the connection is open · Issue #4604 · n0-computer/iroh · GitHub", + "excerpts": [ + "@alpergundogdu\nalpergundogdu\nopened on Oct 14, 2026\nIssue body actions\nDescribe the bug\nAfter opening a connection from inside a short-lived tokio runtime, and dropping the runtime while the connection is still open, Endpoint::close() never returns.\nIn my repro it hangs reliably; when I keep the runtime open, it closes instantly.\nLocal repro/test (has the control tests too): https://github.com/decdn/iroh-tests/blob/main/tests/bug_endpoint_close_after_runtime_drop.rs\n\n...\n\nIf spawning on the caller's runtime is intended, a note on Endpoint::connect saying the driver lives there would have saved us a lot of time. (Maybe it's an expectation/documentation issue?)\nIroh\nVersion: iroh 1.3.0, noq 1.3.0\nEndpoint configuration: bound to 127.0.0.1, no relays.\nPlatform(s)\nDesktop:\n\n...\n\nSorry @flub , that was unclear: the endpoint's runtime stays alive, and close() is called from it. Only a separate short-lived runtime that runs connect() gets dropped." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1640441", + "title": "1640441 - Slack hangs indefinitely in a onResize loop", + "excerpts": [ + "Closed Bug 1640441 Opened 6 years ago Closed 6 years ago\n\nSlack hangs indefinitely in a on Resize loop\n\nCategories\n\n(Web Compatibility :: Site Reports, defect, P1)\n\nProduct:\nWeb Compatibility ▾\nComponent:\nSite Reports ▾\nType:\ndefect\nPriority:\nP1 Severity:\nS2\n\nTracking\n\n(Not tracked)\n\nStatus:\nRESOLVED FIXED\n\nPeople\n\n(Reporter: Harald, Assigned: emilio)\n\nReferences\n\n(Depends on 1 open bug)\n\nDetails\n\n(Keywords: enterprise, perf, webcompat:site-wait)\n\nAttachments\n\n(1 file, 1 obsolete file)\n\n|relevant unminified code.\n6 years ago Emilio Cobos Álvarez [:emilio]\n* 13.50 KB,\n\n...\n\n* Hover over the link\n* ... tab immediately becomes unresponsive ...\nThis may well be a bug in slack, but IMO it's still on us to try and fix what's going on. When it happens, it affects not only Slack but my entire Firefox session. This is a serious usability issue -- I'm now at the point of restarting Firefox several times a day because of this bug, if" + ], + }, + { + "url": "https://www.ibm.com/support/pages/cognos-business-intelligence-1022-fix-lists", + "title": "Cognos Business Intelligence 10.2.2 Fix Lists", + "excerpts": [ + 'IBM Support \n\nCognos Business Intelligence 10.2.2 Fix Lists\n\nRelease Notes\n\nAbstract\n\nA comprehensive list of defect corrections for major releases, refresh packs and fix packs of Cognos Business Intelligence 10.2.2.\nDetails of the APARs listed below can be accessed by clicking the link for the APAR number. If you have questions about a particular defect, please contact Customer Support.\n\n...\n\n* PI05143 | MULTIPLE VALUES IN CONTEXT FILTERS CAUSE A CROSSTAB REPORT TO SHOW DASH "-" INSTEAD OF CORRECT VALUES\n* PI05166 | QE-DEF-0359 The query contains a reference to at least one object that does not exist\n* PI05233 | Export deployments from 8.4.1 with export folder or package tasks in Content Administration contain unknown content steps\n\n...\n\n* PI08523 | Progressive Chart - Incorrect Colours when "Reverse Catagory Order" = True\n* PI08585 | Cognos hangs when starting against a Content Store from previous version (upgrade) and using SSL gateway UR' + ], + }, + { + "url": "https://josm.eu/ticket/8175", + "title": "\n #8175 ([PATCH] Cadastre grab query sometimes hangs completely)\n – JOSM\n\n ", + "excerpts": [ + 'Opened 14 years ago\nClosed 14 years ago\nLast modified 14 years ago\n\n#8175 closed defect ( fixed )\n\n[PATCH] Cadastre grab query sometimes hangs completely\n\n* Reported by: | Don-vip | Owned by: | pieren\n* Priority: | normal | Milestone: | \n* Component: | Plugin cadastre-fr | Version: | \n* Keywords: | | Cc: | \n\nDescription\n\nDuring cadastre grab, the HTTP connection with the WMS server sometimes completely hangs (the progress dialog is frozen and cancel button does not work).\nThe only solution is to restart JOSM to have a clean state and grab the cadastre again. We had a similar problem with history request ( #7640 ) when dealing with the API and solved it in [r5324](https://josm.eu/changeset/5324/josm "fix #7640 - History query will not cancel or timeout\n\n...\n\nby Don-vip , 14 years ago\n\n* Attachment: | 8175.patch \u200b added\n\ncomment:1 by Don-vip , 14 years ago' + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1749908", + "title": "1749908 - Infinite loop in HTTP3 hangs socket thread - Bugzilla", + "excerpts": [ + "Closed Bug 1749908 Opened 4 years ago Closed 4 years ago\n\nInfinite loop in HTTP3 hangs socket thread\n\nCategories\n\n(Core :: Networking, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking ▾\nPlatform:\nDesktop\nAll\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nVERIFIED DUPLICATE of bug 1749910\n\nPeople\n\n(Reporter: heftig, Unassigned)\n\nDetails\n\nBottom ↓\nTags ▾\nTimeline ▾\n| |Jan Alexander Steffens [:heftig]\n* Reporter | \n* ### Description\n•\n* 4 years ago\n* \nWhile browsing, pages suddenly stopped loading.\n\n...\n\nIt's not related to a specific version, we're getting reports ESR is even affected. Suspicion is around some long existing HTTP3 bug that's being triggered by an external service updating.\n\n...\n\nI agree it is not the root cause but it certainly caused a massive problem for many users of Firefox, the majority of whom are opted-in by default. If they had not been, they would not have been affected." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1748414", + "title": "1748414 - Firefox 94 sometimes hangs on TLSv1.3 requests", + "excerpts": [ + "Closed Bug 1748414 Opened 4 years ago Closed 4 years ago\n\nFirefox 94 sometimes hangs on TLSv1 .3 requests\n\nCategories\n\n(Core :: Networking: HTTP, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking: HTTP ▾\nVersion:\nFirefox 94\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nRESOLVED DUPLICATE of bug 1740856\n\nPeople\n\n(Reporter: code, Unassigned)\n\nDetails\n\nBottom ↓\n\n...\n\nSteps to reproduce:\nIn the beginning of November, colleagues started to experience hangs of 1 to 1.5 minutes when working with our own web app. We use latest official releases for Windows, Ubuntu and FreeBSD. The problems exist for all plattforms and very likely started with Firefox 94.\nActual results:\nSometimes while interacting normally with our web app, a request hangs for a long time (up to 1.5 minutes; but usually succeeds). This typically happens on POST requests." + ], + }, +] +DECISION_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "parallel_ai/decision", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.6442559258537284}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.1616964510265985}, + {"value": "high", "probability": 0.8383035489734014}, + ], + "confidence": 0.6766070979468028, + }, + { + "type": "score", + "name": "confidence", + "score": 0.25195637565890455, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.7480436243410955}, + {"value": 1, "label": "sure", "probability": 0.25195637565890455}, + ], + "confidence": 0.4960872486821909, + }, + ], + "usage": { + "input_tokens": 3166, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 3166, + }, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) +DECISION_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "parallel_ai/decision", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py new file mode 100644 index 00000000000..47eb057323b --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.parallel_ai import DECISION_SYSTEMONE_TEST_CASE, DECISION_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [DECISION_TEST_CASE, DECISION_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_parallel_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/rust-python-harness/strategies/e2e_parity/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/__init__.py index 298c75f2af1..a2ea98f19ab 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/__init__.py +++ b/tests/rust-python-harness/strategies/e2e_parity/__init__.py @@ -20,7 +20,7 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( CaseDefinition( "messages", NotImplementedCaseSpec( - reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered." + reason="Parity runs as the pytest suite in sdk/messages/test_native_parity.py, not as a runner case." ), surface="sdk", ), diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py new file mode 100644 index 00000000000..4bb20713c59 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py @@ -0,0 +1,34 @@ +from collections.abc import AsyncIterator, Generator +from importlib import import_module +from pathlib import Path +from typing import Final + +import pytest +import pytest_asyncio + +from tests.test_litellm_rust.support.isolation import isolated_test_state +from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service + +PARITY_ROOT: Final = Path(__file__).parent + + +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def isolate_test_state() -> AsyncIterator[None]: + async with isolated_test_state(): + yield + + +@pytest.fixture +def recording_server() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + try: + import_module("litellm.rust_bridge._native") + except ImportError: + skip: Final = pytest.mark.skip(reason="requires a compiled litellm.rust_bridge._native extension") + for item in items: + if PARITY_ROOT in item.path.parents: + item.add_marker(skip) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py new file mode 100644 index 00000000000..41008f0d57f --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -0,0 +1,1079 @@ +"""Native Anthropic Messages calls must be indistinguishable from the Python implementation. + +Each case sends one `/v1/messages` SDK call through the Python handler and through the native route against the same +scripted upstream, then compares one aspect of what happened: + +- dispatch: the native leg was served natively instead of falling back to Python +- request: every request that reached the upstream (method, path, body, headers) +- outcome: what the caller got back (response body, stream bytes, or the raised exception) +- hidden_params: the `_hidden_params` keys the proxy builds its response headers from +- logged: which terminal callbacks fired and the stable standard logging payload fields +- logged_metadata, logged_model_info, logged_response_headers, logged_api_key, logged_call: single callback kwargs + fields that proxy integrations read + +Both legs run once per case and every aspect of that case reads the same pair + +The native leg prepends an unconstrained Rust rule, so feature detectors that keep a call on Python today do not +apply: a case for a detector-covered feature shows what Rust still lacks before that detector can go. Every known +difference is a strict xfail naming the gap, with an id from the Rust vs Python Messages inventory (A routing, +R request, S response, C callbacks, F covered by a fallback detector today) +""" + +import asyncio +import copy +import json +import os +import re +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field, replace +from types import MappingProxyType +from typing import Final, cast + +import pytest +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.messages import dispatch +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging +from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + +SONNET: Final = "anthropic/claude-sonnet-4-5" +OPUS_ADAPTIVE: Final = "anthropic/claude-opus-4-7" +OPUS_FAST: Final = "anthropic/claude-opus-5-5" +API_KEY: Final = "sk-ant-parity-test" +USER: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": "What is the capital of France?"}) + +_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_LOGGED: Final = TypeAdapter(dict[str, object]) +_TRANSPORT_HEADERS: Final = frozenset( + {"accept", "accept-encoding", "connection", "content-length", "host", "user-agent"} +) +_COMPARED_LOG_FIELDS: Final = ( + "call_type", + "custom_llm_provider", + "model", + "model_group", + "model_id", + "api_base", + "stream", + "cache_hit", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "response_cost", + "status", +) +_TERMINAL_EVENTS: Final = ( + "log_success_event", + "async_log_success_event", + "log_failure_event", + "async_log_failure_event", +) +_VOLATILE: Final = ( + (re.compile(r"time taken=[0-9.]+"), "time taken="), + (re.compile(r"Timeout passed=[0-9.]+"), "Timeout passed="), + (re.compile(r"127\.0\.0\.1:[0-9]+"), "127.0.0.1:"), +) + +TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "id": "msg_01ParityText", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": "Paris"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 14, "output_tokens": 4}, + } +) + +TOOL_USE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityTool", + "content": [ + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}}, + ], + "stop_reason": "tool_use", + } +) + +THINKING_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityThinking", + "content": [ + {"type": "thinking", "thinking": "The capital of France is Paris.", "signature": "EqQBCkYIARgCKkB"}, + {"type": "text", "text": "Paris"}, + ], + } +) + +CACHED_USAGE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "usage": { + "input_tokens": 14, + "output_tokens": 4, + "cache_creation_input_tokens": 2048, + "cache_read_input_tokens": 1024, + "service_tier": "standard", + }, + } +) + +WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "name": "get_weather", + "description": "Current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +) + +TOOL_TURNS: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01Parity", "content": "18C, sunny"}]}, +) + +THINKING_REPLAY: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "What is the capital of France?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Paris.", "signature": "EqQBCkYIARgCKkB"}, + {"type": "text", "text": "Paris"}, + ], + }, + {"role": "user", "content": "And Germany?"}, +) + +NO_INPUT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +PNG_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + +def _events(*events: tuple[str, JsonValue]) -> tuple[tuple[str, object], ...]: + return tuple((name, data) for name, data in events) + + +def _message_start(message_id: str) -> tuple[str, JsonValue]: + return ( + "message_start", + { + "type": "message_start", + "message": { + **TEXT_RESPONSE, + "id": message_id, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 14, "output_tokens": 1}, + }, + }, + ) + + +def _text_block(index: int, *pieces: str) -> tuple[tuple[str, JsonValue], ...]: + return ( + ( + "content_block_start", + {"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}}, + ), + *( + ( + "content_block_delta", + {"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": piece}}, + ) + for piece in pieces + ), + ("content_block_stop", {"type": "content_block_stop", "index": index}), + ) + + +def _message_end(stop_reason: str, output_tokens: int) -> tuple[tuple[str, JsonValue], ...]: + return ( + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": {"output_tokens": output_tokens}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ) + + +TEXT_STREAM: Final = _events( + _message_start("msg_01ParityStream"), *_text_block(0, "Pa", "ris"), *_message_end("end_turn", 4) +) + +TOOL_STREAM: Final = _events( + _message_start("msg_01ParityToolStream"), + *_text_block(0, "Checking."), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {}}, + }, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": '}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '"Paris"}'}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + *_message_end("tool_use", 22), +) + +THINKING_STREAM: Final = _events( + _message_start("msg_01ParityThinkingStream"), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "Paris."}}, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "EqQBCkYIARgCKkB"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + *_text_block(1, "Paris"), + *_message_end("end_turn", 12), +) + +ERROR_MID_STREAM: Final = _events( + _message_start("msg_01ParityOverloaded"), + *_text_block(0, "Pa"), + ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), +) + +SLOW_STREAM: Final = _events( + _message_start("msg_01ParitySlow"), + *_text_block(0, "a", "b", "c", "d", "e", "f"), + *_message_end("end_turn", 6), +) + + +def _upstream_error(kind: str, message: str) -> Mapping[str, JsonValue]: + return MappingProxyType({"type": "error", "error": {"type": kind, "message": message}}) + + +def _ok(body: Mapping[str, JsonValue] = TEXT_RESPONSE, **headers: str) -> ResponseSpec: + return ResponseSpec(body=dict(body), headers=dict(headers)) + + +def _stream(events: tuple[tuple[str, object], ...], chunk_delay: float = 0) -> ResponseSpec: + return ResponseSpec(body={}, events=events, chunk_delay=chunk_delay) + + +def _failure(status: int, kind: str, message: str, **headers: str) -> ResponseSpec: + return ResponseSpec(body=dict(_upstream_error(kind, message)), status=status, headers=dict(headers)) + + +CALLBACK_ASPECTS: Final = ( + "logged_metadata", + "logged_model_info", + "logged_response_headers", + "logged_api_key", + "logged_call", +) +ASPECTS: Final = ("dispatch", "request", "outcome", "hidden_params", "logged", *CALLBACK_ASPECTS) + + +@dataclass(frozen=True, slots=True) +class Case: + name: str + kwargs: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + upstream: tuple[ResponseSpec, ...] = (ResponseSpec(body=dict(TEXT_RESPONSE)),) + model: str = SONNET + sync: bool = False + env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + private_keys: bool = False + gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({})) + + def params(self) -> Iterator[object]: + for aspect in ASPECTS: + gap = {**KNOWN_GAPS.get(self.name, {}), **self.gaps}.get(aspect) + marks = () if not gap else (pytest.mark.xfail(reason=gap, strict=True),) + yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) + + +def _gaps(**gaps: str | None) -> Mapping[str, str | None]: + return MappingProxyType(gaps) + + +def _kw(**kwargs: object) -> Mapping[str, object]: + return MappingProxyType(kwargs) + + +A10_NO_MAX_TOKENS: Final = "A10: native admission hands a call without an int max_tokens to Python, whose signature raises the TypeError; only dispatch differs" +S6_PRIVATE_KEYS: Final = "S6: native writes _hidden_params into the response dict" +S15_COMPLETE_DEADLINE: Final = ( + "S15: by design, native bounds a complete response by one overall deadline where Python bounds each read, " + "so a body trickling in slower than the timeout succeeds in Python and times out natively" +) +F_MOCK: Final = "F: mock_response is Python-only (detector mock_response)" +F_THINKING_RETRY: Final = ( + "F: retry without thinking after a rejected signature is Python-only (detector thinking_signature_retry)" +) +F_CACHE_INJECTION: Final = "F: prompt cache_control injection is Python-only (detector cache_control_injection)" +F_TOOL_CUSTOM: Final = "F: hoisting a tool's custom field is Python-only (detector tool_custom_field)" +F_STREAM_COST: Final = "F: cost injection into streamed usage is Python-only (detector stream_cost_injection)" + +KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( + { + "response_private_keys": _gaps(outcome=S6_PRIVATE_KEYS), + "missing_max_tokens": _gaps(dispatch=A10_NO_MAX_TOKENS), + "mock_response": _gaps( + logged_response_headers=F_MOCK, + request=F_MOCK, + outcome=F_MOCK, + hidden_params=F_MOCK, + logged=F_MOCK, + logged_metadata=F_MOCK, + logged_api_key=F_MOCK, + logged_call=F_MOCK, + ), + "mock_response_rate_limit": _gaps( + logged_response_headers=F_MOCK, + request=F_MOCK, + outcome=F_MOCK, + logged=F_MOCK, + logged_api_key=F_MOCK, + logged_call=F_MOCK, + ), + "thinking_signature_rejected_then_retried": _gaps( + request=F_THINKING_RETRY, + outcome=F_THINKING_RETRY, + hidden_params=F_THINKING_RETRY, + logged=F_THINKING_RETRY, + logged_metadata=F_THINKING_RETRY, + logged_response_headers=F_THINKING_RETRY, + ), + "cache_control_injection_points": _gaps(request=F_CACHE_INJECTION), + "tool_custom_field": _gaps(request=F_TOOL_CUSTOM), + "stream_cost_injection": _gaps(outcome=F_STREAM_COST), + "enable_prompt_caching": _gaps(request=F_CACHE_INJECTION), + } +) + + +REQUEST_CASES: Final = ( + Case("plain_text"), + Case("system_string", _kw(system="Answer in one word.")), + Case( + "system_blocks_with_cache_control", + _kw(system=[{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}]), + ), + Case("multi_turn_tool_result", _kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)])), + Case( + "tools_with_tool_choice", + _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "tool", "name": "get_weather"}), + upstream=(_ok(TOOL_USE_RESPONSE),), + ), + Case("tool_choice_none", _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "none"})), + Case("tool_input_examples", _kw(tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}])), + Case("sampling_params", _kw(temperature=0.3, top_p=0.9, top_k=40)), + Case("stop_sequences", _kw(stop_sequences=["\n\nHuman:"])), + Case("metadata_user_id", _kw(metadata={"user_id": "user-123"})), + Case("metadata_extra_keys_are_filtered", _kw(metadata={"user_id": "user-123", "session": "s-1"})), + Case("metadata_user_id_not_a_string", _kw(metadata={"user_id": 123})), + Case("service_tier", _kw(service_tier="auto")), + Case("thinking_enabled", _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096)), + Case("thinking_disabled", _kw(thinking={"type": "disabled"})), + Case( + "legacy_thinking_on_adaptive_model_keeps_display", + _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_display_updates_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "updates"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_string_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": " 4_096 "}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_non_numeric_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": "lots"}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_null_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": None}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_non_numeric_budget_dropped", + _kw(thinking={"type": "enabled", "budget_tokens": [1]}, max_tokens=8192, drop_params=True), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_carries_extra_keys", + _kw( + thinking={"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "kept"}, + max_tokens=4096, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_drops_extra_keys_with_drop_params", + _kw( + thinking={"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "dropped"}, + max_tokens=4096, + drop_params=True, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_float_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": 8192.0}, max_tokens=16000), + model=OPUS_ADAPTIVE, + ), + Case( + "adaptive_thinking_with_block_binding", + _kw(thinking={"type": "adaptive", "block_binding": "strict"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + ), + Case("reasoning_effort", _kw(reasoning_effort="high", max_tokens=8192)), + Case("reasoning_effort_on_adaptive_model", _kw(reasoning_effort="low", max_tokens=8192), model=OPUS_ADAPTIVE), + Case("output_config_effort", _kw(output_config={"effort": "medium"}), model=OPUS_ADAPTIVE), + Case( + "output_format_json_schema", + _kw( + output_format={ + "type": "json_schema", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ), + ), + Case("speed_on_model_without_speed", _kw(speed="fast")), + Case("speed_on_model_without_speed_drop_params", _kw(speed="fast", drop_params=True)), + Case("speed_on_fast_model", _kw(speed="fast"), model=OPUS_FAST), + Case("temperature_on_adaptive_model", _kw(temperature=0.5), model=OPUS_ADAPTIVE), + Case("top_k_on_adaptive_model", _kw(top_k=5), model=OPUS_ADAPTIVE), + Case("top_k_on_adaptive_model_drop_params", _kw(top_k=5, drop_params=True), model=OPUS_ADAPTIVE), + Case( + "additional_drop_params_hides_rejected_temperature", + _kw(temperature=0.5, additional_drop_params=["temperature"]), + model=OPUS_ADAPTIVE, + ), + Case( + "additional_drop_params_nested_path", + _kw( + tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}], + additional_drop_params=["tools[*].input_examples", "metadata"], + metadata={"user_id": "user-123"}, + ), + ), + Case( + "context_management_edits", + _kw(context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}), + ), + Case( + "container_skills", _kw(container={"skills": [{"type": "anthropic", "skill_id": "pptx", "version": "latest"}]}) + ), + Case( + "remote_mcp_servers", + _kw(mcp_servers=[{"type": "url", "url": "https://mcp.example.com/sse", "name": "example"}]), + ), + Case("inference_geo", _kw(inference_geo="us")), + Case("top_level_cache_control", _kw(cache_control={"type": "ephemeral"})), + Case("null_safeguards_are_stripped", _kw(safeguards=None)), + Case( + "empty_text_blocks_are_stripped", + _kw( + messages=[ + {"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "hi"}]}, + ] + ), + ), + Case( + "tool_use_ids_are_sanitized", + _kw( + messages=[ + {"role": "user", "content": "Weather?"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "call:abc/1", "name": "get_weather", "input": dict(NO_INPUT)} + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call:abc/1", "content": "sunny"}]}, + ], + tools=[dict(WEATHER_TOOL)], + ), + ), + Case( + "provider_specific_fields_are_stripped", + _kw(messages=[{"role": "user", "content": "hi", "provider_specific_fields": {"x": 1}}]), + ), + Case( + "image_block", + _kw( + messages=[ + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_PIXEL}}, + {"type": "text", "text": "Describe it"}, + ], + } + ] + ), + ), + Case( + "document_block", + _kw( + messages=[ + { + "role": "user", + "content": [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Paris."}}, + {"type": "text", "text": "Summarize"}, + ], + } + ] + ), + ), + Case("server_web_search_tool", _kw(tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 2}])), + Case("extra_headers", _kw(extra_headers={"x-trace-id": "trace-1", "anthropic-beta": "files-api-2025-04-14"})), + Case( + "anthropic_beta_from_headers_and_extra_headers", + _kw(headers={"anthropic-beta": "client-beta-1"}, extra_headers={"anthropic-beta": "deploy-beta-2"}), + ), + Case("caller_anthropic_version_capitalised", _kw(extra_headers={"Anthropic-Version": "2023-06-01"})), + Case( + "provider_specific_header", + _kw(provider_specific_header={"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "yes"}}), + ), + Case("explicit_custom_llm_provider", _kw(custom_llm_provider="anthropic"), model="claude-sonnet-4-5"), + Case("bare_model_name", model="claude-sonnet-4-5"), + Case("api_base_with_messages_path", _kw(api_base_suffix="/v1/messages")), + Case( + "api_base_with_messages_path_and_query", + _kw(api_base_suffix="/v1/messages?beta=true"), + ), + Case("api_key_from_environment", _kw(api_key=None), env=MappingProxyType({"ANTHROPIC_API_KEY": API_KEY})), + Case( + "auth_token_from_environment", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_AUTH_TOKEN": "oauth-token"}), + ), + Case( + "reasoning_auto_summary", + _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + env=MappingProxyType({"LITELLM_REASONING_AUTO_SUMMARY": "true"}), + ), + Case("global_drop_params", _kw(top_k=5), model=OPUS_ADAPTIVE, settings=MappingProxyType({"drop_params": True})), + Case("request_timeout_setting", settings=MappingProxyType({"request_timeout": 30})), + Case( + "litellm_metadata_and_metadata", + _kw( + metadata={"user_id": "user-123"}, + litellm_metadata={"user_api_key": "hashed-key", "user_api_key_team_id": "t1"}, + ), + ), + Case("model_info_with_id", _kw(model_info={"id": "deployment-1", "db_model": False})), + Case( + "deployment_effort_levels", + _kw( + output_config={"effort": "max"}, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low", "high"]}, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "deployment_effort_levels_from_legacy_thinking", + _kw( + thinking={"type": "enabled", "budget_tokens": 30000}, + max_tokens=40000, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low", "medium"]}, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "deployment_effort_levels_below_request_with_drop_params", + _kw( + output_config={"effort": "high"}, + drop_params=True, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low"]}, + ), + model=OPUS_ADAPTIVE, + ), +) + +RESPONSE_CASES: Final = ( + Case("response_with_tool_use", _kw(tools=[dict(WEATHER_TOOL)]), upstream=(_ok(TOOL_USE_RESPONSE),)), + Case("response_with_thinking", upstream=(_ok(THINKING_RESPONSE),)), + Case("response_with_cache_usage", upstream=(_ok(CACHED_USAGE_RESPONSE),)), + Case( + "response_with_null_usage_and_container", + upstream=(_ok({**TEXT_RESPONSE, "usage": None, "container": None}),), + ), + Case( + "response_without_stop_sequence", + upstream=(_ok({k: v for k, v in TEXT_RESPONSE.items() if k != "stop_sequence"}),), + ), + Case("response_with_unknown_field", upstream=(_ok({**TEXT_RESPONSE, "future_field": {"a": 1}}),)), + Case("response_key_order", upstream=(_ok({"usage": TEXT_RESPONSE["usage"], **TEXT_RESPONSE}),)), + Case( + "response_upstream_headers", + upstream=( + _ok(TEXT_RESPONSE, **{"request-id": "req_01Parity", "anthropic-ratelimit-requests-remaining": "99"}), + ), + ), + Case("response_private_keys", private_keys=True), +) + +FAILURE_CASES: Final = ( + Case("upstream_400", upstream=(_failure(400, "invalid_request_error", "max_tokens: too large"),)), + Case("upstream_401", upstream=(_failure(401, "authentication_error", "invalid x-api-key"),)), + Case("upstream_403", upstream=(_failure(403, "permission_error", "not allowed"),)), + Case("upstream_404", upstream=(_failure(404, "not_found_error", "model not found"),)), + Case("upstream_413", upstream=(_failure(413, "request_too_large", "too large"),)), + Case( + "upstream_429_with_retry_after", + upstream=( + _failure(429, "rate_limit_error", "slow down", **{"retry-after": "7", "request-id": "req_01Parity"}), + ), + ), + Case("upstream_500", upstream=(_failure(500, "api_error", "internal"),)), + Case("upstream_529_overloaded", upstream=(_failure(529, "overloaded_error", "Overloaded"),)), + Case( + "upstream_400_context_window", + upstream=(_failure(400, "invalid_request_error", "prompt is too long: 250000 tokens > 200000 maximum"),), + ), + Case("upstream_non_json_error", upstream=(ResponseSpec(body=b"bad gateway", status=502),)), + Case("upstream_invalid_json_success", upstream=(ResponseSpec(body=b"not json", status=200),)), + Case("upstream_timeout", _kw(timeout=0.5), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=1.5),)), + Case( + "response_trickles_slower_than_timeout", + _kw(timeout=1), + upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), chunk_delay=0.5, body_chunks=4),), + gaps=_gaps( + outcome=S15_COMPLETE_DEADLINE, + hidden_params=S15_COMPLETE_DEADLINE, + logged=S15_COMPLETE_DEADLINE, + logged_metadata=S15_COMPLETE_DEADLINE, + logged_response_headers=S15_COMPLETE_DEADLINE, + ), + ), + Case("missing_max_tokens", _kw(max_tokens=None), upstream=()), + Case("missing_api_key", _kw(api_key=None), upstream=()), +) + +STREAM_CASES: Final = ( + Case("stream_text", _kw(stream=True), upstream=(_stream(TEXT_STREAM),)), + Case("stream_tool_use", _kw(stream=True, tools=[dict(WEATHER_TOOL)]), upstream=(_stream(TOOL_STREAM),)), + Case( + "stream_thinking", + _kw(stream=True, thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + upstream=(_stream(THINKING_STREAM),), + ), + Case("stream_error_event_mid_stream", _kw(stream=True), upstream=(_stream(ERROR_MID_STREAM),)), + Case( + "stream_longer_than_timeout", + _kw(stream=True, timeout=1), + upstream=(_stream(SLOW_STREAM, chunk_delay=0.25),), + ), + Case( + "stream_read_stalls_past_timeout", + _kw(stream=True, timeout=0.5), + upstream=(_stream(SLOW_STREAM, chunk_delay=1.0),), + ), + Case( + "stream_upstream_429", + _kw(stream=True), + upstream=(_failure(429, "rate_limit_error", "slow down", **{"retry-after": "3"}),), + ), + Case("stream_false_explicit", _kw(stream=False)), +) + +SYNC_CASES: Final = ( + Case("sync_text", sync=True), + Case("sync_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), sync=True), + Case("sync_upstream_429", upstream=(_failure(429, "rate_limit_error", "slow down"),), sync=True), +) + +FALLBACK_CASES: Final = ( + Case("mock_response", _kw(mock_response="Hi from the mock"), upstream=()), + Case("mock_response_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=()), + Case( + "thinking_replay", + _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + ), + Case( + "thinking_signature_rejected_then_retried", + _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + upstream=( + _failure(400, "invalid_request_error", "messages.1.content.0: Invalid `signature` in `thinking` block"), + _ok(), + ), + ), + Case( + "cache_control_injection_points", + _kw(system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "system"}]), + ), + Case("tool_custom_field", _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True}}])), + Case( + "strict_tools_rejected_then_retried", + _kw(tools=[{**WEATHER_TOOL, "strict": True}], drop_params=True), + upstream=( + _failure(400, "invalid_request_error", "tools.0.strict: too many strict tools"), + _ok(), + ), + ), + Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})), + Case( + "stream_cost_injection", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM),), + settings=MappingProxyType({"include_cost_in_streaming_usage": True}), + ), + Case("enable_prompt_caching", _kw(system="Answer in one word.", enable_prompt_caching=True)), + Case("request_timeout_kwarg", _kw(request_timeout=30)), + Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)), +) + +CASES: Final = (*REQUEST_CASES, *RESPONSE_CASES, *FAILURE_CASES, *STREAM_CASES, *SYNC_CASES, *FALLBACK_CASES) + + +@dataclass(frozen=True, slots=True) +class Leg: + fell_back: bool + requests: tuple[Mapping[str, object], ...] + outcome: Mapping[str, object] + hidden_params: Mapping[str, object] + logged: Mapping[str, object] + callback_kwargs: Mapping[str, object] + + +@dataclass(slots=True) +class FallbackProbe: + calls: int = 0 + + +_PYTHON_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch._PYTHON_AMESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison +_PYTHON_MESSAGES: Final = cast(Callable[..., object], dispatch._PYTHON_MESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison +_PUBLIC_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch.anthropic_messages) +_PUBLIC_MESSAGES: Final = cast(Callable[..., object], dispatch.anthropic_messages_handler) + + +@pytest.fixture +def native_messages() -> Iterator[FallbackProbe]: + probe: Final = FallbackProbe() + + async def python_afallback(*args: object, **kwargs: object) -> object: + probe.calls += 1 + return await _PYTHON_AMESSAGES(*args, **kwargs) + + def python_fallback(*args: object, **kwargs: object) -> object: + probe.calls += 1 + return _PYTHON_MESSAGES(*args, **kwargs) + + with ( + rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_OUT), *catalog.RULES)), + rebound(dispatch, "_PYTHON_AMESSAGES", python_afallback), + rebound(dispatch, "_PYTHON_MESSAGES", python_fallback), + ): + yield probe + + +@contextmanager +def _environment(env: Mapping[str, str]) -> Generator[None]: + # LITELLM_RUST overrides the catalog rule, so a shell that sets it would route the native leg to Python. + names: Final = ( + *env, + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "LITELLM_REASONING_AUTO_SUMMARY", + "LITELLM_RUST", + ) + with pytest.MonkeyPatch.context() as patch: + for name in names: + patch.delenv(name, raising=False) + for name, value in env.items(): + patch.setenv(name, value) + yield + + +@contextmanager +def _settings(settings: Mapping[str, object]) -> Generator[None]: + saved: Final[dict[str, object]] = {name: cast(object, getattr(litellm, name)) for name in settings} + for name, value in settings.items(): + setattr(litellm, name, value) + try: + yield + finally: + for name, value in saved.items(): + setattr(litellm, name, value) + + +def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: + # Deep: Python writes into the caller's own `litellm_metadata`, which must not leak into the + # other leg. + supplied: Final = copy.deepcopy(dict(case.kwargs)) + suffix: Final = supplied.pop("api_base_suffix", "") + kwargs: Final[dict[str, object]] = { + "model": case.model, + "max_tokens": 64, + "messages": [dict(USER)], + "api_key": API_KEY, + "api_base": f"{server.base_url}{suffix}", + **supplied, + } + return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards",)} + + +def _mapping(value: object) -> Mapping[str, object]: + return _LOGGED.validate_python(value) if isinstance(value, dict) else {} + + +def _normalized(text: str) -> str: + for pattern, replacement in _VOLATILE: + text = pattern.sub(replacement, text) # rebind-ok: successive substitutions + return text + + +def _error(error: BaseException, phase: str, chunks: Iterable[object] = ()) -> Mapping[str, object]: + header_items: Final = { + name.lower(): value for name, value in _mapping(getattr(error, "litellm_response_headers", None)).items() + } + return { + "kind": "error", + "phase": phase, + "class": type(error).__name__, + "status_code": getattr(error, "status_code", None), + "llm_provider": getattr(error, "llm_provider", None), + "model": getattr(error, "model", None), + "message": _normalized(str(error)), + "headers": {name: header_items.get(name) for name in ("retry-after", "request-id") if name in header_items}, + "partial": _stream_text(chunks), + } + + +def _stream_text(chunks: Iterable[object]) -> str: + return b"".join(chunk if isinstance(chunk, bytes) else str(chunk).encode() for chunk in chunks).decode() + + +def _response(result: object, private_keys: bool) -> Mapping[str, object]: + assert isinstance(result, dict), f"expected a dict response, got {type(result).__name__}" + body: Final = { + name: value + for name, value in cast(dict[str, object], result).items() + if private_keys or not name.startswith("_") + } + return {"kind": "response", "body": _OBJECT.validate_python(body)} + + +def _hidden_params(result: object) -> Mapping[str, object]: + hidden: Final = dict(_mapping(get_hidden_params_dict(result))) + headers: Final = _mapping(hidden.pop("additional_headers", None)) + header_names: Final = sorted(name for name in headers if name not in ("x-litellm-rust", "x-litellm-cache-key")) + return {"keys": sorted(hidden), "additional_headers": header_names} + + +async def _consume_async(stream: AsyncIterator[object]) -> tuple[list[object], BaseException | None]: + chunks: Final[list[object]] = [] + try: + async for chunk in stream: + chunks.append(chunk) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return chunks, error + return chunks, None + + +def _consume_sync(stream: Iterator[object]) -> tuple[list[object], BaseException | None]: + chunks: Final[list[object]] = [] + try: + for chunk in stream: + chunks.append(chunk) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return chunks, error + return chunks, None + + +async def _invoke( + case: Case, kwargs: Mapping[str, object], native: bool +) -> tuple[Mapping[str, object], Mapping[str, object]]: + if case.sync: + call = _PUBLIC_MESSAGES if native else _PYTHON_MESSAGES + try: + result = await asyncio.to_thread(lambda: call(**kwargs)) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return _error(error, "call"), {} + if isinstance(result, dict): + body = cast(Mapping[str, object], result) + return _response(body, case.private_keys), _hidden_params(body) + chunks, failure = await asyncio.to_thread(_consume_sync, cast(Iterator[object], result)) + else: + acall = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + try: + result = await acall(**kwargs) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return _error(error, "call"), {} + if isinstance(result, dict): + body = cast(Mapping[str, object], result) + return _response(body, case.private_keys), _hidden_params(body) + chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) + hidden: Final = _hidden_params(result) + if failure is not None: + return _error(failure, "stream", chunks), hidden + return { + "kind": "stream", + "chunk_types": sorted({type(chunk).__name__ for chunk in chunks}), + "text": _stream_text(chunks), + }, hidden + + +async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]: + for _ in range(100): + await drain_logging() + if any(name in _TERMINAL_EVENTS for name in recorder.names): + break + await asyncio.sleep(0.05) + await asyncio.sleep(0.2) + await drain_logging() + return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS) + + +def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]: + terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS] + if not terminal: + return {"events": events}, {} + details: Final = _mapping(terminal[-1].kwargs) + standard: Final = _mapping(details.get("standard_logging_object")) + error_class: Final = _mapping(standard.get("error_information")).get("error_class") + litellm_params: Final = _mapping(details.get("litellm_params")) + metadata: Final = litellm_params.get("metadata") + metadata_keys: Final = _mapping(metadata).keys() + logged: Final[Mapping[str, object]] = { + "events": events, + **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS}, + "error_class": error_class, + } + callback_kwargs: Final[Mapping[str, object]] = { + "logged_metadata": sorted(metadata_keys) if isinstance(metadata, dict) else metadata, + "logged_model_info": litellm_params.get("model_info"), + "logged_response_headers": bool(details.get("response_headers")), + "logged_api_key": details.get("api_key"), + "logged_call": { + "custom_llm_provider": details.get("custom_llm_provider"), + "model": details.get("model"), + "stream": details.get("stream"), + }, + } + return logged, callback_kwargs + + +async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: + with isolated_callback_registries(), _environment(case.env), _settings(case.settings): + recorder: Final = RecordingLogger() + litellm.callbacks.append(recorder) + for spec in case.upstream: + server.enqueue(replace(spec)) + request_count: Final = len(server.requests) + fallbacks_before: Final = 0 if probe is None else probe.calls + outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) + events: Final = await _terminal_events(recorder) + server.responses.clear() + logged, callback_kwargs = _logged(recorder, events) + return Leg( + fell_back=probe is not None and probe.calls > fallbacks_before, + requests=tuple( + { + "method": sent.method, + "path": sent.path, + "body": sent.body if sent.body is not None else sent.raw_body.decode(errors="replace"), + "headers": { + name: value for name, value in sorted(sent.headers.items()) if name not in _TRANSPORT_HEADERS + }, + } + for sent in server.requests[request_count:] + ), + outcome=outcome, + hidden_params=hidden, + logged=logged, + callback_kwargs=callback_kwargs, + ) + + +@pytest.fixture +def parity_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.expected_requests = None + return recording_server + + +_LEGS: Final[dict[str, tuple[Leg, Leg]]] = {} + + +async def _legs(case: Case, server: RecordingServer, probe: FallbackProbe) -> tuple[Leg, Leg]: + if case.name not in _LEGS: + _LEGS[case.name] = (await _leg(case, server, probe=None), await _leg(case, server, probe=probe)) + _dump(case.name, *_LEGS[case.name]) + return _LEGS[case.name] + + +def _dump(name: str, python: Leg, native: Leg) -> None: + """Writes both legs to ``$MESSAGES_PARITY_DUMP/.json`` when that variable is set.""" + dump: Final = os.environ.get("MESSAGES_PARITY_DUMP") + if not dump: + return + os.makedirs(dump, exist_ok=True) + with open(os.path.join(dump, f"{name}.json"), "w") as out: + json.dump({"python": asdict(python), "native": asdict(native)}, out, indent=1, default=repr) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("case", "aspect"), tuple(param for case in CASES for param in case.params())) +async def test_native_messages_match_python( + case: Case, aspect: str, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python, native = await _legs(case, parity_server, native_messages) + match aspect: + case "dispatch": + assert not native.fell_back + case "request": + assert native.requests == python.requests + case "outcome": + assert native.outcome == python.outcome + case "hidden_params": + assert native.hidden_params == python.hidden_params + case "logged": + assert native.logged == python.logged + case _: + assert native.callback_kwargs.get(aspect) == python.callback_kwargs.get(aspect) diff --git a/tests/test_litellm_rust/cache/test_python_cache.py b/tests/test_litellm_rust/cache/test_python_cache.py index 722cdbcbaca..c9ac7af1a78 100644 --- a/tests/test_litellm_rust/cache/test_python_cache.py +++ b/tests/test_litellm_rust/cache/test_python_cache.py @@ -5,27 +5,21 @@ import pytest from pydantic import TypeAdapter import litellm -from litellm import _v2 from litellm.caching.caching import Cache, CacheMode from litellm.caching.caching_handler import ( _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request ) from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict -from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth +from litellm.proxy._types import Litellm_EntityType from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, model_budget_spend_cache_key, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 from litellm.proxy.utils import InternalUsageCache -from litellm.rust_bridge import runtime -from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule -from litellm.rust_bridge.configuration import Rollout -from litellm.rust_bridge.public_call import NativeCall -from litellm.types.caching import CachingSupportedCallTypes +from litellm.rust_bridge.catalog import Route from tests.test_litellm_rust.cache.support import cache_key, collect, invoke, payload from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_EVENTS, MESSAGES_MODEL, MESSAGES_RESPONSE -from tests.test_litellm_rust.responses.data import RESPONSES_RESPONSE from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging, request_body from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.routing import native_route @@ -35,27 +29,18 @@ pytestmark = pytest.mark.requires_rust_extension @pytest.mark.asyncio @pytest.mark.parametrize( - ("route", "stream", "legacy"), - ( - ("chat", False, False), - ("messages", False, False), - ("responses", False, False), - ("messages", True, False), - ("messages", False, True), - ("messages", True, True), - ), + "stream", + (False, True), ) @pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) -async def test_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( +async def test_legacy_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( recording_server: RecordingServer, - route: Literal["chat", "messages", "responses"], stream: bool, native: bool, monkeypatch: pytest.MonkeyPatch, - legacy: bool, ) -> None: monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") - litellm.cache = Cache() if legacy else _v2.Cache.memory() + litellm.cache = Cache() counters: Final = litellm.DualCache() budget: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(counters) limiter: Final = PROXY_MaxParallelRequestsHandler_v3( @@ -75,7 +60,7 @@ async def test_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( } spend_key: Final = model_budget_spend_cache_key(Litellm_EntityType.KEY, key_hash, "cached-model", "1h") token_key: Final = limiter.create_rate_limit_keys("api_key", key_hash, "tokens") - first: Final = await invoke(route, recording_server, options, native=native) + first: Final = await invoke("messages", recording_server, options, native=native) if stream: await collect(first) await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) @@ -84,13 +69,13 @@ async def test_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( first_log: Final = TypeAdapter(dict[str, object]).validate_python(first_events[0].kwargs) first_payload: Final = TypeAdapter(dict[str, object]).validate_python(first_log["standard_logging_object"]) expected_cost: Final = TypeAdapter(float).validate_python(first_log["response_cost"]) - usage: Final = RESPONSES_RESPONSE["usage"] if route == "responses" else MESSAGES_RESPONSE["usage"] + usage: Final = MESSAGES_RESPONSE["usage"] expected_tokens: Final = usage["input_tokens"] + usage["output_tokens"] assert expected_cost > 0 assert counters.get_cache(spend_key) == pytest.approx(expected_cost) assert counters.get_cache(token_key) == first_payload["total_tokens"] == expected_tokens - second: Final = await invoke(route, recording_server, options, native=native) + second: Final = await invoke("messages", recording_server, options, native=native) if stream: await collect(second) await drain_logging() @@ -104,7 +89,7 @@ async def test_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( assert cached_payload["cache_hit"] is True assert cached_payload["id"] != first_payload["id"] assert cached_payload["custom_llm_provider"] == first_payload["custom_llm_provider"] - assert cached_payload["custom_llm_provider"] == ("openai" if route == "responses" else "anthropic"), { + assert cached_payload["custom_llm_provider"] == "anthropic", { "miss_provider": first_log.get("custom_llm_provider"), "hit_provider": cached_log.get("custom_llm_provider"), } @@ -114,110 +99,22 @@ async def test_cache_hit_keeps_model_budget_spend_but_accounts_for_usage( @pytest.mark.asyncio -@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) -@pytest.mark.parametrize("backend", ("disabled", "memory", "redis")) -async def test_response_cache_backend_does_not_control_coordination( - recording_server: RecordingServer, - native: bool, - monkeypatch: pytest.MonkeyPatch, - backend: Literal["disabled", "memory", "redis"], - redis_url: str, -) -> None: - monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") - litellm.cache = ( - None - if backend == "disabled" - else _v2.Cache.memory() - if backend == "memory" - else _v2.Cache.redis(redis_url, namespace="independent-coordination") - ) - recording_server.expected_requests = 2 if backend == "disabled" else 1 - counters: Final = litellm.DualCache() - budget: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(counters) - limiter: Final = PROXY_MaxParallelRequestsHandler_v3( - InternalUsageCache(counters), model_group_resolver=lambda model: model - ) - key_hash: Final = "b" * 64 - identity: Final = UserAPIKeyAuth(api_key=key_hash, rpm_limit=2, tpm_limit=1000, max_parallel_requests=1) - spend_key: Final = model_budget_spend_cache_key(Litellm_EntityType.KEY, key_hash, "cached-model", "1h") - request_key: Final = limiter.create_rate_limit_keys("api_key", key_hash, "requests") - token_key: Final = limiter.create_rate_limit_keys("api_key", key_hash, "tokens") - parallel_key: Final = limiter.create_rate_limit_keys("api_key", key_hash, "max_parallel_requests") - expected_tokens: Final = MESSAGES_RESPONSE["usage"]["input_tokens"] + MESSAGES_RESPONSE["usage"]["output_tokens"] - recorder: Final = RecordingLogger() - - async def request(call_id: str, successes: int) -> object: - data: Final = { - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "litellm_call_id": call_id, - "max_tokens": 32, - "metadata": { - "user_api_key": key_hash, - "model_group": "cached-model", - "user_api_key_model_max_budget": {"cached-model": {"max_budget": 1, "budget_duration": "1h"}}, - }, - } - await limiter.async_pre_call_hook(identity, counters, data, "acompletion") - assert len(TypeAdapter(dict[str, float]).validate_python(counters.get_cache(parallel_key))) == 1 - response: Final = await invoke( - "chat", recording_server, {**data, "callbacks": [budget, limiter, recorder]}, native=native - ) - await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) - await recorder.wait_for_async("async_log_success_event", count=successes) - return response - - await asyncio.create_task(request("cache-miss", 1)) - assert counters.get_cache(parallel_key) == {} - assert counters.get_cache(request_key) == 1 - assert counters.get_cache(token_key) == expected_tokens - first_events: Final = await recorder.wait_for_async("async_log_success_event") - first_cost: Final = TypeAdapter(float).validate_python(first_events[0].kwargs["response_cost"]) - assert first_cost > 0 - assert counters.get_cache(spend_key) == pytest.approx(first_cost) - await asyncio.create_task(request("cache-hit", 2)) - expected_spend: Final = first_cost * recording_server.expected_requests - assert counters.get_cache(spend_key) == pytest.approx(expected_spend) - assert len(recording_server.requests) == recording_server.expected_requests - assert counters.get_cache(parallel_key) == {} - assert counters.get_cache(request_key) == 2 - assert counters.get_cache(token_key) == 2 * expected_tokens - with pytest.raises(litellm.RateLimitError): - await asyncio.create_task(request("over-rpm-limit", 3)) - assert len(recording_server.requests) == recording_server.expected_requests - assert counters.get_cache(parallel_key) == {} - assert counters.get_cache(token_key) == 2 * expected_tokens - - assert counters.get_cache(spend_key) == pytest.approx(expected_spend) - if litellm.cache is not None: - await litellm.cache.disconnect() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("route", "legacy"), (("chat", False), ("messages", False), ("responses", False), ("messages", True)) -) -async def test_cache_controls_and_backend_credential_key_semantics( - recording_server: RecordingServer, - route: Literal["chat", "messages", "responses"], - legacy: bool, -) -> None: - recording_server.expected_requests = 3 if legacy else 4 - litellm.cache = Cache() if legacy else _v2.Cache.memory() - await invoke(route, recording_server, {"cache": {"no-store": True}}) - await invoke(route, recording_server, {}) - await invoke(route, recording_server, {}) +async def test_legacy_cache_controls_and_backend_credential_key_semantics(recording_server: RecordingServer) -> None: + recording_server.expected_requests = 3 + litellm.cache = Cache() + await invoke("messages", recording_server, {"cache": {"no-store": True}}) + await invoke("messages", recording_server, {}) + await invoke("messages", recording_server, {}) assert len(recording_server.requests) == 2 - await invoke(route, recording_server, {"cache": {"no-cache": True}}) - await invoke(route, recording_server, {"api_key": "another-key"}) + await invoke("messages", recording_server, {"cache": {"no-cache": True}}) + await invoke("messages", recording_server, {"api_key": "another-key"}) assert len(recording_server.requests) == recording_server.expected_requests @pytest.mark.asyncio -@pytest.mark.parametrize("legacy", (False, True)) -async def test_v2_messages_replays_a_completed_stream(recording_server: RecordingServer, legacy: bool) -> None: +async def test_legacy_messages_replays_a_completed_stream(recording_server: RecordingServer) -> None: recording_server.default_response = ResponseSpec(body=None, events=MESSAGES_EVENTS) - litellm.cache = Cache() if legacy else _v2.Cache.memory() + litellm.cache = Cache() recorder: Final = RecordingLogger() parameters: Final = { "model": MESSAGES_MODEL, @@ -283,74 +180,30 @@ async def test_rust_messages_uses_a_legacy_cache_without_python_inference( @pytest.mark.asyncio -@pytest.mark.parametrize("route", ("chat", "messages", "responses")) @pytest.mark.parametrize("native", (False, True)) -@pytest.mark.parametrize("excluded", (None, [], ["embedding"])) -async def test_v2_cache_honors_supported_call_types_for_reads_and_writes( +async def test_legacy_cache_default_off_requires_opt_in_even_for_existing_entries( recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, - route: Literal["chat", "messages", "responses"], native: bool, - excluded: list[CachingSupportedCallTypes] | None, ) -> None: monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") - litellm.cache = _v2.Cache.memory() - call_type: Final[CachingSupportedCallTypes] = ( - "acompletion" if route == "chat" else "anthropic_messages" if route == "messages" else "aresponses" - ) - recording_server.expected_requests = 4 - litellm.cache.supported_call_types = excluded - await invoke(route, recording_server, {}, native=native) - await invoke(route, recording_server, {}, native=native) - assert len(recording_server.requests) == 2 - litellm.cache.supported_call_types = [call_type] - await invoke(route, recording_server, {}, native=native) - assert len(recording_server.requests) == 3 - await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) - await invoke(route, recording_server, {}, native=native) - assert len(recording_server.requests) == 3 - litellm.cache.supported_call_types = excluded - await invoke(route, recording_server, {}, native=native) - assert len(recording_server.requests) == 4 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("native", (False, True)) -@pytest.mark.parametrize( - ("route", "legacy"), (("chat", False), ("messages", False), ("responses", False), ("messages", True)) -) -async def test_v2_default_off_requires_opt_in_even_for_existing_entries( - recording_server: RecordingServer, - monkeypatch: pytest.MonkeyPatch, - route: Literal["chat", "messages", "responses"], - native: bool, - legacy: bool, -) -> None: - monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") - litellm.cache = Cache() if legacy else _v2.Cache.memory() + litellm.cache = Cache() litellm.cache.mode = CacheMode.default_off recording_server.expected_requests = 4 - await invoke(route, recording_server, {}, native=native) - await invoke(route, recording_server, {}, native=native) + await invoke("messages", recording_server, {}, native=native) + await invoke("messages", recording_server, {}, native=native) assert len(recording_server.requests) == 2 - await invoke(route, recording_server, {"cache": {"use-cache": True}}, native=native) + await invoke("messages", recording_server, {"cache": {"use-cache": True}}, native=native) assert len(recording_server.requests) == 3 await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) - await invoke(route, recording_server, {"cache": {"use-cache": True}}, native=native) + await invoke("messages", recording_server, {"cache": {"use-cache": True}}, native=native) assert len(recording_server.requests) == 3 - await invoke(route, recording_server, {}, native=native) + await invoke("messages", recording_server, {}, native=native) assert len(recording_server.requests) == 4 @pytest.mark.asyncio -@pytest.mark.parametrize( - ("route", "legacy"), (("chat", False), ("messages", False), ("responses", False), ("messages", True)) -) -async def test_cache_lookup_uses_backend_request_callback_semantics( - recording_server: RecordingServer, - route: Literal["chat", "messages", "responses"], - legacy: bool, -) -> None: +async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recording_server: RecordingServer) -> None: class Rewrite(RecordingLogger): temperature = 0.1 @@ -359,22 +212,40 @@ async def test_cache_lookup_uses_backend_request_callback_semantics( super().log_pre_api_call(model, messages, kwargs) logger: Final = Rewrite() - litellm.cache = Cache() if legacy else _v2.Cache.memory() - recording_server.expected_requests = 1 if legacy else 2 - await invoke(route, recording_server, {"callbacks": [logger]}) - first_hit: Final = await invoke(route, recording_server, {"callbacks": [logger]}) + litellm.cache = Cache() + recording_server.expected_requests = 1 + await invoke("messages", recording_server, {"callbacks": [logger]}) + first_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) logger.temperature = 0.8 - await invoke(route, recording_server, {"callbacks": [logger]}) - second_hit: Final = await invoke(route, recording_server, {"callbacks": [logger]}) + await invoke("messages", recording_server, {"callbacks": [logger]}) + second_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) assert logger.names.count("log_pre_api_call") == 4 assert len(recording_server.requests) == recording_server.expected_requests assert recording_server.requests[0].body["temperature"] == 0.1 - if not legacy: - assert recording_server.requests[1].body["temperature"] == 0.8 assert isinstance(cache_key(first_hit), str) assert isinstance(cache_key(second_hit), str) - if not legacy: - assert cache_key(first_hit) != cache_key(second_hit) + + +@pytest.mark.asyncio +async def test_rust_chat_completions_rejects_a_configured_python_cache( + recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + recording_server.expected_requests = 0 + monkeypatch.setattr(litellm, "cache", Cache()) + with pytest.raises(RuntimeError, match="this inference route does not support configured caches"): + await invoke("chat", recording_server, {}) + assert recording_server.requests == [] + + +@pytest.mark.asyncio +async def test_rust_responses_rejects_a_configured_python_cache( + recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch +) -> None: + recording_server.expected_requests = 0 + monkeypatch.setattr(litellm, "cache", Cache()) + with pytest.raises(RuntimeError, match="this inference route does not support configured caches"): + await invoke("responses", recording_server, {}) + assert recording_server.requests == [] @pytest.mark.asyncio @@ -430,49 +301,6 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task( assert len(recording_server.requests) == 1 -def test_sync_rust_messages_calls_python_cache(recording_server: RecordingServer) -> None: - from litellm.rust_bridge.messages.entrypoints import NATIVE_MESSAGES - - litellm.cache = Cache() - recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) - arguments: Final = { - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "max_tokens": 32, - "api_key": "test-key", - "api_base": recording_server.base_url, - } - request: Final = NativeCall( - args=(), - kwargs=arguments, - base={ - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "max_tokens": 32, - "stream": None, - "api_key": "test-key", - "api_base": recording_server.base_url, - "custom_llm_provider": "anthropic", - **arguments, - }, - ) - - def call() -> object: - return runtime.run( - RouteContext(Route.MESSAGES), - binding=NATIVE_MESSAGES, - native=lambda hook: hook(request), - python=runtime.NO_PYTHON, - rules=(RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),), - ) - - first: Final = call() - second: Final = call() - assert payload(first) == payload(second) - assert cache_key(second) - assert len(recording_server.requests) == 1 - - @pytest.mark.asyncio @pytest.mark.parametrize("namespace_source", ("cache", "metadata")) async def test_rust_messages_legacy_cache_honors_request_namespaces( diff --git a/tests/test_litellm_rust/cache/v2/test_cache.py b/tests/test_litellm_rust/cache/v2/test_cache.py deleted file mode 100644 index 1461e6eac61..00000000000 --- a/tests/test_litellm_rust/cache/v2/test_cache.py +++ /dev/null @@ -1,207 +0,0 @@ -import asyncio -from collections.abc import AsyncIterator -from typing import Final, Literal - -import pytest -from pydantic import TypeAdapter - -import litellm -from litellm import _v2 -from litellm._v2.cache import NativeBackend -from litellm.caching.caching_handler import ( - _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request -) -from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict -from litellm.types.utils import ModelResponse -from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE -from tests.test_litellm_rust.responses.data import RESPONSES_MODEL, RESPONSES_RESPONSE, RESPONSES_TEXT -from tests.test_litellm_rust.cache.support import cache_key, invoke, payload -from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging -from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec - -pytestmark = pytest.mark.requires_rust_extension - - -@pytest.mark.asyncio -@pytest.mark.parametrize("route", ("chat", "messages", "responses")) -@pytest.mark.parametrize("backend", ("memory", "redis")) -async def test_v2_cache_skips_provider_and_reports_one_success_per_call( - recording_server: RecordingServer, - route: Literal["chat", "messages", "responses"], - backend: Literal["memory", "redis"], - redis_url: str, -) -> None: - recording_server.expected_requests = 2 - litellm.cache = _v2.Cache.memory() if backend == "memory" else _v2.Cache.redis(redis_url, namespace="headers") - recorder: Final = RecordingLogger() - first: Final = await invoke(route, recording_server, {"callbacks": [recorder]}) - await recorder.wait_for_async("async_log_success_event") - second: Final = await invoke(route, recording_server, {"callbacks": [recorder]}) - assert payload(first) == payload(second) - assert cache_key(first) is None - key: Final = cache_key(second) - assert isinstance(key, str) - assert key == get_hidden_params_dict(second)["cache_key"] - assert len(recording_server.requests) == 1 - await drain_logging() - successes: Final = await recorder.wait_for_async("async_log_success_event", count=2) - assert len(successes) == 2 - cached_log: Final = TypeAdapter(dict[str, object]).validate_python(successes[-1].kwargs) - assert cached_log["cache_hit"] is True - assert cached_log["response_cost"] == 0 - await litellm.cache.delete_cache_keys([key]) - refreshed: Final = await invoke(route, recording_server, {"callbacks": [recorder]}) - assert cache_key(refreshed) is None - assert len(recording_server.requests) == 2 - assert len(await recorder.wait_for_async("async_log_success_event", count=3)) == 3 - await litellm.cache.disconnect() - - -@pytest.mark.asyncio -async def test_v2_global_cache_leaves_legacy_only_calls_usable() -> None: - litellm.cache = _v2.Cache.memory() - response: Final = await litellm.aembedding( - model="openai/cache-test-embedding", - input=["hello"], - api_key="test-key", - mock_response=[0.25, 0.75], - ) - assert response.model_dump(include={"data"}) == { - "data": [{"embedding": [0.25, 0.75], "index": 0, "object": "embedding"}] - } - - -@pytest.mark.parametrize("route", ("chat", "responses")) -def test_v2_cache_works_through_python_inference( - recording_server: RecordingServer, route: Literal["chat", "messages", "responses"], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("LITELLM_RUST", "0") - litellm.cache = _v2.Cache.memory() - common: Final = {"api_key": "test-key", "api_base": recording_server.base_url} - if route == "responses": - recording_server.default_response = ResponseSpec(body=RESPONSES_RESPONSE) - parameters: Final = {"model": RESPONSES_MODEL, "input": "hello", **common} - first: Final = litellm.responses(**parameters) - second: Final = litellm.responses(**parameters) - assert payload(first) == payload(second) - else: - recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) - arguments: Final = {"model": MESSAGES_MODEL, "messages": list(MESSAGES), "max_tokens": 32, **common} - initial: Final = litellm.completion(**arguments) - cached: Final = litellm.completion(**arguments) - assert isinstance(initial, ModelResponse) and isinstance(cached, ModelResponse) - assert ( - initial.choices[0].message.content - == cached.choices[0].message.content - == MESSAGES_RESPONSE["content"][0]["text"] - ) - assert len(recording_server.requests) == 1 - - -@pytest.mark.asyncio -async def test_v2_facade_and_backend_share_storage_and_management() -> None: - cache: Final = _v2.Cache.memory() - await cache.async_add_cache({"answer": 7}, cache_key="shared") - assert cache.get_cache(cache_key="shared") == {"answer": 7} - assert await cache.ping() is True - await cache.delete_cache_keys(["shared"]) - assert await cache.async_get_cache(cache_key="shared") is None - cache.add_cache({"answer": 8}, cache_key="flush") - backend: Final = cache.cache - assert isinstance(backend, NativeBackend) - backend.flush_cache() - assert cache.get_cache(cache_key="flush") is None - await cache.disconnect() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("control", ("s-maxage", "s-max-age")) -async def test_v2_native_cache_accepts_existing_freshness_aliases( - recording_server: RecordingServer, control: str -) -> None: - litellm.cache = _v2.Cache.memory() - first: Final = await invoke("responses", recording_server, {}) - second: Final = await invoke("responses", recording_server, {"cache": {control: 600}}) - assert payload(first) == payload(second) - assert len(recording_server.requests) == 1 - - -@pytest.mark.asyncio -async def test_v2_cache_does_not_force_native_responses_streaming( - recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("LITELLM_RUST", "0") - litellm.cache = _v2.Cache.memory() - recording_server.default_response = ResponseSpec( - body=None, - events=( - ("response.created", {"type": "response.created", "sequence_number": 0, "response": RESPONSES_RESPONSE}), - ( - "response.completed", - {"type": "response.completed", "sequence_number": 1, "response": RESPONSES_RESPONSE}, - ), - ), - ) - response: Final = await litellm.aresponses( - model=RESPONSES_MODEL, - input="hello", - stream=True, - caching=False, - api_key="test-key", - api_base=recording_server.base_url, - ) - assert isinstance(response, AsyncIterator) - chunks: Final = [chunk async for chunk in response] - assert chunks[-1].type == "response.completed" - assert chunks[-1].response.output[0].content[0].text == RESPONSES_TEXT - - -@pytest.mark.asyncio -async def test_v2_cache_works_through_python_messages( - recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("LITELLM_RUST", "0") - litellm.cache = _v2.Cache.memory() - recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) - parameters: Final = { - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "max_tokens": 32, - "api_key": "test-key", - "api_base": recording_server.base_url, - } - first: Final = await litellm.anthropic_messages(**parameters) - await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) - second: Final = await litellm.anthropic_messages(**parameters) - assert payload(first) == payload(second) - assert len(recording_server.requests) == 1 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", (False, True)) -async def test_v2_redis_flush_only_removes_its_namespace( - redis_url: str, recording_server: RecordingServer, asynchronous: bool -) -> None: - own: Final = _v2.Cache.redis(redis_url, namespace="flush-own") - other: Final = _v2.Cache.redis(redis_url, namespace="flush-other") - litellm.cache = own - recording_server.expected_requests = 2 - await own.async_add_cache({"answer": "own"}, cache_key="shared") - await other.async_add_cache({"answer": "other"}, cache_key="shared") - await invoke("responses", recording_server, {}) - hit: Final = await invoke("responses", recording_server, {}) - assert isinstance(cache_key(hit), str) - assert await own.async_get_cache(cache_key="shared") == {"answer": "own"} - backend: Final = own.cache - assert isinstance(backend, NativeBackend) - if asynchronous: - await backend.async_flush_cache() - else: - backend.flush_cache() - assert await own.async_get_cache(cache_key="shared") is None - assert await other.async_get_cache(cache_key="shared") == {"answer": "other"} - refreshed: Final = await invoke("responses", recording_server, {}) - assert cache_key(refreshed) is None - assert len(recording_server.requests) == 2 - await own.disconnect() - await other.disconnect() diff --git a/tests/test_litellm_rust/messages/test_callbacks.py b/tests/test_litellm_rust/messages/test_callbacks.py index cd5478dfc3a..a3c03898a12 100644 --- a/tests/test_litellm_rust/messages/test_callbacks.py +++ b/tests/test_litellm_rust/messages/test_callbacks.py @@ -14,7 +14,6 @@ from litellm.types.integrations.argilla import ArgillaItem from litellm.types.utils import ModelResponse, StandardLoggingPayload from tests.test_litellm_rust.messages.data import BAD_REQUEST, MESSAGES_RESPONSE, RATE_LIMITED, call from tests.test_litellm_rust.support.callback_recorder import HookRecorder, RecordingLogger, request_body -from tests.test_litellm_rust.support.gaps import gap from tests.test_litellm_rust.support.isolation import rebound from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, relaxed from tests.test_litellm_rust.support.results import response_id, with_id @@ -22,7 +21,6 @@ from tests.test_litellm_rust.support.routing import assert_served_natively pytestmark = pytest.mark.requires_rust_extension -SYNC_HANDLERS_ON_ASYNC_CALLS: Final = gap("native async calls also run the sync success and failure handlers") _OBJECT: Final = TypeAdapter(dict[str, object]) _HEADERS: Final = TypeAdapter(dict[str, str]) @@ -211,7 +209,7 @@ async def test_pre_call_deployment_rejection_logs_failure_without_provider_io(se failures: Final = await recorder.wait_for_async("async_log_failure_event") assert raised.value is rejection - assert server.requests == [] + assert not server.requests assert len(failures) == 1 assert _OBJECT.validate_python(failures[0].kwargs)["exception"] is raised.value assert not {"log_pre_api_call", "log_success_event", "async_log_success_event"} & set(recorder.names) @@ -341,8 +339,8 @@ async def test_a_failing_logger_does_not_fail_the_call(server: RecordingServer) pytest.param("input", False, False, 1, id="global-input"), pytest.param("success", True, False, 1, id="global-async-success"), pytest.param("success", True, True, 1, id="request-async-success"), - pytest.param("success", False, False, 0, id="global-sync-success", marks=SYNC_HANDLERS_ON_ASYNC_CALLS), - pytest.param("success", False, True, 0, id="request-sync-success", marks=SYNC_HANDLERS_ON_ASYNC_CALLS), + pytest.param("success", False, False, 0, id="global-sync-success"), + pytest.param("success", False, True, 0, id="request-sync-success"), pytest.param("failure", True, False, 1, id="global-async-failure"), pytest.param("failure", False, False, 1, id="global-sync-failure"), pytest.param("failure", False, True, 1, id="request-sync-failure"), @@ -384,12 +382,11 @@ async def test_pre_call_rule_rejects_before_provider_io(server: RecordingServer) checked.append(text) return False - with rebound(litellm, "pre_call_rules", [reject]): - with pytest.raises(litellm.APIResponseValidationError): - await call(server) + with rebound(litellm, "pre_call_rules", [reject]), pytest.raises(litellm.APIResponseValidationError): + await call(server) assert checked == ["Hello"] - assert server.requests == [] + assert not server.requests @pytest.mark.asyncio @@ -419,11 +416,11 @@ async def test_dataset_hook_returning_none_drops_the_record_from_the_success_sin @pytest.mark.asyncio -async def test_sync_call_returns_the_provider_message(server: RecordingServer) -> None: +async def test_sync_call_raises_like_python_without_logging(server: RecordingServer) -> None: recorder: Final = RecordingLogger() - response: Final = await call(server, asynchronous=False, callbacks=[recorder]) + with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): + await call(server, asynchronous=False, callbacks=[recorder]) - assert_served_natively(server, response) - assert response_id(response) == MESSAGES_RESPONSE["id"] - assert len(await recorder.wait_for_async("log_success_event")) == 1 + assert not server.requests + assert not recorder.names diff --git a/tests/test_litellm_rust/messages/test_streaming.py b/tests/test_litellm_rust/messages/test_streaming.py index c700c093d1a..e57631f48b9 100644 --- a/tests/test_litellm_rust/messages/test_streaming.py +++ b/tests/test_litellm_rust/messages/test_streaming.py @@ -6,7 +6,6 @@ import pytest from tests.test_litellm_rust.messages.data import MESSAGES_EVENTS, MESSAGES_RESPONSE, call from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, relaxed -from tests.test_litellm_rust.support.results import consume from tests.test_litellm_rust.support.routing import assert_served_natively pytestmark = pytest.mark.requires_rust_extension @@ -46,16 +45,14 @@ async def test_stream_relays_provider_events_and_logs_success_once_after_the_las @pytest.mark.asyncio -async def test_sync_stream_relays_provider_events_and_logs_success_once(server: RecordingServer) -> None: - server.enqueue(STREAM) +async def test_sync_stream_raises_like_python_before_sending(server: RecordingServer) -> None: recorder: Final = RecordingLogger() - stream: Final = await call(server, asynchronous=False, stream=True, callbacks=[recorder]) - chunks: Final = await consume(stream) + with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): + await call(server, asynchronous=False, stream=True, callbacks=[recorder]) - assert_served_natively(server) - assert b"".join(chunks) == b"".join(STREAM.payloads()) - assert len(await recorder.wait_for_async("async_log_success_event")) == 1 + assert not server.requests + assert not recorder.names @pytest.mark.asyncio diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py index d7b82767b97..5743658abdf 100644 --- a/tests/test_litellm_rust/support/recording_server.py +++ b/tests/test_litellm_rust/support/recording_server.py @@ -25,13 +25,16 @@ class ResponseSpec: status: int = 200 headers: dict[str, str] = field(default_factory=dict) delay: float = 0 + chunk_delay: float = 0 events: tuple[tuple[str, object], ...] = () + # Splits a non-stream body into this many writes, `chunk_delay` apart, so it trickles in. + body_chunks: int = 1 def payloads(self) -> tuple[bytes, ...]: - if isinstance(self.body, bytes): - return (self.body,) if not self.events: - return (json.dumps(self.body).encode(),) + body: Final = self.body if isinstance(self.body, bytes) else json.dumps(self.body).encode() + size: Final = max(-(-len(body) // max(self.body_chunks, 1)), 1) + return tuple(body[start : start + size] for start in range(0, len(body), size)) or (body,) return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events) @@ -89,7 +92,9 @@ def recording_service() -> Iterator[RecordingServer]: self.send_header(name, value) self.end_headers() try: - for payload in payloads: + for index, payload in enumerate(payloads): + if index and response.chunk_delay: + time.sleep(response.chunk_delay) self.wfile.write(payload) self.wfile.flush() except (BrokenPipeError, ConnectionResetError): diff --git a/tests/unit/accounting/test_delivery_projection.py b/tests/unit/accounting/test_delivery_projection.py new file mode 100644 index 00000000000..0b1f36d6b48 --- /dev/null +++ b/tests/unit/accounting/test_delivery_projection.py @@ -0,0 +1,85 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from typing import Final +from urllib.parse import unquote + +from litellm.accounting.delivery_projection import entity_transactions, project_deliveries +from litellm.accounting.projection import Contribution +from litellm.repositories.accounting_delivery_repository import DeliveryProjection + +_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) + + +def _payload(spend: float, started: datetime = _NOW, window_start: datetime = _NOW) -> Mapping[str, object]: + return { + "request_id": f"request-{spend}-{started.isoformat()}", + "startTime": started, + "api_key": "key", + "user": "user/1", + "team_id": "team", + "organization_id": "org::1", + "spend": spend, + "metadata": json.dumps( + { + "accounting_entities": { + "api_key": "key", + "user": "user/1", + "team_id": "team", + "organization_id": "org::1", + }, + "accounting_windows": [ + { + "entity_type": "key", + "entity_id": "key", + "window_duration": "1d", + "window_start": window_start.isoformat(), + } + ], + } + ), + } + + +def _spend(contributions: tuple[Contribution, ...], table: str) -> tuple[float, ...]: + return tuple(float(dict(item.increments)["spend"]) for item in contributions if item.table == table) + + +def test_deliveries_for_one_destination_sum_and_keep_the_earliest_request_start() -> None: + later: Final = _NOW + timedelta(hours=1) + + contributions: Final = project_deliveries( + (DeliveryProjection(_payload(0.5, started=later)), DeliveryProjection(_payload(0.25))) + ) + + assert _spend(contributions, "LiteLLM_VerificationToken") == (0.75,) + windows: Final = tuple(item for item in contributions if item.table == "LiteLLM_BudgetWindowSpend") + assert len(windows) == 1 + assert dict(windows[0].increments) == {"spend": 0.75} + assert dict(windows[0].values)["started_at"] == str(_NOW) + + +def test_the_same_window_in_a_new_period_is_charged_to_its_own_row() -> None: + next_day: Final = _NOW + timedelta(days=1) + + contributions: Final = project_deliveries( + (DeliveryProjection(_payload(0.25)), DeliveryProjection(_payload(0.5, window_start=next_day))) + ) + + windows: Final = sorted( + (dict(item.values)["window_start"], dict(item.increments)["spend"]) + for item in contributions + if item.table == "LiteLLM_BudgetWindowSpend" + ) + assert windows == [(_NOW.isoformat(), 0.25), (next_day.isoformat(), 0.5)] + + +def test_entity_transactions_key_memberships_the_way_the_writer_applies_them() -> None: + transactions: Final = entity_transactions(project_deliveries((DeliveryProjection(_payload(0.25)),))) + + assert transactions["key_list_transactions"] == {"key": 0.25} + assert transactions["org_list_transactions"] == {"org::1": 0.25} + assert transactions["team_member_list_transactions"] == {"team_id::team::user_id::user/1": 0.25} + ((org_member_key, org_member_spend),) = transactions["org_member_list_transactions"].items() + _, quoted_org_id, _, quoted_user_id = org_member_key.split("::") + assert (unquote(quoted_org_id), unquote(quoted_user_id), org_member_spend) == ("org::1", "user/1", 0.25) diff --git a/tests/unit/accounting/test_measurements.py b/tests/unit/accounting/test_measurements.py deleted file mode 100644 index b65c3076c5d..00000000000 --- a/tests/unit/accounting/test_measurements.py +++ /dev/null @@ -1,235 +0,0 @@ -from datetime import datetime, timezone -from decimal import Decimal -from typing import Final - -import pytest - -from litellm.accounting import ( - CostEvent, - Dimension, - DimensionName, - InboundUsageEvent, - Measurement, - Metric, - MetricName, - Outcome, - OutboundUsageEvent, - PricingSnapshot, - measurements_for, -) -from litellm.types.utils import CallTypes - -_START: Final = datetime(2026, 1, 1, 23, 59, 59, tzinfo=timezone.utc) -_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc) -_DIMENSIONS: Final = ( - Dimension(name=DimensionName.TEAM, value="team"), - Dimension(name=DimensionName.TAG, value="alpha"), - Dimension(name=DimensionName.TAG, value="beta"), -) - - -def _snapshot(amount: Decimal) -> PricingSnapshot: - return PricingSnapshot( - calculator_version="test", - model="test-model", - provider=None, - pricing_source="upstream_header", - priced_model_key="test-model", - upstream_reported_cost=amount, - total_cost=amount, - ) - - -@pytest.mark.parametrize( - ("outcome", "outcome_metric"), - [ - (Outcome.SUCCEEDED, MetricName.SUCCESSFUL_REQUESTS), - (Outcome.FAILED, MetricName.FAILED_REQUESTS), - (Outcome.CANCELLED, MetricName.FAILED_REQUESTS), - ], -) -def test_caller_measurements_keep_both_request_and_outcome_counts(outcome, outcome_metric): - event = InboundUsageEvent( - event_id="caller-event", - request_id="caller", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=outcome, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)), Metric(name=outcome_metric, value=Decimal(1))), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="caller-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.REQUESTS, value=Decimal(1)), - ), - Measurement( - source_event_id="caller-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=outcome_metric, value=Decimal(1)), - ), - ) - - -def test_usage_measurements_keep_every_metric_in_the_start_time_bucket(): - event = OutboundUsageEvent( - event_id="usage-event", - attempt_id="attempt", - request_id="caller", - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=( - Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)), - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)), - Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)), - Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)), - ), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)), - ), - Measurement( - source_event_id="usage-event", - occurred_at=_START, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)), - ), - ) - - -@pytest.mark.parametrize("metrics", [None, ()]) -def test_unknown_or_empty_usage_does_not_fabricate_zero_measurements(metrics): - event = OutboundUsageEvent( - event_id="usage-event", - attempt_id="attempt", - request_id=None, - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=(), - metrics=metrics, - ) - assert measurements_for(event) == () - - -@pytest.mark.parametrize("amount", [Decimal(0), Decimal("0.1234567890123456789")]) -def test_cost_measurements_preserve_authoritative_amount_identity_and_occurrence(amount): - event = CostEvent( - event_id="cost-event", - usage_event_id="usage-event", - occurred_at=_END, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=amount, unit="USD"),), - snapshot=_snapshot(amount), - ) - assert measurements_for(event) == ( - Measurement( - source_event_id="cost-event", - occurred_at=_END, - dimensions=_DIMENSIONS, - metric=Metric(name=MetricName.SPEND, value=amount, unit="USD"), - ), - ) - - -def test_billed_retry_and_success_contribute_one_caller_request_and_both_charges(): - caller = InboundUsageEvent( - event_id="caller-event", - request_id="caller", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), - ) - retry = OutboundUsageEvent( - event_id="retry-usage", - attempt_id="retry", - request_id="caller", - retry_of_attempt_id=None, - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.FAILED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(4)),), - ) - success = OutboundUsageEvent( - event_id="success-usage", - attempt_id="success", - request_id="caller", - retry_of_attempt_id="retry", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_END, - outcome=Outcome.SUCCEEDED, - dimensions=_DIMENSIONS, - metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),), - ) - retry_cost = CostEvent( - event_id="retry-cost", - usage_event_id="retry-usage", - occurred_at=_START, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=Decimal("0.025"), unit="USD"),), - snapshot=_snapshot(Decimal("0.025")), - ) - success_cost = CostEvent( - event_id="success-cost", - usage_event_id="success-usage", - occurred_at=_START, - dimensions=_DIMENSIONS, - charges=(Metric(name=MetricName.SPEND, value=Decimal("0.125"), unit="USD"),), - snapshot=_snapshot(Decimal("0.125")), - ) - measurements = ( - *measurements_for(caller), - *measurements_for(retry), - *measurements_for(success), - *measurements_for(retry_cost), - *measurements_for(success_cost), - ) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.REQUESTS) == Decimal(1) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.TOTAL_TOKENS) == Decimal(16) - assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.SPEND) == Decimal("0.150") - assert {m.source_event_id for m in measurements} == { - "caller-event", - "retry-usage", - "success-usage", - "retry-cost", - "success-cost", - } diff --git a/tests/unit/accounting/test_metrics.py b/tests/unit/accounting/test_metrics.py new file mode 100644 index 00000000000..49b46eb36c2 --- /dev/null +++ b/tests/unit/accounting/test_metrics.py @@ -0,0 +1,28 @@ +from prometheus_client.metrics import MetricWrapperBase + +from litellm.accounting import metrics +from litellm.accounting.metrics import record_deliveries +from litellm.repositories.accounting_delivery_repository import DeliveryMetrics + + +def _sample(metric: MetricWrapperBase, suffix: str = "") -> float: + family = metric.collect()[0] + return next(sample.value for sample in family.samples if sample.name == family.name + suffix) + + +def test_delivery_metrics_publish_backlog_and_clamp_clock_skew() -> None: + batches_before = _sample(metrics._BATCH_TIME, "_count") + + record_deliveries( + DeliveryMetrics(pending_count=4, parked_count=1, oldest_pending_seconds=-3.0, batch_count=7, table_bytes=8192), + 0.5, + ) + + assert ( + _sample(metrics._PENDING), + _sample(metrics._PARKED), + _sample(metrics._OLDEST), + _sample(metrics._BATCH_COUNT), + _sample(metrics._TABLE_BYTES), + ) == (4.0, 1.0, 0.0, 7.0, 8192.0) + assert _sample(metrics._BATCH_TIME, "_count") == batches_before + 1 diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py new file mode 100644 index 00000000000..17186a89922 --- /dev/null +++ b/tests/unit/accounting/test_projection.py @@ -0,0 +1,557 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.accounting.projection import project +from litellm.accounting.snapshot import build_accounting_payload +from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.proxy_server import proxy_logging_obj +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.proxy.utils import PrismaClient +from litellm.types.utils import ModelResponse, Usage + +_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) +_FIELDS: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.parametrize("status", ["success", "failure"]) +@pytest.mark.parametrize("origin", [None, "auto_router", "billed_retry"]) +@pytest.mark.parametrize("spend", [0.0, 0.25]) +@pytest.mark.asyncio +async def test_daily_projection_matches_legacy_writer(status: str, origin: str | None, spend: float) -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "projection-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "projection-request", + "litellm_params": {"metadata": {"user_api_key": "key", "user_api_key_user_id": "user", "status": status}}, + }, + response_obj=ModelResponse( + id="projection-request", model="projection-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ) + metadata: Final = { + **_FIELDS.validate_json(payload["metadata"]), + **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if origin == "billed_retry" else {}), + **({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin} if origin not in (None, "billed_retry") else {}), + } + row: Final = {**payload, "spend": spend, "metadata": json.dumps(metadata), "request_duration_ms": 25} + legacy: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( + row, PrismaClient(database_url="postgresql://unused", proxy_logging_obj=proxy_logging_obj), "user" + ) + assert legacy is not None + accounting_input: Final[SpendLogsPayload] = {**row} + accounting: Final = build_accounting_payload( + accounting_input, + entities={"api_key": payload["api_key"], "user": payload["user"]}, + model_access_groups=(), + windows=(), + router=lambda: None, + ) + restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str)) + stored_daily: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend") + assert dict(stored_daily.increments) == {name: legacy[name] for name, _ in stored_daily.increments} + assert dict(stored_daily.key)["date"] == legacy["date"] + assert dict(stored_daily.key)["api_key"] == legacy["api_key"] + assert not {"messages", "response", "proxy_server_request", "requester_ip_address"}.intersection(accounting) + + +def test_projection_uses_frozen_savings_and_keeps_duplicate_entity_tags() -> None: + row: Final[Mapping[str, object]] = { + "request_id": "saved", + "startTime": _NOW, + "api_key": "key", + "user": "user", + "model": "unpriced-projection-test", + "model_group": "group", + "custom_llm_provider": "openai", + "request_tags": '["tag", "tag"]', + "spend": 0.5, + "metadata": json.dumps( + { + "accounting_savings": { + "compression": 1.0, + "prompt_caching": -2.0, + "gateway_injected_caching": 3.0, + "autorouter": 4.0, + }, + "autorouter_savings": 9000, + } + ), + } + contributions: Final = project(row) + tags: Final = tuple(item for item in contributions if item.table == "LiteLLM_TagTable") + daily_tags: Final = tuple(item for item in contributions if item.table == "LiteLLM_DailyTagSpend") + assert sum(dict(item.increments)["spend"] for item in tags) == 1.0 + assert len(daily_tags) == 1 + assert {key: value for key, value in daily_tags[0].increments if key.endswith("savings_spend")} == { + "compression_savings_spend": 1.0, + "prompt_caching_savings_spend": -2.0, + "gateway_injected_caching_savings_spend": 3.0, + "autorouter_savings_spend": 4.0, + } + + +def test_projection_carries_windows_and_does_not_reprice_guardrails() -> None: + row: Final = { + "request_id": "guarded", + "startTime": _NOW, + "api_key": "key", + "spend": 0.75, + "metadata": json.dumps( + { + "accounting_windows": [ + { + "entity_type": "key", + "entity_id": "key", + "window_duration": "1d", + "window_start": _NOW.isoformat(), + } + ], + "guardrail_information": [ + { + "guardrail_id": "guard", + "guardrail_usage": {"text": 3, "images": 2}, + "guardrail_cost_by_unit": {"text": 0.3, "images": None}, + } + ], + } + ), + } + contributions: Final = project(row) + window: Final = next(item for item in contributions if item.table == "LiteLLM_BudgetWindowSpend") + units: Final = { + dict(item.key)["usage_unit"]: dict(item.increments) + for item in contributions + if item.table == "LiteLLM_DailyGuardrailUsageUnits" + } + assert dict(window.increments) == {"spend": 0.75} + assert dict(window.values)["window_start"] == _NOW.isoformat() + assert units == { + "text": {"units": 3, "cost": 0.3, "untracked_units": 0}, + "images": {"units": 2, "cost": 0.0, "untracked_units": 2}, + } + + +@pytest.mark.asyncio +async def test_entity_projection_matches_real_request_queues(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.responses.streaming_iterator import drain_logging_tasks + + client: Final = PrismaClient(database_url="postgresql://unused", proxy_logging_obj=proxy_logging_obj) + monkeypatch.setattr(proxy_server, "prisma_client", client) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setattr(client, "spend_log_transactions", []) + monkeypatch.setattr(client, "model_usage_transactions", []) + writer: Final = DBSpendUpdateWriter() + charged: Final = await writer.update_database( + token="sk-projection", + user_id="user", + end_user_id="end-user", + team_id="team", + org_id="org", + project_id="project", + kwargs={ + "model": "projection-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "entity-request", + "agent_id": "agent", + "user": "end-user", + "litellm_params": { + "metadata": { + "user_api_key": "sk-projection", + "user_api_key_user_id": "user", + "user_api_key_team_id": "team", + "user_api_key_org_id": "org", + "user_api_key_project_id": "project", + "user_api_key_end_user_id": "end-user", + "tags": ["tag", "tag"], + }, + "user": "end-user", + }, + }, + completion_response=ModelResponse( + id="entity-request", model="projection-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + response_cost=0.25, + ) + await drain_logging_tasks() + assert charged + assert len(client.spend_log_transactions) == 1 + queued: Final = await writer.spend_update_queue.flush_all_updates_from_in_memory_queue() + logged: Final[SpendLogsPayload] = client.spend_log_transactions[0] + projected: Final = project( + _FIELDS.validate_json( + json.dumps( + build_accounting_payload( + logged, + entities={ + "api_key": logged["api_key"], + "user": "user", + "end_user": "end-user", + "team_id": "team", + "organization_id": "org", + "project_id": "project", + "agent_id": "agent", + }, + model_access_groups=(), + windows=(), + router=lambda: None, + ), + default=str, + ) + ) + ) + legacy: Final = {(item["entity_type"].value, item["entity_id"]): item["response_cost"] for item in queued} + assert legacy[("key", proxy_server.hash_token("sk-projection"))] == next( + dict(item.increments)["spend"] for item in projected if item.table == "LiteLLM_VerificationToken" + ) + for table, entity, identity in ( + ("LiteLLM_UserTable", "user", "user"), + ("LiteLLM_TeamTable", "team", "team"), + ("LiteLLM_OrganizationTable", "organization", "org"), + ("LiteLLM_EndUserTable", "end_user", "end-user"), + ("LiteLLM_AgentsTable", "agent", "agent"), + ("LiteLLM_ProjectTable", "project", "project"), + ): + assert legacy[(entity, identity)] == next( + dict(item.increments)["spend"] for item in projected if item.table == table + ) + assert sum(dict(item.increments)["spend"] for item in projected if item.table == "LiteLLM_TagTable") == sum( + item["response_cost"] or 0 for item in queued if item["entity_type"].value == "tag" + ) + + daily_queues: Final = ( + ("LiteLLM_DailyUserSpend", writer.daily_spend_update_queue), + ("LiteLLM_DailyTeamSpend", writer.daily_team_spend_update_queue), + ("LiteLLM_DailyOrganizationSpend", writer.daily_org_spend_update_queue), + ("LiteLLM_DailyEndUserSpend", writer.daily_end_user_spend_update_queue), + ("LiteLLM_DailyAgentSpend", writer.daily_agent_spend_update_queue), + ("LiteLLM_DailyTagSpend", writer.daily_tag_spend_update_queue), + ) + for table, queue in daily_queues: + transactions = await queue.flush_and_get_aggregated_daily_spend_update_transactions() + matching = tuple(item for item in projected if item.table == table) + assert matching, table + contribution = matching[0] + assert len(transactions) == 1 + transaction = next(iter(transactions.values())) + assert dict(contribution.increments) == {name: transaction[name] for name, _ in contribution.increments} + assert len(client.model_usage_transactions) == 1 + model: Final = client.model_usage_transactions[0] + model_projection: Final = next(item for item in projected if item.table == "LiteLLM_DailyModelUsage") + assert dict(model_projection.increments) == { + "spend": model.spend, + "prompt_tokens": model.prompt_tokens, + "completion_tokens": model.completion_tokens, + "request_count": int(model.counts_as_request), + "successful_requests": int(model.successful), + "failed_requests": int(not model.successful), + } + + +@pytest.mark.parametrize("served_id", ["requested", "fallback", "missing"]) +@pytest.mark.asyncio +async def test_projection_preserves_fallback_narrowed_model_access_groups(served_id: str) -> None: + from litellm import Router + from litellm.proxy.db.db_spend_update_writer import debitable_model_access_groups + + router: Final = Router( + model_list=[ + { + "model_name": "first", + "litellm_params": {"model": "openai/projection-test", "api_key": "unused"}, + "model_info": {"id": "requested", "access_groups": ["first-group"]}, + }, + { + "model_name": "second", + "litellm_params": {"model": "openai/projection-test", "api_key": "unused"}, + "model_info": {"id": "fallback", "access_groups": ["second-group", "unattributed"]}, + }, + ] + ) + attributed: Final = ("first-group", "second-group", "first-group") + writer: Final = DBSpendUpdateWriter() + client: Final = PrismaClient(database_url="postgresql://unused", proxy_logging_obj=proxy_logging_obj) + await writer._update_model_access_group_db(0.5, attributed, served_id, client, router) + queued: Final = await writer.spend_update_queue.flush_all_updates_from_in_memory_queue() + frozen_groups: Final = debitable_model_access_groups(attributed, served_id, router) + projected: Final = project( + {"spend": 0.5, "metadata": json.dumps({"accounting_model_access_groups": frozen_groups})} + ) + assert {dict(item.key)["access_group_name"]: dict(item.increments)["spend"] for item in projected} == { + item["entity_id"]: item["response_cost"] for item in queued + } + assert "unattributed" not in frozen_groups + + +def test_guardrail_projection_skips_unmeasured_counters() -> None: + row: Final = { + "request_id": "guarded", + "startTime": _NOW, + "metadata": json.dumps( + { + "guardrail_information": [ + {"guardrail_usage": {"text": 3}}, + { + "guardrail_id": "guard", + "guardrail_usage": {"zero": 0, "negative": -1, "bool": True, "unknown": None, "text": 2}, + }, + ] + } + ), + } + units: Final = tuple(item for item in project(row) if item.table == "LiteLLM_DailyGuardrailUsageUnits") + assert len(units) == 1 + assert dict(units[0].key)["usage_unit"] == "text" + assert dict(units[0].increments) == {"units": 2, "cost": 0.0, "untracked_units": 2} + + +def test_lean_row_without_model_does_not_fabricate_daily_entity_usage() -> None: + projected: Final = project( + { + "request_id": "lean", + "api_key": "key", + "startTime": _NOW, + "spend": 0.5, + "metadata": {"accounting_entities": {"api_key": "key"}}, + } + ) + assert tuple(item.table for item in projected) == ("LiteLLM_VerificationToken", "LiteLLM_DailyModelUsage") + + +@pytest.mark.parametrize( + "cost_fields", + [ + {"guardrail_cost_in_spend": False, "guardrail_cost_by_unit": {"text": 1.0}}, + {"guardrail_cost_by_unit": {"text": -1.0}}, + ], +) +def test_report_only_or_invalid_guardrail_prices_remain_untracked(cost_fields: Mapping[str, object]) -> None: + row: Final = { + "request_id": "guarded", + "startTime": _NOW, + "request_tags": '["task:debugging"]', + "metadata": json.dumps( + { + "guardrail_information": [ + {"guardrail_id": "guard", "guardrail_usage": {"text": 2}, **cost_fields}, + ] + } + ), + } + contributions: Final = project(row) + guardrail: Final = next(item for item in contributions if item.table == "LiteLLM_DailyGuardrailUsageUnits") + model: Final = next(item for item in contributions if item.table == "LiteLLM_DailyModelUsage") + assert dict(guardrail.increments) == {"units": 2, "cost": 0.0, "untracked_units": 2} + assert dict(model.key)["task_type"] == "debugging" + + +@pytest.mark.parametrize("organization_id", [None, "", "charged-org"]) +def test_daily_organization_uses_frozen_charged_identity(organization_id: str | None) -> None: + row: Final[Mapping[str, object]] = { + "startTime": _NOW, + "api_key": "key", + "model": "projection-test", + "model_group": "group", + "custom_llm_provider": "openai", + "organization_id": "stored-org", + "spend": 0.25, + "metadata": {"accounting_entities": {"organization_id": organization_id}}, + } + daily: Final = tuple(item for item in project(row) if item.table == "LiteLLM_DailyOrganizationSpend") + if organization_id is None: + assert daily == () + else: + assert len(daily) == 1 + assert dict(daily[0].key)["organization_id"] == organization_id + assert dict(daily[0].increments)["spend"] == 0.25 + + +@pytest.mark.parametrize("start", [_NOW, "2026-11-02T12:00:00+00:00", "2026-11-02 12:00:00+00:00"]) +def test_every_daily_row_uses_the_calendar_date_of_the_request(start: object) -> None: + row: Final[Mapping[str, object]] = { + "request_id": "dated", + "startTime": start, + "api_key": "key", + "user": "user", + "team_id": "team", + "model": "unpriced-projection-test", + "model_group": "group", + "custom_llm_provider": "openai", + "spend": 0.5, + "metadata": "{}", + } + dates: Final = { + item.table: dict(item.key)["date"] for item in project(row) if item.table.startswith("LiteLLM_Daily") + } + assert {"LiteLLM_DailyUserSpend", "LiteLLM_DailyModelUsage"} <= dates.keys() + assert set(dates.values()) == {"2026-11-02"} + + +def test_unparseable_start_time_skips_only_guardrail_usage() -> None: + projected: Final = project( + { + "request_id": "bad-time", + "startTime": "yesterday at noon", + "spend": 0.5, + "metadata": { + "accounting_entities": {"api_key": "key"}, + "guardrail_information": [{"guardrail_id": "guard", "guardrail_usage": {"text": 2}}], + }, + } + ) + tables: Final = {item.table for item in projected} + assert "LiteLLM_VerificationToken" in tables + assert "LiteLLM_DailyGuardrailUsageUnits" not in tables + + +def test_guardrail_usage_is_dated_in_utc() -> None: + projected: Final = project( + { + "request_id": "offset", + "startTime": "2026-11-02T22:30:00-05:00", + "metadata": {"guardrail_information": [{"guardrail_id": "guard", "guardrail_usage": {"text": 1}}]}, + } + ) + assert [dict(item.key)["date"] for item in projected if item.table == "LiteLLM_DailyGuardrailUsageUnits"] == [ + "2026-11-03" + ] + + +@pytest.mark.parametrize( + ("untrusted", "metadata"), + [ + ({"request_tags": json.dumps(["x" * 16384, "normal"])}, {}), + ({"end_user": "x" * 3072}, {}), + ({}, {"compression_savings": {"tokens_saved": 2**63}}), + ], +) +def test_untrusted_analytics_values_never_block_the_charge( + untrusted: Mapping[str, str], metadata: Mapping[str, object] +) -> None: + row: Final[Mapping[str, object]] = { + "request_id": "untrusted", + "startTime": _NOW, + "api_key": "key", + "user": "user", + "team_id": "team", + "model": "unpriced-projection-test", + "model_group": "group", + "custom_llm_provider": "openai", + "spend": 0.5, + **untrusted, + "metadata": { + **metadata, + "accounting_entities": { + "api_key": "key", + "user": "user", + "team_id": "team", + "end_user": untrusted.get("end_user"), + }, + }, + } + contributions: Final = project(row) + texts: Final = tuple(value for item in contributions for _, value in (*item.key, *item.values) if value) + numbers: Final = tuple(value for item in contributions for _, value in item.increments) + assert {"LiteLLM_VerificationToken", "LiteLLM_TeamTable", "LiteLLM_UserTable"} <= { + item.table for item in contributions + } + assert "LiteLLM_DailyUserSpend" in {item.table for item in contributions} + assert all(len(text.encode()) <= 2000 for text in texts) + assert all(abs(number) <= 2**40 for number in numbers) + + +def test_stored_payload_keeps_guardrail_billing_but_not_guardrail_content() -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "projection-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "guardrail-request", + "litellm_params": {"metadata": {"user_api_key": "key", "user_api_key_user_id": "user"}}, + }, + response_obj=ModelResponse( + id="guardrail-request", model="projection-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ) + guardrail: Final = { + "guardrail_name": "pii", + "guardrail_usage": {"requests": 1}, + "guardrail_cost_by_unit": {"requests": 0.25}, + "guardrail_response": "card 4111 1111 1111 1111", + "match_details": [{"matched": "4111 1111 1111 1111"}], + } + headroom: Final = { + "guardrail_name": "compress", + "guardrail_provider": "headroom", + "guardrail_response": {"tokens_saved": 7, "compressed_prompt": "card 4111 1111 1111 1111"}, + } + metadata: Final = {**_FIELDS.validate_json(payload["metadata"]), "guardrail_information": [guardrail, headroom]} + accounting_input: Final[SpendLogsPayload] = { + **payload, + "spend": 0.5, + "metadata": json.dumps(metadata), + "request_tags": '["nul\x00tag"]', + } + accounting: Final = build_accounting_payload( + accounting_input, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None + ) + stored: Final = json.dumps(accounting, default=str) + restored: Final = _FIELDS.validate_json(stored) + usage_rows: Final = tuple(item for item in project(restored) if item.table == "LiteLLM_DailyGuardrailUsageUnits") + assert "4111" not in stored + assert "\\u0000" not in stored + assert [dict(item.increments) for item in usage_rows] == [{"units": 1, "cost": 0.25, "untracked_units": 0}] + daily_user: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend") + assert dict(daily_user.increments)["compression_saved_tokens"] == 7 + + +def test_caller_supplied_savings_that_cannot_be_computed_never_drop_the_charge(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + monkeypatch.setitem( + litellm.model_cost, + "savings-overflow-test", + {"input_cost_per_token": 0.002, "output_cost_per_token": 0.004, "litellm_provider": "openai", "mode": "chat"}, + ) + payload: Final[SpendLogsPayload] = { + **get_logging_payload( + kwargs={ + "model": "savings-overflow-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "overflow-request", + "litellm_params": {"metadata": {"user_api_key": "key"}}, + }, + response_obj=ModelResponse( + id="overflow-request", model="savings-overflow-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ), + "spend": 0.5, + "metadata": json.dumps({"compression_savings": {"tokens_saved": 10**400}}), + } + accounting: Final = build_accounting_payload( + payload, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None + ) + restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str)) + assert dict(restored["metadata"])["accounting_savings"]["compression"] == 0.0 + assert [dict(item.increments)["spend"] for item in project(restored) if item.table == "LiteLLM_VerificationToken"] == [ + 0.5 + ] diff --git a/tests/unit/accounting/test_types.py b/tests/unit/accounting/test_types.py deleted file mode 100644 index f645ee6c2cc..00000000000 --- a/tests/unit/accounting/test_types.py +++ /dev/null @@ -1,484 +0,0 @@ -import re -from collections.abc import Mapping -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from enum import Enum -from pathlib import Path -from typing import Final - -import pytest -from pydantic import ValidationError - -from litellm.accounting import ( - AdmissionDecision, - AggregateQuery, - AggregateRow, - CostEvent, - DeliveryState, - DeliveryTarget, - Dimension, - DimensionFilter, - DimensionName, - EventKind, - Granularity, - InboundUsageEvent, - Metric, - MetricName, - Multiplier, - Outcome, - OutboundUsageEvent, - PricedQuantity, - PricingSnapshot, - Rate, - validate_aggregate_rows, -) -from litellm.accounting.types import PricingUnit -from litellm.types.utils import CallTypes - -_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) -_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc) -_REQUEST: Final = Metric(name=MetricName.REQUESTS, value=Decimal(1)) -_TOKEN: Final = Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)) -_SPEND: Final = Metric(name=MetricName.SPEND, value=Decimal("0.1234567890123456789"), unit="USD") -_TEAM: Final = Dimension(name=DimensionName.TEAM, value="team") - - -def _inbound(metrics=(_REQUEST,), outcome=Outcome.SUCCEEDED): - return InboundUsageEvent( - event_id="inbound:request", - request_id="request", - call_type=CallTypes.acompletion, - started_at=_START, - ended_at=_START, - outcome=outcome, - dimensions=(_TEAM,), - metrics=metrics, - ) - - -def _outbound(metrics=(_TOKEN,), **changes): - return OutboundUsageEvent.model_validate( - { - "event_id": "outbound:attempt", - "attempt_id": "attempt", - "request_id": "request", - "retry_of_attempt_id": None, - "call_type": CallTypes.acompletion, - "started_at": _START, - "ended_at": _END, - "outcome": Outcome.SUCCEEDED, - "dimensions": (_TEAM,), - "metrics": metrics, - **changes, - } - ) - - -def _query(**changes): - return AggregateQuery.model_validate( - { - "metric": MetricName.SPEND, - "filters": (DimensionFilter(name=DimensionName.TEAM, values=("one", "two")),), - "group_by": (DimensionName.MODEL,), - "start": _START, - "end": _END, - "granularity": Granularity.DAY, - **changes, - } - ) - - -def test_wire_values_preserve_decimal_precision_and_normalize_aware_times(): - offset = timezone(timedelta(hours=-8)) - event = CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=datetime(2025, 12, 31, 16, tzinfo=offset), - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - wire = event.model_dump(mode="json") - assert wire["occurred_at"] == "2026-01-01T00:00:00Z" - assert wire["charges"] == [{"name": "spend", "value": "0.1234567890123456789", "unit": "USD"}] - assert event.occurred_at == _START - assert CostEvent.model_validate_json(event.model_dump_json()).charges[0].value == _SPEND.value - - -@pytest.mark.parametrize( - ("name", "value", "unit", "message"), - [ - (MetricName.SPEND, Decimal(1), "count", "SPEND requires USD"), - (MetricName.REQUESTS, Decimal(1), "USD", "SPEND requires USD"), - (MetricName.PROMPT_TOKENS, Decimal("1.5"), "count", "integral"), - (MetricName.SPEND, Decimal("NaN"), "USD", "finite"), - (MetricName.SPEND, Decimal("Infinity"), "USD", "finite"), - (MetricName.SPEND, Decimal(-1), "USD", "greater than or equal"), - (MetricName.SPEND, 0.1, "USD", "instance of Decimal"), - ], -) -def test_metrics_reject_wrong_units_nonfinite_values_and_binary_float_inputs(name, value, unit, message): - with pytest.raises(ValidationError, match=message): - Metric(name=name, value=value, unit=unit) - - -@pytest.mark.parametrize("metrics", [(_TOKEN,), (_SPEND,), ()]) -def test_inbound_activity_cannot_own_usage_or_cost_or_omit_request_count(metrics): - with pytest.raises(ValidationError, match="Inbound events"): - _inbound(metrics) - - -@pytest.mark.parametrize( - ("outcome", "metric"), - [ - (Outcome.FAILED, Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(1))), - (Outcome.SUCCEEDED, Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(1))), - ], -) -def test_inbound_outcome_counts_cannot_contradict_outcome(outcome, metric): - with pytest.raises(ValidationError, match="count must match outcome"): - _inbound((_REQUEST, metric), outcome) - - -@pytest.mark.parametrize("metrics", [(_REQUEST,), (_SPEND,)]) -def test_outbound_attempts_cannot_own_caller_counts_or_charges(metrics): - with pytest.raises(ValidationError, match="Outbound events own only"): - _outbound(metrics) - - -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"started_at": datetime(2026, 1, 1)}, "timezone"), - ({"ended_at": _START - timedelta(seconds=1)}, "End must follow start"), - ({"retry_of_attempt_id": "attempt"}, "cannot retry itself"), - ({"dimensions": (_TEAM, _TEAM)}, "Duplicate dimension values"), - ({"dimensions": (_TEAM, Dimension(name=DimensionName.TEAM, value="other"))}, "Only tag and model access group"), - ( - { - "metrics": ( - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(5)), - Metric(name=MetricName.REASONING_TOKENS, value=Decimal(6)), - ) - }, - "reasoning_tokens cannot exceed completion_tokens", - ), - ( - { - "metrics": ( - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(5)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(6)), - ) - }, - "cache_creation_input_tokens_above_1hr cannot exceed cache_creation_input_tokens", - ), - ({"event_id": " "}, "pattern"), - ({"metrics": (_TOKEN, _TOKEN)}, "Duplicate metric names"), - ], -) -def test_outbound_events_reject_invalid_identity_time_and_duplicate_attribution(changes, message): - with pytest.raises(ValidationError, match=message): - _outbound(**changes) - - -def test_unknown_and_known_zero_usage_are_distinct_and_values_are_immutable(): - unknown = _outbound(None) - zero = _outbound((Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(0)),)) - assert unknown.metrics is None - assert zero.metrics is not None and zero.metrics[0].value == 0 - assert _inbound().started_at == _inbound().ended_at - with pytest.raises(ValidationError, match="frozen"): - zero.attempt_id = "different" - with pytest.raises(ValidationError, match="frozen"): - zero.metrics[0].value = Decimal(10) - - -@pytest.mark.parametrize("charges", [(), (_TOKEN,), (_SPEND, _SPEND)]) -def test_cost_events_require_one_authoritative_spend(charges): - with pytest.raises(ValidationError, match=r"authoritative USD spend|Duplicate metric"): - CostEvent( - event_id="cost", - usage_event_id="usage", - occurred_at=_START, - dimensions=(), - charges=charges, - snapshot=_snapshot(), - ) - - -def test_cost_event_cannot_reuse_its_usage_identity(): - with pytest.raises(ValidationError, match="distinct identities"): - CostEvent( - event_id="usage", - usage_event_id="usage", - occurred_at=_START, - dimensions=(), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - - -@pytest.mark.parametrize("granularity", [Granularity.DAY, None]) -def test_daily_and_whole_interval_queries_preserve_filter_and_group_semantics(granularity): - query = _query(granularity=granularity) - assert query.granularity is granularity - assert query.filters[0].values == ("one", "two") - assert query.group_by == (DimensionName.MODEL,) - assert query.start == _START and query.end == _END - - -@pytest.mark.parametrize( - ("changes", "message"), - [ - ({"end": _START}, "End must follow start"), - ({"start": _START + timedelta(hours=1)}, "UTC day boundaries"), - ({"end": _END + timedelta(microseconds=1)}, "UTC day boundaries"), - ({"granularity": "hour"}, "day"), - ({"group_by": (DimensionName.TEAM, DimensionName.TEAM)}, "Duplicate group-by"), - ({"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one",)),) * 2}, "one IN filter"), - ({"filters": ({"name": "team", "values": ()},)}, "at least 1"), - ], -) -def test_queries_reject_unsupported_or_ambiguous_ranges_and_filters(changes, message): - with pytest.raises(ValidationError, match=message): - _query(**changes) - - -def test_rows_allow_adjacent_buckets_and_overlapping_distinct_groups(): - first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM,), value=Decimal(1)) - second = AggregateRow(start=_END, end=_END + timedelta(days=1), dimensions=(_TEAM,), value=Decimal(2)) - other = AggregateRow(start=_START, end=_END, dimensions=(), value=Decimal(3)) - rows = (second, other, first) - assert validate_aggregate_rows(rows) is rows - assert validate_aggregate_rows(()) == () - - -def test_rows_reject_overlap_within_a_group_regardless_of_dimension_order(): - model = Dimension(name=DimensionName.MODEL, value="model") - first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM, model), value=Decimal(1)) - overlapping = AggregateRow(start=_START, end=_END, dimensions=(model, _TEAM), value=Decimal(2)) - with pytest.raises(ValueError, match="overlap within a dimension group"): - validate_aggregate_rows((first, overlapping)) - - -def test_rows_reject_empty_intervals(): - with pytest.raises(ValidationError, match="End must follow start"): - AggregateRow(start=_START, end=_START, dimensions=(), value=Decimal(0)) - - -def test_admission_decision_preserves_a_denial_reason(): - decision = AdmissionDecision(allowed=False, reason="Team budget exhausted") - assert not decision.allowed - assert decision.reason == "Team budget exhausted" - - -def _snapshot(**changes: object) -> PricingSnapshot: - return PricingSnapshot.model_validate( - { - "calculator_version": "1.80.0", - "model": "gpt-4o", - "provider": "openai", - "pricing_source": "cost_map", - "priced_model_key": "openai/gpt-4o", - "multipliers": (Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")),), - "rates": (Rate(measure="input", unit="token", usd_per_unit=Decimal("0.0000025")),), - "usage": (PricedQuantity(measure="input", unit="token", value=_TOKEN.value),), - "input_cost": Decimal("0.000025"), - "total_cost": _SPEND.value, - **changes, - } - ) - - -def test_cost_event_carries_a_snapshot_whose_total_matches_the_spend(): - event = CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=_START, - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(), - ) - assert event.snapshot is not None - assert event.snapshot.total_cost == _SPEND.value - assert event.snapshot.multipliers[0].factor == Decimal("1.1") - with pytest.raises(ValidationError, match="Snapshot total must equal the authoritative spend"): - CostEvent( - event_id="cost:attempt", - usage_event_id="outbound:attempt", - occurred_at=_START, - dimensions=(_TEAM,), - charges=(_SPEND,), - snapshot=_snapshot(total_cost=Decimal("0.5")), - ) - - -@pytest.mark.parametrize( - ("changes", "message"), - ( - ( - { - "rates": ( - Rate(measure="input", unit="token", usd_per_unit=Decimal(1)), - Rate(measure="input", unit="token", usd_per_unit=Decimal(2)), - ) - }, - "Duplicate rate measures and units", - ), - ({"usage": (_SPEND,)}, "PricedQuantity"), - ({"usage": (_REQUEST,)}, "PricedQuantity"), - ({"pricing_source": "guess"}, "pricing_source"), - ({"multipliers": ({"name": "x", "value": "y", "factor": Decimal(0)},)}, "factor"), - ({"schema_version": 2}, "schema_version"), - ( - {"additional_costs": ({"name": "fee", "amount": Decimal(1)}, {"name": "fee", "amount": Decimal(2)})}, - "Duplicate additional cost names", - ), - ), -) -def test_snapshots_reject_ambiguous_rates_wrong_usage_and_unknown_sources(changes: Mapping[str, object], message: str): - with pytest.raises(ValidationError, match=message): - _snapshot(**changes) - - -@pytest.mark.parametrize( - ("unit", "value"), - [("token", "10"), ("second", "1.25"), ("character", "25"), ("image", "2"), ("request", "1")], -) -def test_snapshot_preserves_matching_rates_and_quantities_for_each_billing_unit(unit: PricingUnit, value: str) -> None: - quantity: Final = PricedQuantity(measure="input", unit=unit, value=Decimal(value)) - rate: Final = Rate(measure="input", unit=unit, usd_per_unit=Decimal("0.003")) - amount: Final = quantity.value * rate.usd_per_unit - snapshot: Final = _snapshot(rates=(rate,), usage=(quantity,), total_cost=amount) - restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json()) - assert restored == snapshot - assert restored.usage[0].value * restored.rates[0].usd_per_unit == amount - - -@pytest.mark.parametrize( - ("usage", "message"), - [ - ((), "Every rate requires"), - (({"measure": "output", "unit": "token", "value": Decimal(10)},), "Every rate requires"), - (({"measure": "input", "unit": "second", "value": Decimal(10)},), "Every rate requires"), - ( - ({"measure": "input", "unit": "token", "value": Decimal(10)},) * 2, - "Duplicate priced quantity", - ), - ], -) -def test_snapshot_rejects_missing_mismatched_or_duplicate_priced_quantities( - usage: tuple[Mapping[str, object], ...], message: str -) -> None: - with pytest.raises(ValidationError, match=message): - _snapshot(usage=usage) - - -@pytest.mark.parametrize("unit", ["token", "character", "image", "request"]) -def test_priced_counts_remain_integral(unit: PricingUnit) -> None: - with pytest.raises(ValidationError, match="integral"): - PricedQuantity(measure="input", unit=unit, value=Decimal("1.25")) - - -@pytest.mark.parametrize( - ("value", "message"), - [ - (Decimal(-1), "greater than or equal"), - (Decimal("NaN"), "finite"), - (Decimal("Infinity"), "finite"), - (1.25, "instance of Decimal"), - ], -) -def test_pricing_values_reject_invalid_amounts(value: Decimal | float, message: str) -> None: - with pytest.raises(ValidationError, match=message): - PricedQuantity.model_validate({"measure": "input", "unit": "second", "value": value}) - with pytest.raises(ValidationError, match=message): - Rate.model_validate({"measure": "input", "unit": "second", "usd_per_unit": value}) - - -@pytest.mark.parametrize("snapshot_fields", [{}, {"snapshot": None}]) -def test_cost_event_cannot_omit_its_derivation(snapshot_fields: Mapping[str, object]) -> None: - with pytest.raises(ValidationError, match="snapshot"): - CostEvent.model_validate( - { - "event_id": "cost", - "usage_event_id": "usage", - "occurred_at": _START, - "dimensions": (), - "charges": (_SPEND,), - **snapshot_fields, - } - ) - - -def test_upstream_charge_snapshot_preserves_reported_cost_without_inventing_rates() -> None: - snapshot: Final = _snapshot( - pricing_source="upstream_header", rates=(), usage=(), upstream_reported_cost=_SPEND.value - ) - restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json()) - assert restored.upstream_reported_cost == restored.total_cost == _SPEND.value - assert restored.rates == restored.usage == () - - -def test_subset_token_counts_round_trip_without_changing_totals() -> None: - metrics: Final = ( - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(10)), - Metric(name=MetricName.REASONING_TOKENS, value=Decimal(3)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(8)), - Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(2)), - ) - event: Final = _outbound(metrics=metrics) - assert OutboundUsageEvent.model_validate_json(event.model_dump_json()).metrics == metrics - - -@pytest.mark.parametrize( - "schema_path", - [ - "schema.prisma", - "litellm/proxy/schema.prisma", - "litellm-proxy-extras/litellm_proxy_extras/schema.prisma", - ], -) -@pytest.mark.parametrize( - ("enum_name", "python_enum"), - [ - ("AccountingEventKind", EventKind), - ("AccountingDeliveryState", DeliveryState), - ("AccountingDeliveryTarget", DeliveryTarget), - ], -) -def test_prisma_accepts_exactly_the_public_accounting_enum_values( - schema_path: str, enum_name: str, python_enum: type[Enum] -) -> None: - schema: Final = (Path(__file__).resolve().parents[3] / schema_path).read_text() - declaration: Final = re.search(r"enum " + enum_name + r"\s*\{([^}]+)\}", schema) - assert declaration is not None, f"Missing persisted enum {enum_name}" - assert frozenset(declaration[1].split()) == frozenset(member.value for member in python_enum) - - -def test_one_event_carries_every_matched_model_access_group(): - groups = ( - Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="gpt-group"), - Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="claude-group"), - ) - assert _outbound(dimensions=(_TEAM, *groups)).dimensions == (_TEAM, *groups) - with pytest.raises(ValidationError, match="Duplicate dimension values"): - _outbound(dimensions=(groups[0], groups[0])) - - -def test_snapshots_reject_duplicate_multipliers_and_contradictory_token_counts(): - multiplier = Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")) - with pytest.raises(ValidationError, match="Duplicate multiplier names"): - _snapshot(multipliers=(multiplier, multiplier)) - with pytest.raises(ValidationError, match="reasoning_tokens cannot exceed completion_tokens"): - _snapshot( - usage=( - PricedQuantity(measure="input", unit="token", value=_TOKEN.value), - PricedQuantity(measure="completion_tokens", unit="token", value=Decimal(5)), - PricedQuantity(measure="reasoning_tokens", unit="token", value=Decimal(6)), - ) - ) - - -def test_agent_live_spend_is_a_delivery_target(): - assert DeliveryTarget.AGENT_SPEND.value == "LiteLLM_AgentsTable" diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index b32afd51fd3..0f85f1e3993 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -22,7 +22,6 @@ from typing import Final, Literal import httpx import pytest import respx -from openai.types.batch import BatchRequestCounts import litellm @@ -2028,61 +2027,6 @@ def test_extract_credentials_keeps_openai_workload_identity_params(): } -# --------------------------------------------------------------------------- # -# batch_cost_is_final -# --------------------------------------------------------------------------- # - - -def _retrieved_batch( - status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None -) -> LiteLLMBatch: - return LiteLLMBatch( - id="batch_abc", - completion_window="24h", - created_at=1, - endpoint="/v1/chat/completions", - input_file_id="file-in", - object="batch", - status="validating", - output_file_id=output_file_id, - request_counts=counts, - ).model_copy(update={"status": status}) - - -class TestBatchCostIsFinal: - """Every retrieve of one batch writes the same spend row, so the first retrieve - that prices it decides the row for good. A poll before the output exists must - therefore not count as final: pricing it recorded $0 and pinned it (LIT-7048).""" - - @pytest.mark.parametrize("status", ["validating", "in_progress", "finalizing", "cancelling"]) - def test_in_flight_batch_is_not_final(self, status): - assert bu.batch_cost_is_final(_retrieved_batch(status)) is False - - @pytest.mark.parametrize("status", ["completed", "complete"]) - def test_completed_with_output_is_final(self, status): - assert bu.batch_cost_is_final(_retrieved_batch(status, output_file_id="file-out")) is True - - def test_completed_without_output_and_unknown_counts_is_not_final(self): - assert bu.batch_cost_is_final(_retrieved_batch("completed")) is False - - def test_completed_without_output_and_zero_counts_is_not_final(self): - counts = BatchRequestCounts(total=0, completed=0, failed=0) - assert bu.batch_cost_is_final(_retrieved_batch("completed", counts=counts)) is False - - def test_completed_without_output_but_successful_lines_is_not_final(self): - counts = BatchRequestCounts(total=2, completed=2, failed=0) - assert bu.batch_cost_is_final(_retrieved_batch("completed", counts=counts)) is False - - @pytest.mark.parametrize("status", ["completed", "complete"]) - def test_completed_without_output_and_every_line_failed_is_final(self, status): - counts = BatchRequestCounts(total=2, completed=0, failed=2) - assert bu.batch_cost_is_final(_retrieved_batch(status, counts=counts)) is True - - @pytest.mark.parametrize("status", ["failed", "expired", "cancelled"]) - def test_other_terminal_statuses_are_final(self, status): - assert bu.batch_cost_is_final(_retrieved_batch(status)) is True - - def _ocr_row(pages_processed, annotation_pages=None, model="mistral-ocr-latest"): usage_info = {"pages_processed": pages_processed, "doc_size_bytes": 4096} if annotation_pages is not None: @@ -2766,9 +2710,9 @@ def test_aggregate_batch_cost_uses_custom_model_info(): assert result.cost == pytest.approx(expected), f"Expected total cost {expected}, got {result.cost}" @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -@pytest.mark.parametrize("data_residency", ["eu", "us"]) +@pytest.mark.parametrize("data_residency", ["eu", "us", "fedramp"]) def test_batch_cost_calculator_applies_data_residency_uplift(data_residency, monkeypatch): - """batch_cost_calculator should apply the regional uplift multiplier when + """batch_cost_calculator should apply the regional or FedRAMP uplift multiplier when data_residency is set and the model carries a configured multiplier.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") prev_model_cost = litellm.model_cost @@ -2965,7 +2909,7 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos from unittest.mock import AsyncMock, patch from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import CallTypes, LiteLLMBatch + from litellm.types.utils import CallTypes # Mock batch result with completed status mock_batch = LiteLLMBatch( @@ -3060,7 +3004,6 @@ async def test_handle_completed_batch_computes_real_cost_from_output_file( wrong token sum, or mispriced model fails this test. """ from litellm.batches.batch_utils import handle_completed_batch - from litellm.types.utils import LiteLLMBatch batch = LiteLLMBatch( id="batch-real-cost-123", @@ -3105,7 +3048,7 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): from unittest.mock import AsyncMock, patch from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import CallTypes, LiteLLMBatch + from litellm.types.utils import CallTypes # Mock batch result with completed status mock_batch = LiteLLMBatch( @@ -3187,7 +3130,7 @@ async def test_batch_retrieve_explicit_cost_split_sets_cost_breakdown(): cost_breakdown carries real input/output costs; without it the UI's Cost Breakdown card renders blank for every batch. Regression for the split being dropped.""" from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import CallTypes, LiteLLMBatch + from litellm.types.utils import CallTypes mock_batch = LiteLLMBatch( id="batch-breakdown-1", @@ -3243,7 +3186,7 @@ async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batc from unittest.mock import AsyncMock, patch from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import CallTypes, LiteLLMBatch, SpecialEnums + from litellm.types.utils import CallTypes, SpecialEnums # Create a proper unified file ID by encoding the correct prefix unified_id_str = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:test_file_789;unified_id:batch-789" @@ -3322,7 +3265,7 @@ async def test_batch_retrieve_cost_tracking_with_partial_explicit_data(): from unittest.mock import AsyncMock, patch from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import CallTypes, LiteLLMBatch + from litellm.types.utils import CallTypes # Mock batch result with completed status mock_batch = LiteLLMBatch( diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 8b22395f462..5ffe6558dc3 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -858,7 +858,7 @@ async def test_batch_logging_azure_credentials_regression(): "timeout": 600, } - batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n' + batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4.1", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n' print("\n1. Testing credential extraction...") diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9e40e128952..19aefb430c7 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1,9 +1,13 @@ +import asyncio import copy import datetime import json import os +import queue import unittest -from collections.abc import Mapping, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args from unittest.mock import ANY, MagicMock, Mock, patch @@ -24,15 +28,15 @@ import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) -from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, StandardLoggingPayload, TextCompletionResponse if TYPE_CHECKING: from openai.types.responses import ResponseOutputItem from openai.types.responses.response_reasoning_item import ResponseReasoningItem - from litellm.types.llms.openai import ResponsesAPIResponse - from litellm.types.utils import ModelResponse - def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_search() -> None: response_body: Final = { @@ -191,6 +195,368 @@ def test_auto_summary_sends_the_level_the_target_accepts( assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary} +@dataclass(frozen=True, slots=True) +class _LoggedSuccess: + call_type: str + payload: StandardLoggingPayload + response_obj: object + complete_streaming_response: object + + +class _LoggedCallTypes(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_by_caller: Final = MappingProxyType( + { + "completion": queue.SimpleQueue[_LoggedSuccess](), + "acompletion": queue.SimpleQueue[_LoggedSuccess](), + } + ) + + @staticmethod + def _logged(kwargs: Mapping[str, object], response_obj: object, streaming_key: str) -> _LoggedSuccess: + return _LoggedSuccess( + call_type=str(kwargs["call_type"]), + payload=cast(StandardLoggingPayload, kwargs["standard_logging_object"]), + response_obj=response_obj, + complete_streaming_response=kwargs.get(streaming_key), + ) + + def log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["completion"].put(self._logged(kwargs, response_obj, "complete_streaming_response")) + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["acompletion"].put( + self._logged(kwargs, response_obj, "async_complete_streaming_response") + ) + + +_GET_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + +_BRIDGED_PROVIDERS: Final = pytest.mark.parametrize( + ("model", "url", "provider_kwargs"), + [ + ("openai/gpt-5.3-codex", "https://api.openai.com/v1/responses", {}), + ( + "azure/gpt-6-sol", + "https://example.invalid/openai/responses", + {"api_base": "https://example.invalid", "api_version": "2025-04-01-preview"}, + ), + ("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"aws_region_name": "us-east-1"}), + ], + ids=["openai-responses-only", "azure-tools", "bedrock-mantle-tools"], +) + + +_PONG_MESSAGE: Final[Mapping[str, object]] = { + "type": "message", + "id": "msg_logged_call_type", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], +} + + +def _searched_responses_body(model: str) -> Mapping[str, object]: + return { + "id": "resp_logged_call_type", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": model.rsplit("/", 1)[-1], + "output": [ + { + "type": "web_search_call", + "id": "ws_logged_call_type", + "status": "completed", + "action": {"type": "search", "query": "weather"}, + }, + _PONG_MESSAGE, + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _responses_sse(response_body: Mapping[str, object]) -> str: + events: Final = ( + {"type": "response.created", "response": {**response_body, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": "msg_logged_call_type", + "output_index": 1, + "content_index": 0, + "delta": "pong", + }, + {"type": "response.completed", "response": response_body}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +def _bridged_request(model: str, provider_kwargs: Mapping[str, object], stream: bool) -> Mapping[str, object]: + return { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "tools": [_GET_WEATHER_TOOL], + "api_key": "test-key", + "cache": {"no-cache": True}, + "stream": stream, + **provider_kwargs, + } + + +async def _served_chunk_ids(caller: str, request: Mapping[str, object]) -> frozenset[str]: + if caller == "acompletion": + async_stream: Final = await litellm.acompletion(**request) + assert isinstance(async_stream, CustomStreamWrapper) + return frozenset([chunk.id async for chunk in async_stream]) + sync_stream: Final = litellm.completion(**request) + assert isinstance(sync_stream, CustomStreamWrapper) + return frozenset(chunk.id for chunk in sync_stream) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_call_is_logged_with_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=False) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock(return_value=httpx.Response(200, json=response_body)) + if caller == "acompletion": + await litellm.acompletion(**request) + else: + litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == (caller, caller) + assert isinstance(logged.response_obj, ModelResponse) + assert logged.response_obj.id == response_body["id"] + assert logged.response_obj.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_stream_is_logged_as_the_chat_completion_it_streamed( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock( + return_value=httpx.Response( + 200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"} + ) + ) + served_ids: Final = await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.response_obj is logged.complete_streaming_response + assert served_ids == {logged.complete_streaming_response.id} + assert logged.complete_streaming_response.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +def _json_message(item_id: str, document: Mapping[str, object]) -> Mapping[str, object]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": json.dumps(document), "annotations": []}], + } + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_structured_output_call_is_logged_with_the_json_document_the_client_got( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = { + **_searched_responses_body("openai/gpt-5.3-codex"), + "output": [_json_message("msg_first", {"answer": "pong"}), _json_message("msg_second", {"answer": "again"})], + } + request: Final = { + **_bridged_request("openai/gpt-5.3-codex", {}, stream=False), + "response_format": {"type": "json_object"}, + } + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = await litellm.acompletion(**request) if caller == "acompletion" else litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(served, ModelResponse) + assert isinstance(logged.response_obj, ModelResponse) + served_message: Final = served.choices[0].message + logged_message: Final = logged.response_obj.choices[0].message + assert json.loads(served_message.content) == {"answer": "pong"} + assert logged_message.content == served_message.content + assert logged_message.provider_specific_fields == served_message.provider_specific_fields + assert logged.payload["response"]["choices"][0]["message"]["content"] == served_message.content + + +async def _logged_text_completion( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> tuple[TextCompletionResponse, _LoggedSuccess, Mapping[str, object]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body("openai/gpt-5.3-codex") + request: Final = {"model": "openai/gpt-5.3-codex", "prompt": "ping", "api_key": "test-key"} + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = ( + await litellm.atext_completion(**request) if caller == "acompletion" else litellm.text_completion(**request) + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + assert isinstance(served, TextCompletionResponse) + return served, logged, response_body + + +@pytest.mark.parametrize(("caller", "call_type"), [("completion", "text_completion"), ("acompletion", "atext_completion")]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_and_priced_under_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, call_type: str +) -> None: + served, logged, response_body = await _logged_text_completion(monkeypatch, caller) + + assert (logged.call_type, logged.payload["call_type"]) == (call_type, call_type) + assert getattr(logged.response_obj, "id") == served.id == response_body["id"] + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model="openai/gpt-5.3-codex" + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_as_the_text_completion_it_served( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + served, logged, _ = await _logged_text_completion(monkeypatch, caller) + + assert isinstance(logged.response_obj, TextCompletionResponse) + assert logged.response_obj.choices[0].text == served.choices[0].text == "pong" + assert logged.payload["response"]["object"] == served.object == "text_completion" + assert logged.payload["response"]["choices"][0]["text"] == "pong" + + +def _served_message_start_id(events: Sequence[bytes]) -> object: + lines: Final = b"".join(events).decode().splitlines() + starts: Final = tuple( + json.loads(line.removeprefix("data:")) for line in lines if line.startswith("data:") and '"message_start"' in line + ) + assert len(starts) == 1 + return starts[0]["message"]["id"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"]) +@pytest.mark.asyncio +async def test_bridged_messages_call_is_logged_as_anthropic_messages_under_the_id_the_client_got( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": [_PONG_MESSAGE]} + upstream: Final = ( + httpx.Response(200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"}) + if stream + else httpx.Response(200, json=response_body) + ) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock(return_value=upstream) + served: Final = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.3-codex", + max_tokens=64, + messages=[{"role": "user", "content": "ping"}], + api_key="test-key", + stream=stream, + ) + served_id: Final = ( + _served_message_start_id([event async for event in cast(AsyncIterator[bytes], served)]) + if stream + else cast(Mapping[str, object], served)["id"] + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller["acompletion"].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == ("anthropic_messages", "anthropic_messages") + assert logged.payload["id"] == served_id + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_chat_stream_keeps_the_upstream_headers_when_its_output_cannot_be_translated( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": []} + request: Final = _bridged_request("openai/gpt-5.3-codex", {}, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + text=_responses_sse(response_body), + headers={"content-type": "text/event-stream", "x-request-id": "req_upstream"}, + ) + ) + await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.complete_streaming_response.id == response_body["id"] + assert logged.payload["hidden_params"]["additional_headers"]["llm_provider-x-request-id"] == "req_upstream" + + def test_convert_chat_completion_messages_to_responses_api_image_input(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, @@ -1490,6 +1856,51 @@ def test_transform_request_system_only_message_maps_to_system_input_item(): assert not result.get("instructions") +_JSON_SYSTEM_PROMPT: Final = "You output JSON objects." +_PLAIN_USER_ITEM: Final = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "List three colors."}], +} +_COLORS_SCHEMA: Final = { + "type": "json_schema", + "json_schema": {"name": "colors", "schema": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize( + ("response_format", "expected_input", "expected_instructions"), + [ + ( + {"type": "json_object"}, + [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": _JSON_SYSTEM_PROMPT}]}, + _PLAIN_USER_ITEM, + ], + None, + ), + (_COLORS_SCHEMA, [_PLAIN_USER_ITEM], _JSON_SYSTEM_PROMPT), + ], + ids=["json_object_keeps_system_in_input", "json_schema_keeps_system_in_instructions"], +) +def test_transform_request_json_object_keeps_system_prompt_where_the_json_word_check_reads_it( + response_format, expected_input, expected_instructions +): + result = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-5.4-mini", + messages=[ + {"role": "system", "content": _JSON_SYSTEM_PROMPT}, + {"role": "user", "content": "List three colors."}, + ], + optional_params={"response_format": response_format}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert (result["input"], result.get("instructions")) == (expected_input, expected_instructions), result + + def test_transform_request_single_char_keys_not_matched(): """Test that single-character keys are not incorrectly matched to 'metadata' or 'previous_response_id' @@ -5482,6 +5893,38 @@ def test_transform_request_strips_the_breakpoint_when_the_served_providers_row_o ] +@pytest.mark.parametrize( + ("sibling_flag", "model_info", "keeps_the_breakpoint"), + [ + (True, {"id": "flagless"}, False), + (True, {"id": "opted-out", "supports_prompt_cache_breakpoint": False}, False), + (False, {"id": "opted-in", "supports_prompt_cache_breakpoint": True}, True), + ], + ids=["flagless", "opted-out", "opted-in"], +) +def test_transform_request_reads_the_routed_deployments_own_flag_not_a_siblings_on_the_shared_key( + monkeypatch: pytest.MonkeyPatch, + sibling_flag: bool, + model_info: Mapping[str, object], + keeps_the_breakpoint: bool, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bedrock_mantle/qa-mantle-shared", + {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": sibling_flag}, + ) + + system_content: Final = _bridged_input_for_served_provider( + "qa-mantle-shared", "bedrock_mantle", model_info=model_info + )[0]["content"] + + assert system_content == [ + {**_MARKED_SYSTEM_PART, "type": "input_text"} + if keeps_the_breakpoint + else {"type": "input_text", "text": "Stable prefix"} + ] + + def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a_row_of_its_own( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ed7dd5ec146..4a07c97b709 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -216,6 +216,7 @@ def _reset_aws_auth_caches() -> None: def _flush_client_caches() -> None: litellm.in_memory_llm_clients_cache.flush_cache() image_handling_module.in_memory_cache.flush_cache() + image_handling_module.pdf_probe_verdicts.flush_cache() _reset_aws_auth_caches() @@ -251,6 +252,12 @@ def socket_tempdir(socket_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return socket_dir +@pytest.fixture +def python_router_core(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep ``Router`` on ``PythonCore``, for a test that patches a member the Rust core never reads.""" + monkeypatch.setenv("LITELLM_RUST", "0") + + @pytest.fixture(autouse=True, scope="session") def bundled_tiktoken_cache() -> None: importlib.import_module("litellm.litellm_core_utils.default_encoding") diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py new file mode 100644 index 00000000000..a40a3265600 --- /dev/null +++ b/tests/unit/cookbook/test_gpt_realtime_translate.py @@ -0,0 +1,141 @@ +import asyncio +import base64 +import json +import wave +from pathlib import Path +from types import SimpleNamespace +from typing import Final, cast + +import pytest +from websockets.asyncio.client import ClientConnection + +from cookbook import gpt_realtime_translate as translate + + +@pytest.mark.asyncio +async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.1) + audio: Final = bytes(480) + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, + {"type": "error", "error": {"message": "session closed"}}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event["type"] == "session.output_audio.delta": + await asyncio.sleep(0.03) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result == 'Realtime API error: {"message": "session closed"}' + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio + + +@pytest.mark.asyncio +async def test_long_upload_keeps_waiting_for_first_translated_audio_while_still_sending( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) + audio: Final = bytes(480) + pending: Final[asyncio.Queue[str]] = asyncio.Queue() + sender_finished: Final = asyncio.Event() + + async def speak_after_a_long_pause() -> None: + await asyncio.sleep(0.2) + await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) + await asyncio.sleep(0.02) + sender_finished.set() + await pending.put(json.dumps({"type": "session.closed"})) + + output: Final = tmp_path / "translation.wav" + speaker: Final = asyncio.create_task(speak_after_a_long_pause()) + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=pending.get)), output, sender_finished + ) + await speaker + + assert result is None + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio + + +@pytest.mark.asyncio +async def test_a_quiet_gap_after_the_sender_stops_keeps_the_rest_of_the_translation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 1.0) + first: Final = bytes(480) + second: Final = bytes(range(240)) * 2 + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(first).decode()}, + {"type": "session.output_audio.delta", "delta": base64.b64encode(second).decode()}, + {"type": "session.closed"}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event.get("delta") == base64.b64encode(second).decode(): + await asyncio.sleep(0.3) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result is None + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(480) == first + second + + +@pytest.mark.asyncio +async def test_a_stream_that_never_closes_the_session_reports_an_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) + audio: Final = bytes(480) + pending: Final[asyncio.Queue[str]] = asyncio.Queue() + await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) + sender_finished: Final = asyncio.Event() + sender_finished.set() + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=pending.get)), tmp_path / "translation.wav", sender_finished + ) + + assert result == "The translation stream went quiet for 0.05 s without closing the session" + + +@pytest.mark.asyncio +async def test_the_sender_closes_the_session_after_the_last_audio_chunk() -> None: + sent: Final[asyncio.Queue[str]] = asyncio.Queue() + finished: Final = asyncio.Event() + pcm: Final = bytes(translate.CHUNK_BYTES + 2) + + await translate.send_audio(cast(ClientConnection, SimpleNamespace(send=sent.put)), pcm, 0.0, finished) + + messages: Final = tuple(json.loads(sent.get_nowait()) for _ in range(sent.qsize())) + assert [message["type"] for message in messages] == [ + "session.input_audio_buffer.append", + "session.input_audio_buffer.append", + "session.close", + ] + assert b"".join(base64.b64decode(message["audio"]) for message in messages[:-1]) == pcm + assert finished.is_set() diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 789d59bd613..0e2813e0a0d 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,8 +6,10 @@ import copy import json import re import threading +import uuid from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from concurrent.futures import wait +from datetime import datetime from functools import partial from types import MappingProxyType from typing import Final, TypeAlias @@ -132,6 +134,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( "https://openrouter.ai/api/alpha/decisions", "typesafe/jev-1.13", ), + ("empiriolabs", "empiriolabs/aplomb-1", "https://api.empiriolabs.ai/v1/decisions", "aplomb-1"), ) _OPENAI_RESPONSE: Final[Mapping[str, object]] = { @@ -299,6 +302,43 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( assert response.answers["sentiment"].choice == "positive" +@pytest.mark.asyncio +async def test_least_busy_frees_a_decisions_deployment_once_a_proxy_logged_call_finishes( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("jev-a", "jev-b") + ], + routing_strategy="least-busy", + ) + + async def call_through_proxy_logging() -> object: + logging_obj, _ = litellm.utils.function_setup( + original_function="adecisions", + rules_obj=litellm.utils.Rules(), + start_time=datetime(2026, 1, 1), + model="jev", + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.adecisions( + model="jev", + input="Is this a defect?", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + litellm_logging_obj=logging_obj, + ) + await _drain_logging_worker() + return response._hidden_params["model_id"] + + assert (await call_through_proxy_logging(), await call_through_proxy_logging()) == ("jev-a", "jev-a") + + def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) @@ -1549,6 +1589,7 @@ _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset( LlmProviders.LM_STUDIO.value, LlmProviders.OLLAMA.value, LlmProviders.OLLAMA_CHAT.value, + LlmProviders.EMPIRIOLABS.value, } ) _DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple( @@ -3449,6 +3490,147 @@ async def test_fireworks_ai_bills_cached_input_tokens_at_the_cache_read_rate( assert recording_logger.standard_logging_object["response_cost"] < 300 * float(prices["input_cost_per_token"]) +_EMPIRIOLABS_MODEL: Final = "empiriolabs/aplomb-1" +_EMPIRIOLABS_DEFAULT_URL: Final = "https://api.empiriolabs.ai/v1/decisions" +_EMPIRIOLABS_STATE: Final = "I was charged twice for order 4411 and want a refund today." +_EMPIRIOLABS_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( + {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}} +) +_EMPIRIOLABS_RESPONSE: Final[Mapping[str, object]] = { + "model": "aplomb-1", + "answers": {"is_refund": {"type": "noul", "noul": 0.97, "confidence": 0.94}}, + "usage": {"input_tokens": 24, "output_tokens": 0}, +} + + +@pytest.mark.parametrize( + ("api_base", "environment_api_base", "url"), + ( + (None, None, _EMPIRIOLABS_DEFAULT_URL), + (None, "https://environment-gateway.example/v1", "https://environment-gateway.example/v1/decisions"), + ( + "https://gateway.example/v1/", + "https://environment-gateway.example/v1", + "https://gateway.example/v1/decisions", + ), + ("https://gateway.example", None, "https://gateway.example/v1/decisions"), + ), + ids=("default_base", "environment_base", "explicit_base_over_environment", "explicit_base_without_v1"), +) +def test_empiriolabs_posts_the_systemone_body_to_v1_decisions_on_the_resolved_base( + api_base: str | None, + environment_api_base: str | None, + url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + if environment_api_base is not None: + monkeypatch.setenv("EMPIRIOLABS_API_BASE", environment_api_base) + route: Final = respx_mock.post(url).respond(json=_EMPIRIOLABS_RESPONSE) + + response: Final = litellm.systemone( + model=_EMPIRIOLABS_MODEL, + state=_EMPIRIOLABS_STATE, + questions=_EMPIRIOLABS_QUESTIONS, + api_base=api_base, + api_key="caller-key", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "aplomb-1", + "state": _EMPIRIOLABS_STATE, + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + } + assert response.answers["is_refund"] == NoulAnswer(type="noul", noul=0.97, confidence=0.94) + assert response._hidden_params["custom_llm_provider"] == "empiriolabs" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_key", "authorization"), + ((None, "Bearer environment-key"), ("caller-key", "Bearer caller-key")), + ids=("environment_key", "caller_key_over_environment"), +) +async def test_empiriolabs_sends_the_caller_key_or_else_the_environment_key( + api_key: str | None, + authorization: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + monkeypatch.setenv("EMPIRIOLABS_API_KEY", "environment-key") + route: Final = respx_mock.post(_EMPIRIOLABS_DEFAULT_URL).respond(json=_EMPIRIOLABS_RESPONSE) + + response: Final = await litellm.adecisions( + model=_EMPIRIOLABS_MODEL, + input=_EMPIRIOLABS_STATE, + questions=[{"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"}], + api_key=api_key, + ) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == authorization + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_refund", "probability": 0.97} + ] + + +def test_empiriolabs_without_a_key_is_refused_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("EMPIRIOLABS_API_KEY", raising=False) + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + + with pytest.raises(litellm.AuthenticationError, match="Missing API key for Decisions provider 'empiriolabs'"): + litellm.systemone(model=_EMPIRIOLABS_MODEL, state=_EMPIRIOLABS_STATE, questions=_EMPIRIOLABS_QUESTIONS) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + ("status", "error_type", "code", "exception"), + ( + (401, "authentication_error", "invalid_api_key", litellm.AuthenticationError), + (402, "insufficient_credits", "insufficient_credits", litellm.PaymentRequiredError), + (404, "not_found_error", "model_not_found", litellm.NotFoundError), + (422, "invalid_request_error", "invalid_request", litellm.BadRequestError), + (429, "rate_limit_error", "rate_limit_exceeded", litellm.RateLimitError), + (503, "service_unavailable", "model_unavailable", litellm.ServiceUnavailableError), + ), +) +def test_empiriolabs_upstream_errors_keep_their_status_and_message( + status: int, + error_type: str, + code: str, + exception: type[Exception], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + message: Final = f"upstream refused the decision with {code}" + route: Final = respx_mock.post(_EMPIRIOLABS_DEFAULT_URL).respond( + status, + json={"error": {"message": message, "type": error_type, "code": code}}, + headers={"retry-after": "1"}, + ) + + with pytest.raises(exception, match=message) as caught: + litellm.systemone( + model=_EMPIRIOLABS_MODEL, + state=_EMPIRIOLABS_STATE, + questions=_EMPIRIOLABS_QUESTIONS, + api_key="caller-key", + num_retries=0, + ) + + assert getattr(caught.value, "status_code") == status + assert route.call_count == 1 + + _BOX_QUESTION_REQUESTS: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( "decisions", diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index c037b6be80d..f4f6a127fc7 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -5,16 +5,24 @@ All sandbox dependencies are injected: a FakeSandbox stands in for the real e2b config and records how it is called. """ +import asyncio import time +from collections.abc import Mapping from typing import Final +import httpx import pytest +import respx + +import litellm from litellm.integrations.code_interpreter_interception.handler import ( _INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY, ) from litellm.integrations.code_interpreter_interception.handler import ( _SANDBOX_KEY, + _SANDBOX_LOST_RESULT_OUTPUT, + _SANDBOX_UNAVAILABLE_OUTPUT, _SESSION_SCOPED_KEY, LITELLM_CODE_EXECUTION_TOOL_NAME, CodeInterpreterInterceptionLogger, @@ -23,6 +31,7 @@ from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, CODE_INTERPRETER_STREAM_OPTIONS_KEY, + AgenticLoopPlan, NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, is_interception_internal_key, ) @@ -831,11 +840,7 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels(): @pytest.mark.asyncio -async def test_build_plan_deletes_sandbox_when_execution_raises(): - """If sandbox execution raises before a plan is built (e.g. E2B aborts - output over its cap), the cached sandbox must be deleted before re-raising, - otherwise a caller can leak paid containers until the prune TTL.""" - +async def test_a_failed_code_run_still_deletes_its_sandbox_once_the_loop_ends(): class RaisingSandbox(FakeSandbox): async def arun_code(self, *, container, code, **kwargs): raise ValueError("output exceeded cap") @@ -843,13 +848,11 @@ async def test_build_plan_deletes_sandbox_when_execution_raises(): sandbox = RaisingSandbox() logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) - with pytest.raises(ValueError, match="exceeded cap"): - await _build_plan(logger, sandbox, call_id="k1") + plan = await _build_plan(logger, sandbox, call_id="k1") + await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={}) - assert len(sandbox.create_calls) == 1, "the sandbox must have been created" - assert len(sandbox.delete_calls) == 1, ( - "a build failure must delete the cached sandbox so it does not keep running and billing" - ) + assert len(sandbox.create_calls) == 1 + assert len(sandbox.delete_calls) == 1 assert "sbxkey1" not in logger._container_cache @@ -1448,3 +1451,215 @@ async def test_per_identity_cap_evicts_lru_session(): "the entry with the oldest last_accessed timestamp must be evicted first (LRU)" ) assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache" + + +class _SandboxThatCannotStart(FakeSandbox): + async def acreate_sandbox(self, **kwargs): + raise ConnectionError("sandbox provider unreachable") + + +class _SandboxThatCannotRunCode(FakeSandbox): + async def arun_code(self, *, container, code, **kwargs): + raise TimeoutError("sandbox run timed out") + + +_SANDBOX_FAILURES: Final = ( + pytest.param(None, _SANDBOX_UNAVAILABLE_OUTPUT, id="no_sandbox"), + pytest.param(_SandboxThatCannotStart(), _SANDBOX_UNAVAILABLE_OUTPUT, id="create_fails"), + pytest.param(_SandboxThatCannotRunCode(), _SANDBOX_LOST_RESULT_OUTPUT, id="run_fails"), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_responses_tool_call_with_a_fixed_failure(sandbox, expected_output): + plan: Final = await _build_plan(CodeInterpreterInterceptionLogger(sandbox_config=sandbox), sandbox) + + outputs: Final = [message for message in _iter_messages(plan) if message.get("type") == "function_call_output"] + assert plan.run_agentic_loop is True + assert [(output["call_id"], output["output"]) for output in outputs] == [("c1", expected_output)] + assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_chat_tool_call_with_a_fixed_failure(sandbox, expected_output): + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + plan: Final = await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + {"id": "call_1", "name": LITELLM_CODE_EXECUTION_TOOL_NAME, "arguments": '{"code":"print(40 + 2)"}'} + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id="k1"), + stream=False, + kwargs={ + "litellm_call_id": "k1", + _ACTIVE_KEY: True, + _SANDBOX_KEY: "sbxkey1", + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + }, + ) + + tool_messages: Final = [message for message in _iter_messages(plan) if message.get("role") == "tool"] + assert plan.run_agentic_loop is True + assert [(message["tool_call_id"], message["content"]) for message in tool_messages] == [ + ("call_1", expected_output) + ] + assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"] + + +_SESSION_PLAN_SURFACES: Final = ( + pytest.param({}, FakeResponse(output=[_function_call_item()]), id="responses"), + pytest.param( + {_ACTIVE_KEY: True, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE}, + {"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, + id="chat", + ), +) + + +async def _build_session_plan( + logger: CodeInterpreterInterceptionLogger, + request_id: str, + surface_kwargs: Mapping[str, object] | None = None, + response: object | None = None, +) -> AgenticLoopPlan: + return await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_1", + "call_id": "c1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"print(1)"}', + } + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response=response or FakeResponse(output=[_function_call_item()]), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id=request_id), + stream=False, + kwargs={ + "litellm_call_id": request_id, + _SANDBOX_KEY: "conv-1", + _SESSION_SCOPED_KEY: True, + **(surface_kwargs or {}), + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES) +async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request( + surface_kwargs: Mapping[str, object], response: object +) -> None: + sandbox: Final = _SandboxThatCannotRunCode() + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + await _build_session_plan(logger, "req1", surface_kwargs, response) + await _build_session_plan(logger, "req2", surface_kwargs, response) + + assert len(sandbox.create_calls) == 2 + assert len(sandbox.delete_calls) == 2 + + +class _SandboxWhoseFirstContainerDies(FakeSandbox): + def __init__(self) -> None: + super().__init__() + self.handles: list[FakeHandle] = [] + self.first_run_started = asyncio.Event() + self.first_run_may_finish = asyncio.Event() + + async def acreate_sandbox(self, **kwargs: object) -> FakeHandle: + self.create_calls.append(kwargs) + handle: Final = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}") + self.handles.append(handle) + return handle + + async def arun_code(self, *, container: FakeHandle, code: str, **kwargs: object) -> CodeExecutionResult: + self.run_calls.append({"container": container, "code": code}) + if len(self.run_calls) == 1: + self.first_run_started.set() + await self.first_run_may_finish.wait() + if container is self.handles[0]: + raise TimeoutError("sandbox expired") + return CodeExecutionResult(stdout=self.stdout) + + +@pytest.mark.asyncio +async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement() -> None: + sandbox: Final = _SandboxWhoseFirstContainerDies() + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + slow_request: Final = asyncio.create_task(_build_session_plan(logger, "req-slow")) + await sandbox.first_run_started.wait() + await _build_session_plan(logger, "req-fails-fast") + await _build_session_plan(logger, "req-gets-replacement") + sandbox.first_run_may_finish.set() + await slow_request + await _build_session_plan(logger, "req-after") + + assert [call["container"] for call in sandbox.delete_calls] == [sandbox.handles[0]] + assert [call["container"] for call in sandbox.run_calls[-2:]] == [sandbox.handles[1], sandbox.handles[1]] + + +def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]: + return { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + +@pytest.mark.asyncio +async def test_a_responses_request_bridged_to_claude_runs_the_code_and_ends_without_the_internal_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + sandbox: Final = FakeSandbox(stdout="42") + monkeypatch.setattr(litellm, "callbacks", [CodeInterpreterInterceptionLogger(sandbox_config=sandbox)]) + provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=[ + httpx.Response( + 200, + json=_anthropic_message( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "input": {"code": "print(40 + 2)"}, + } + ], + "tool_use", + ), + ), + httpx.Response(200, json=_anthropic_message([{"type": "text", "text": "The answer is 42."}], "end_turn")), + ] + ) + + response: Final = await litellm.aresponses( + model="anthropic/claude-sonnet-5-5", + input="compute 40 + 2", + api_key="sk-test", + tools=[{"type": "code_interpreter", "container": {"type": "auto"}}], + ) + + assert provider.call_count == 2 + assert [run["code"] for run in sandbox.run_calls] == ["print(40 + 2)"] + assert not [item for item in response.output if getattr(item, "type", None) == "function_call"] diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 0a6f37e4c15..766038d5c4f 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -8,14 +8,18 @@ import subprocess import sys import textwrap from collections.abc import Mapping +from types import MappingProxyType from typing import Final, List, Optional, Tuple, cast from unittest.mock import MagicMock, patch import pytest +import respx +from httpx import Response from openai.types.chat import ChatCompletionMessage from pydantic import BaseModel, ConfigDict import litellm +from litellm import Router from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( CARRY_UNMATCHED_MESSAGE_POINTS, @@ -1397,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages _SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"} +_CLIENT_MARKED_CONVERSATION: Final = ( + {"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second"}, +) + + +def _hook_result( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> tuple[list[AllMessageValues], Mapping[str, object]]: + _, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))), + non_default_params={"cache_control_injection_points": list(points), **params}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return processed, returned_params + + +def _hook_output( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> list[AllMessageValues]: + return _hook_result(model, points, **params)[0] + + +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude(): + request: Final = AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=_hook_output("anthropic/claude-sonnet-5-5"), + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + assert _marks_in(request["system"]) == 0 + assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1] + assert "prompt_cache_breakpoint" not in json.dumps(request) + + +@pytest.mark.parametrize( + "sibling_model_infos", + [ + pytest.param((), id="one-deployment"), + pytest.param( + ({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},), + id="beside-a-deployment-of-the-same-profile-with-a-base-model", + ), + ], +) +@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"]) +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile( + local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str +): + from litellm import Router + + model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + Router( + model_list=[ + { + "model_name": f"claude-profile-{position}", + "litellm_params": {"model": model, "aws_region_name": "us-east-1"}, + "model_info": model_info, + } + for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos)) + ] + ) + processed, params = _hook_result(model) + + assert "prompt_cache_breakpoint" not in json.dumps(processed) + assert "prompt_cache_options" not in params + assert [_marks_in(message) for message in processed] == [0, 0, 0, 1] + + +def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude(): + processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},)) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + +@pytest.mark.parametrize( + ("model", "params"), + [ + pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"), + pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"), + pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"), + pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"), + ], +) +def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them( + model: str, params: Mapping[str, object] +): + processed: Final = _hook_output(model, **params) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + @pytest.mark.parametrize( ("model", "custom_llm_provider"), [ @@ -4961,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored: assert kwargs == {"cache_control_injection_points": value} +def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times(): + def marked_parts(prefix: str, count: int) -> list[dict]: + return [ + {"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + messages: Final = [ + {"role": "user", "content": marked_parts("part", 4)}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)}, + ] + sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points) + + capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0 + ) + + assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None] + assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3 + assert sent_breakpoints.call_count < 3 * 20 + + def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): @@ -5055,6 +5189,635 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi ] +_SHARED_FOUNDRY_MODEL: Final = "azure_ai/qa-shared-gpt-deployment" +_SIBLING_BREAKPOINT_FLAGS: Final = MappingProxyType( + { + "opted-in": MappingProxyType({"supports_prompt_cache_breakpoint": True}), + "opted-out": MappingProxyType({"supports_prompt_cache_breakpoint": False}), + } +) +_FOUNDRY_CHAT_ROUTE: Final = ( + "/models/chat/completions", + MappingProxyType( + { + "id": "chatcmpl-azure-ai", + "object": "chat.completion", + "created": 1700000000, + "model": "qa-shared-gpt-deployment", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + ), +) +_FOUNDRY_RESPONSES_ROUTE: Final = ( + "/openai/v1/responses", + MappingProxyType( + { + "id": "resp_azure_ai", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "qa-shared-gpt-deployment", + "output": [ + { + "type": "message", + "id": "msg_azure_ai", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + } + ), +) + + +def _foundry_siblings_sharing_one_model_string( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + upstream: tuple[str, Mapping[str, object]], +) -> tuple[Router, Mapping[str, respx.Route]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + path, reply = upstream + routes: Final = MappingProxyType( + { + host: respx_mock.post(host=f"{host}.services.ai.azure.com", path=path).mock( + return_value=Response(200, json=dict(reply)) + ) + for host in registration_order + } + ) + router: Final = Router( + model_list=[ + { + "model_name": host, + "litellm_params": { + "model": _SHARED_FOUNDRY_MODEL, + "api_base": f"https://{host}.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})}, + } + for host in registration_order + ], + num_retries=0, + ) + return router, routes + + +async def _send_unmarked_system_prompt(router: Router, host: str, entrypoint: str) -> None: + chat_messages: Final = [{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + match entrypoint: + case "completion": + router.completion(model=host, messages=chat_messages) + case "acompletion": + await router.acompletion(model=host, messages=chat_messages) + case "aanthropic_messages": + await router.aanthropic_messages( + model=host, system="stable", messages=[{"role": "user", "content": "hi"}], max_tokens=16 + ) + case "aresponses": + await router.aresponses(model=host, input=chat_messages) + + +@pytest.mark.parametrize( + ("entrypoint", "upstream"), + [ + ("completion", _FOUNDRY_CHAT_ROUTE), + ("acompletion", _FOUNDRY_CHAT_ROUTE), + ("aanthropic_messages", _FOUNDRY_CHAT_ROUTE), + ("aresponses", _FOUNDRY_RESPONSES_ROUTE), + ], + ids=["completion", "acompletion", "aanthropic_messages", "aresponses"], +) +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +async def test_foundry_deployment_reads_only_its_own_prompt_cache_flag_not_a_siblings_sharing_its_model_string( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + entrypoint: str, + upstream: tuple[str, Mapping[str, object]], +) -> None: + router, routes = _foundry_siblings_sharing_one_model_string(respx_mock, monkeypatch, registration_order, upstream) + + for host in registration_order: + await _send_unmarked_system_prompt(router, host, entrypoint) + + def sent_prompt_caching(host: str) -> tuple[object, bool]: + body: Final = routes[host].calls.last.request.content + return json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body + + assert {host: sent_prompt_caching(host) for host in registration_order} == { + "opted-in": ({"mode": "implicit"}, True), + "opted-out": (None, False), + "unflagged": (None, False), + } + + +@pytest.mark.parametrize( + ("model", "sibling_flag", "expected"), + [("qa-openai-shared", True, False), ("gpt-99.0-qa-shared", False, True)], + ids=["sibling-opt-in-does-not-turn-it-on", "sibling-opt-out-does-not-turn-it-off"], +) +def test_openai_deployment_reads_the_catalog_not_a_siblings_flag_on_the_shared_key( + monkeypatch: pytest.MonkeyPatch, model: str, sibling_flag: bool, expected: bool +) -> None: + sibling_entry: Final = { + "litellm_provider": "openai", + "mode": "chat", + "supports_prompt_cache_breakpoint": sibling_flag, + } + monkeypatch.setitem(litellm.model_cost, model, sibling_entry) + + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openai", "https://api.openai.com/v1", model_info={"id": "flagless-deployment"} + ) + is expected + ) + + +@pytest.mark.parametrize( + ("model", "declared", "api_base", "expected"), + [ + ("qa-openai-shared", True, "https://api.openai.com/v1", True), + ("qa-openai-shared", True, "https://compatible-host.example.com/v1", False), + ("gpt-99.0-qa-shared", False, "https://api.openai.com/v1", False), + ], + ids=["opt-in-on-openai", "opt-in-on-a-compatible-host-stays-off", "opt-out-beats-the-version-rule"], +) +def test_openai_deployment_declared_flag_answers_support_while_the_host_still_decides( + model: str, declared: bool, api_base: str, expected: bool +) -> None: + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openai", api_base, model_info={"supports_prompt_cache_breakpoint": declared} + ) + is expected + ) + + +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +async def test_foundry_responses_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...] +) -> None: + router, routes = _foundry_siblings_sharing_one_model_string( + respx_mock, monkeypatch, registration_order, _FOUNDRY_RESPONSES_ROUTE + ) + client_marked_system_prompt: Final = { + "role": "system", + "content": [{"type": "input_text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + + for host in registration_order: + await router.aresponses( + model=host, input=[client_marked_system_prompt, {"role": "user", "content": "hi"}], drop_params=True + ) + + assert { + host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order + } == { + "opted-in": True, + "opted-out": False, + "unflagged": True, + } + + +@pytest.mark.parametrize( + ("entrypoint", "part_type", "reply"), + [("acompletion", "text", _FOUNDRY_CHAT_ROUTE[1]), ("aresponses", "input_text", _FOUNDRY_RESPONSES_ROUTE[1])], + ids=["chat", "responses"], +) +@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")]) +async def test_azure_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag_not_a_siblings( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + entrypoint: str, + part_type: str, + reply: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + routes: Final = MappingProxyType( + { + host: respx_mock.post(host=f"{host}.openai.azure.com").mock(return_value=Response(200, json=dict(reply))) + for host in registration_order + } + ) + router: Final = Router( + model_list=[ + { + "model_name": host, + "litellm_params": { + "model": "azure/gpt-4.1-qa-shared", + "api_base": f"https://{host}.openai.azure.com", + "api_key": "azure-test-key", + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})}, + } + for host in registration_order + ], + num_retries=0, + ) + marked_system_prompt: Final = { + "role": "system", + "content": [{"type": part_type, "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + request: Final = [marked_system_prompt, {"role": "user", "content": "hi"}] + + for host in registration_order: + match entrypoint: + case "acompletion": + await router.acompletion(model=host, messages=request, drop_params=True) + case "aresponses": + await router.aresponses(model=host, input=request, drop_params=True) + + assert { + host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order + } == {"opted-in": True, "unflagged": False} + + +@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")]) +async def test_openai_messages_bridge_sends_a_breakpoint_by_the_deployments_own_flag_not_a_siblings( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + _, reply = _FOUNDRY_RESPONSES_ROUTE + responses_route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=Response(200, json=dict(reply)) + ) + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "openai/qa-shared-gpt-deployment", "api_key": "openai-test-key"}, + "model_info": {"id": f"{name}-id", **_SIBLING_BREAKPOINT_FLAGS.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + + async def sends_a_breakpoint(name: str) -> bool: + await router.aanthropic_messages( + model=name, + system=[{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + ) + return b"prompt_cache_breakpoint" in responses_route.calls.last.request.content + + assert {name: await sends_a_breakpoint(name) for name in registration_order} == { + "opted-in": True, + "unflagged": False, + } + + +_OPENAI_ROUTES: Final = MappingProxyType( + { + "acompletion": ("https://api.openai.com/v1/chat/completions", _FOUNDRY_CHAT_ROUTE[1]), + "aresponses": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]), + "aanthropic_messages": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]), + } +) + + +def _openai_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str) -> respx.Route: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + url, reply = _OPENAI_ROUTES[entrypoint] + return respx_mock.post(url).mock(return_value=Response(200, json=dict(reply))) + + +@pytest.mark.parametrize("declared", [True, False]) +async def test_a_wildcard_deployments_prompt_cache_flag_leaves_each_matched_model_to_the_catalog( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, declared: bool +) -> None: + chat: Final = _openai_route(respx_mock, monkeypatch, "acompletion") + router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": declared}, + } + ], + num_retries=0, + ) + + async def sends_prompt_cache_options(model: str) -> bool: + await _send_unmarked_system_prompt(router, model, "acompletion") + return b"prompt_cache_options" in chat.calls.last.request.content + + assert {model: await sends_prompt_cache_options(model) for model in ("openai/gpt-4.1-qa", "openai/gpt-99.0-qa")} == { + "openai/gpt-4.1-qa": False, + "openai/gpt-99.0-qa": True, + } + + +async def test_an_azure_ai_wildcard_deployments_breakpoint_flag_still_speaks_for_each_matched_model( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + path, reply = _FOUNDRY_CHAT_ROUTE + chat: Final = respx_mock.post(host="qa-wildcard.services.ai.azure.com", path=path).mock( + return_value=Response(200, json=dict(reply)) + ) + router: Final = Router( + model_list=[ + { + "model_name": "azure_ai/*", + "litellm_params": { + "model": "azure_ai/*", + "api_base": "https://qa-wildcard.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + + await _send_unmarked_system_prompt(router, "azure_ai/qa-unlisted-alias", "acompletion") + + body: Final = chat.calls.last.request.content + assert (json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body) == ( + {"mode": "implicit"}, + True, + ) + + +@pytest.mark.parametrize("entrypoint", ["acompletion", "aresponses", "aanthropic_messages"]) +@pytest.mark.parametrize("declared_in", ["litellm_params", "model_info"]) +async def test_a_base_model_that_takes_breakpoints_leaves_an_openai_deployments_request_as_it_was( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, declared_in: str +) -> None: + upstream: Final = _openai_route(respx_mock, monkeypatch, entrypoint) + + def deployment(name: str, base_model: Mapping[str, str]) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": { + "model": "openai/gpt-4.1-qa", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + **(base_model if declared_in == "litellm_params" else {}), + }, + "model_info": {"id": f"{name}-id", **(base_model if declared_in == "model_info" else {})}, + } + + router: Final = Router( + model_list=[deployment("plain", {}), deployment("priced-as-breakpoint-model", {"base_model": "gpt-99.0-qa"})], + num_retries=0, + ) + + async def sent_body(name: str) -> object: + await _send_unmarked_system_prompt(router, name, entrypoint) + return json.loads(upstream.calls.last.request.content) + + assert await sent_body("priced-as-breakpoint-model") == await sent_body("plain") + + +def test_a_routed_call_honors_a_register_model_breakpoint_opt_out_made_under_the_provider_prefixed_name() -> None: + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "openai" and entry.get("supports_prompt_cache_breakpoint") is True + ) + litellm.register_model({f"openai/{model}": {"supports_prompt_cache_breakpoint": False}}) + + assert supports_openai_prompt_cache_breakpoint(model, "openai", {"id": "plain-id"}) is False + + +_APPLICATION_INFERENCE_PROFILE: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/qa" + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected"), + [ + ("us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("converse/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("invoke/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("converse_like/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("openai/gpt-oss-120b", "bedrock", False), + ("moonshotai.kimi-k3", "bedrock", False), + (_APPLICATION_INFERENCE_PROFILE, "bedrock", False), + ("claude-sonnet-5-5", "anthropic", False), + ("claude-sonnet-5-5", "azure_ai", False), + ("claude-sonnet-5-5", "vertex_ai", False), + ("gemini-3-pro", "gemini", False), + ("qa-custom-model", "qa-custom-handler", False), + ("qa-shared-gpt-deployment", "azure_ai", True), + ("my-custom-deployment", "azure", True), + ("qa-mantle-model", "bedrock_mantle", True), + ("anthropic/qa-gateway-claude", "openrouter", False), + ("qa-downstream-claude", "litellm_proxy", False), + ("qa-self-hosted-model", "hosted_vllm", False), + ], +) +def test_a_declared_breakpoint_flag_picks_openais_dialect_only_where_the_request_goes_out_openai_shaped( + model: str, custom_llm_provider: str, expected: bool +) -> None: + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, model_info={"supports_prompt_cache_breakpoint": True} + ) + is expected + ) + + +def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_for_a_model_the_cost_map_lists_there() -> None: + litellm.register_model({"openrouter/qa-listed-gpt": {"litellm_provider": "openrouter", "mode": "chat"}}) + + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openrouter", model_info={"supports_prompt_cache_breakpoint": True} + ) + for model in ("qa-listed-gpt", "qa-unlisted-gpt") + ] == [True, False] + + +def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_when_its_model_info_lists_that_provider() -> None: + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + "qa-self-hosted-gpt", + "hosted_vllm", + model_info={"supports_prompt_cache_breakpoint": True, "litellm_provider": listed_provider}, + ) + for listed_provider in ("hosted_vllm", "openrouter") + ] == [True, False] + +def test_a_custom_handler_provider_declaring_the_flag_keeps_the_anthropic_mark() -> None: + litellm.register_model( + {"qa-custom-handler/qa-model": {"litellm_provider": "qa-custom-handler", "mode": "chat"}} + ) + + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + "qa-model", "qa-custom-handler", model_info={"supports_prompt_cache_breakpoint": True} + ) + is False + ) + + +def test_a_routed_deployment_without_its_own_flag_reads_a_register_model_opt_in_but_not_a_siblings() -> None: + litellm.register_model( + {"azure_ai/qa-registered-gpt": {"litellm_provider": "azure_ai", "supports_prompt_cache_breakpoint": True}} + ) + Router( + model_list=[ + { + "model_name": "qa-opted-in-sibling", + "litellm_params": {"model": "azure_ai/qa-sibling-gpt", "api_base": "https://qa.services.ai.azure.com"}, + "model_info": {"id": "qa-opted-in-sibling-id", "supports_prompt_cache_breakpoint": True}, + } + ] + ) + + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "azure_ai", model_info={"id": "qa-flagless-id"} + ) + for model in ("qa-registered-gpt", "qa-sibling-gpt") + ] == [True, False] + + +@pytest.mark.parametrize( + ("entrypoint", "injection_points"), + [ + ("completion", [{"location": "message", "role": "system"}]), + ("acompletion", [{"location": "message", "role": "system"}]), + ("aresponses", [{"location": "message", "role": "system"}]), + ("aresponses-client-marked", None), + ], +) +async def test_an_azure_custom_deployment_declaring_the_breakpoint_flag_gets_openais_breakpoint( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + entrypoint: str, + injection_points: list[dict[str, str]] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + azure: Final = respx_mock.post(host="qa-azure.openai.azure.com").mock( + side_effect=lambda request: Response( + 200, + json=dict( + _FOUNDRY_RESPONSES_ROUTE[1] if request.url.path.endswith("/responses") else _FOUNDRY_CHAT_ROUTE[1] + ), + ) + ) + deployment_params: Final = { + "model": "azure/my-custom-deployment", + "api_base": "https://qa-azure.openai.azure.com", + "api_key": "azure-test-key", + "api_version": "2025-04-01-preview", + } + router: Final = Router( + model_list=[ + { + "model_name": "custom-azure", + "litellm_params": ( + deployment_params + if injection_points is None + else {**deployment_params, "cache_control_injection_points": injection_points} + ), + "model_info": {"id": "custom-azure-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + client_marked_system_prompt: Final = { + "role": "system", + "content": [{"type": "input_text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + } + + match entrypoint: + case "aresponses-client-marked": + await router.aresponses( + model="custom-azure", input=[client_marked_system_prompt, {"role": "user", "content": "hi"}] + ) + case _: + await _send_unmarked_system_prompt(router, "custom-azure", entrypoint) + + body: Final = azure.calls.last.request.content + assert (b"prompt_cache_breakpoint" in body, b"cache_control" in body) == (True, False) + + +_BEDROCK_CONVERSE_REPLY: Final = MappingProxyType( + { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6}, + } +) +_BEDROCK_INVOKE_REPLY: Final = MappingProxyType( + { + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, + } +) + + +@pytest.mark.parametrize( + ("entrypoint", "expected_system"), + [ + ("completion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]), + ("acompletion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]), + ("aanthropic_messages", [{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}]), + ], +) +async def test_a_bedrock_claude_deployment_declaring_the_breakpoint_flag_keeps_bedrocks_own_cache_marks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, expected_system: list[object] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + bedrock: Final = respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com").mock( + side_effect=lambda request: Response( + 200, + json=dict(_BEDROCK_INVOKE_REPLY if request.url.path.endswith("/invoke") else _BEDROCK_CONVERSE_REPLY), + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "claude-on-bedrock", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-5-5", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "claude-on-bedrock-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + + await _send_unmarked_system_prompt(router, "claude-on-bedrock", entrypoint) + + body: Final = bedrock.calls.last.request.content + assert (json.loads(body)["system"], b"prompt_cache" in body) == (expected_system, False) + + def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]: return [ cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}), diff --git a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py index 453fd28636d..3f3e2e0d465 100644 --- a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py @@ -512,6 +512,20 @@ def _provider_auth_failure_kwargs() -> dict[str, object]: } +def _deployment_entra_rejection_kwargs() -> dict[str, object]: + from litellm.llms.azure.common_utils import EntraRejectionError + + reason: Final = "Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided." + try: + raise litellm.AuthenticationError(message=reason, llm_provider="azure", model="gpt-4.1") from ( + EntraRejectionError(status_code=401, message=reason) + ) + except litellm.AuthenticationError as raised: + return { + key: value for key, value in _provider_auth_failure_kwargs().items() if key not in CALLED_THE_PROVIDER + } | {"exception": raised} + + def _metric_values(name: str, model_id: str) -> list[float]: return [ sample.value @@ -541,6 +555,20 @@ class TestProviderAuthFailures: assert _metric_values("litellm_deployment_failure_responses_total", "deployment-1") == [1.0] assert _metric_values("litellm_deployment_state", "deployment-1") == [1.0] + @pytest.mark.asyncio + async def test_log_failure_event_records_entra_rejecting_the_deployment_credential_before_the_call( + self, prometheus_logger + ): + await prometheus_logger.async_log_failure_event( + kwargs=_deployment_entra_rejection_kwargs(), + response_obj=None, + start_time=None, + end_time=None, + ) + + assert _metric_values("litellm_llm_api_failed_requests_metric_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_failure_responses_total", "deployment-1") == [1.0] + def test_deployment_failure_metrics_record_a_provider_401(self, prometheus_logger): prometheus_logger.set_llm_deployment_failure_metrics(_provider_auth_failure_kwargs()) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 351ca9beb4d..e5ef356b0f3 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -12,6 +12,7 @@ config.yaml through to the settings the loop actually reads. """ import json +import time from unittest.mock import MagicMock, patch import pytest @@ -24,6 +25,8 @@ from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, @@ -31,6 +34,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.secret_managers.main import get_secret +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, @@ -38,6 +42,7 @@ from litellm.types.integrations.custom_logger import ( ) INTERNAL_TOOL_NAME = "litellm_web_search" +CODE_EXECUTION_TOOL_NAME = "litellm_code_execution" @pytest.fixture(autouse=True) @@ -99,6 +104,29 @@ def _tool_use_names(response: dict) -> list[str]: return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"] +def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + status="completed", + model="claude-sonnet-4-5", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}], + }, + *( + {"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"} + for call_id, name in function_calls + ), + ], + ) + + class _InterceptingCallback(CustomLogger): """ Stands in for the websearch interceptor: asks for another loop whenever the @@ -152,14 +180,46 @@ class _InterceptingCallback(CustomLogger): return response -def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock: +class _ResponsesInterceptingCallback(_InterceptingCallback): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + calls = [ + {"id": item.call_id, "call_id": item.call_id, "name": item.name} + for item in response.output + if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME + ] + return bool(calls), {"tool_calls": calls} + + +def _logging_obj(*callbacks: CustomLogger, converted_stream: bool = False) -> MagicMock: logging_obj = MagicMock() logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream} - logging_obj.dynamic_success_callbacks = [callback] + logging_obj.dynamic_success_callbacks = list(callbacks) logging_obj.litellm_call_id = "call-abc" return logging_obj +class _CodeExecutionGate(CustomLogger): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + calls = ( + [ + {"id": block["id"], "name": block["name"]} + for block in response["content"] + if block.get("type") == "tool_use" and block.get("name") == CODE_EXECUTION_TOOL_NAME + ] + if isinstance(response, dict) + else [ + {"call_id": item.call_id, "name": item.name} + for item in response.output + if item.type == "function_call" and item.name == CODE_EXECUTION_TOOL_NAME + ] + ) + return bool(calls), {"tool_calls": calls} + + async def _run_hooks( handler: BaseLLMHTTPHandler, callback: CustomLogger, @@ -314,6 +374,48 @@ class TestCappedLoopReturnsTerminalResponse: assert _tool_use_names(result) == ["get_weather"] assert result["stop_reason"] == "tool_use" + @pytest.mark.asyncio + async def test_every_hooks_internal_call_is_dropped_and_the_clients_survives(self): + response = _response_asking_for_another_search() + response["content"] += [ + {"id": "toolu_code_1", "type": "tool_use", "name": CODE_EXECUTION_TOOL_NAME, "input": {}}, + {"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}}, + ] + + result = await self.handler._call_agentic_completion_hooks( + response=response, + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(self.callback, _CodeExecutionGate()), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert _tool_use_names(result) == ["get_weather"] + assert self.callback.plan_calls == 0 + + @pytest.mark.asyncio + async def test_responses_surface_drops_every_hooks_internal_call_and_keeps_the_clients(self): + result = await self.handler._call_agentic_completion_hooks( + response=_responses_turn( + ("call_search", INTERNAL_TOOL_NAME), ("call_code", CODE_EXECUTION_TOOL_NAME), ("call_client", "lookup") + ), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(_ResponsesInterceptingCallback(), _CodeExecutionGate()), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + api_surface="responses", + ) + + assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_client"] + def test_only_the_refused_tool_calls_are_dropped(self): """ A block is matched on the id the rail refused, not on the tool name, so a @@ -347,6 +449,18 @@ class TestCappedLoopReturnsTerminalResponse: assert _tool_use_names(result) == [] assert result["stop_reason"] == "end_turn" + def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self): + response = _response_asking_for_another_search() + response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}} + + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=response, + tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]}, + ) + + assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"] + assert result["stop_reason"] == "end_turn" + @pytest.mark.asyncio async def test_streaming_caller_is_left_to_its_existing_behavior(self): """ @@ -365,19 +479,34 @@ class TestCappedLoopReturnsTerminalResponse: assert self.callback.plan_calls == 0 @pytest.mark.asyncio - async def test_responses_surface_is_left_to_its_existing_behavior(self): - """ - The responses surface carries a pydantic model rather than the anthropic - dict this finalizer rewrites, so it keeps raising instead of being handed - a response that was never actually finalized. - """ - with pytest.raises(AgenticLoopSafetyError): - await _run_hooks( - self.handler, - self.callback, - kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, - api_surface="responses", - ) + async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self): + response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup")) + callback = _ResponsesInterceptingCallback() + + result = await _run_hooks( + self.handler, + callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=response, + api_surface="responses", + ) + + assert isinstance(result, ResponsesAPIResponse) + assert [(item.type, getattr(item, "name", None)) for item in result.output] == [ + ("message", None), + ("function_call", "lookup"), + ] + assert callback.plan_calls == 0 + + def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self): + response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME)) + + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=response, + tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]}, + ) + + assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"] @pytest.mark.asyncio async def test_non_dict_response_is_returned_untouched(self): @@ -509,6 +638,156 @@ def _response_asking_for_searches(*queries: str) -> dict: } +def _request_logging_obj(call_type: str, *callbacks: CustomLogger) -> Logging: + logging_obj = Logging( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + stream=False, + call_type=call_type, + start_time=time.time(), + litellm_call_id="call-abc", + function_id="f", + dynamic_success_callbacks=list(callbacks), + ) + logging_obj.update_environment_variables( + model="claude-sonnet-4-5", user="", optional_params={}, litellm_params={}, custom_llm_provider="anthropic" + ) + return logging_obj + + +def _assert_first_call_billed(logging_obj: Logging, *, input_tokens: int, output_tokens: int) -> None: + price = litellm.model_cost["claude-sonnet-4-5"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == input_tokens + output_tokens + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + input_tokens * price["input_cost_per_token"] + output_tokens * price["output_cost_per_token"] + ) + + +class TestFailedFollowUpCallReachesTheClient: + def setup_method(self): + self.handler = BaseLLMHTTPHandler() + self.logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"]) + + @staticmethod + async def _rate_limited_follow_up(**call_kwargs): + raise RateLimitError( + "UserByModelByMinuteUncachedInputTokens", llm_provider="anthropic", model="claude-sonnet-4-5" + ) + + @staticmethod + async def _search(query, kwargs=None, rich=None): + found = SearchResult(title="Result", url="https://example.com", snippet="A result.", date=None) + return ("Title: Result\nURL: https://example.com", SearchResponse(results=[found])) + + @pytest.mark.asyncio + @pytest.mark.parametrize("converted_stream", (False, True), ids=("non_streaming", "streaming_client")) + async def test_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch, converted_stream): + monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up) + + with ( + patch.object(self.logger, "_execute_search", side_effect=self._search), + pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"), + ): + await self.handler._call_agentic_completion_hooks( + response=_response_asking_for_searches("who won the world cup"), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]}, + logging_obj=_logging_obj(self.logger, converted_stream=converted_stream), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + @pytest.mark.asyncio + async def test_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch): + monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up) + logging_obj = _request_logging_obj("anthropic_messages", self.logger) + + with ( + patch.object(self.logger, "_execute_search", side_effect=self._search), + pytest.raises(RateLimitError), + ): + await self.handler._call_agentic_completion_hooks( + response=_response_asking_for_searches("who won the world cup"), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + _assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5) + + @pytest.mark.asyncio + async def test_a_crashed_plan_build_is_not_retried_and_bills_the_first_call(self): + logging_obj = _request_logging_obj("anthropic_messages", self.logger) + + with ( + patch.object(self.logger, "async_build_agentic_loop_plan", side_effect=ValueError("the plan crashed")), + pytest.raises(ValueError, match="the plan crashed") as raised, + ): + await self.handler._call_agentic_completion_hooks( + response=_response_asking_for_searches("who won the world cup"), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + assert is_agentic_followup_failure(raised.value) + _assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5) + + @pytest.mark.asyncio + async def test_responses_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch): + monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up) + callback = _ResponsesInterceptingCallback() + logging_obj = _request_logging_obj("aresponses", callback) + first_call = _responses_turn(("call_search", INTERNAL_TOOL_NAME)).model_copy( + update={"usage": {"input_tokens": 13, "output_tokens": 6, "total_tokens": 19}} + ) + + with pytest.raises(RateLimitError): + await self.handler._call_agentic_completion_hooks( + response=first_call, + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3}, + api_surface="responses", + ) + + _assert_first_call_billed(logging_obj, input_tokens=13, output_tokens=6) + + @pytest.mark.asyncio + async def test_responses_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch): + monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up) + callback = _ResponsesInterceptingCallback() + + with pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"): + await _run_hooks( + self.handler, + callback, + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3}, + response=_responses_turn(("call_search", INTERNAL_TOOL_NAME)), + api_surface="responses", + ) + + assert callback.plan_calls == 1 + + class TestFailedSearchEndsTheTurn: """ A search that failed used to come back to the client as an empty successful diff --git a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py index b2d5225070c..ee4c4d32478 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py @@ -371,8 +371,7 @@ class TestLoggingObjExcludedFromFollowUp: class TestFollowUpErrorScenarios: """Regression tests: the agentic loop must surface errors properly and - not silently swallow them (except at the _call_agentic_completion_hooks - level which intentionally catches to return the initial response).""" + not silently swallow them.""" @pytest.mark.asyncio async def test_followup_400_raises(self): diff --git a/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py new file mode 100644 index 00000000000..5296ea3b15a --- /dev/null +++ b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py @@ -0,0 +1,319 @@ +import datetime +import io +import math +import wave +from collections.abc import AsyncIterator, Iterator +from typing import Final + +import anyio +import anyio.lowlevel +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm import Router +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, +) +from litellm.types.utils import TranscriptionResponse + +_STARTED_AT: Final = datetime.datetime(2026, 10, 28, 12, 0, 0) # noqa: DTZ001 # callback timestamps use the legacy naive contract + + +class _FirstDeltaThenStall(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + await anyio.sleep_forever() + + +class _FirstDeltaThenFails(httpx.AsyncByteStream): + def __init__(self, failure: httpx.TransportError) -> None: + self._failure = failure + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise self._failure + + +def _transcription_router(failing_api_base: str, healthy_api_base: str) -> Router: + return Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": api_base, + "order": order, + }, + } + for order, api_base in ((1, failing_api_base), (2, healthy_api_base)) + ], + allowed_fails=0, + cooldown_time=60, + ) + + +class _RecordingLogging: + def __init__(self) -> None: + self.logged: tuple[TranscriptionResponse, ...] = () + + def success_handler( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + return None + + async def async_success_handler( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + await anyio.lowlevel.checkpoint() + self.logged = (*self.logged, result) + + def handle_sync_success_callbacks_for_async_calls( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + return None + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + *, + for_async_request: bool = False, + ) -> None: + return None + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + return None + + +@pytest.mark.asyncio +async def test_a_stream_the_client_drops_mid_transcript_is_still_logged_as_a_success() -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenStall() + ) + ) + ), + ) + stream: Final = await upstream.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.wav", b"audio"), stream=True + ) + logging_obj: Final = _RecordingLogging() + wrapped: Final = wrap_transcription_stream(stream, logging_obj, _STARTED_AT, litellm_exception_mapper({})) + + with anyio.CancelScope() as client_connection: + async for _ in wrapped: + client_connection.cancel() + + assert [result.text for result in logging_obj.logged] == ["Good"] + + +@pytest.mark.asyncio +async def test_a_deployment_whose_stream_times_out_mid_transcript_goes_into_cooldown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _transcription_router("https://stalls.example/v1", "https://healthy.example/v1") + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://stalls.example/v1/audio/transcriptions").mock( + side_effect=lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadTimeout("read timed out")), + ) + ) + upstream.post("https://healthy.example/v1/audio/transcriptions").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=b'data: {"type":"transcript.text.done","text":"Good morning"}\n\n', + ) + ) + stalled: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True) + with pytest.raises(litellm.Timeout): + async for _ in stalled: + pass + await anyio.sleep(0.1) + retried: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True) + events: Final = [event async for event in retried] + + assert [getattr(event, "text", None) for event in events] == ["Good morning"] + + +@pytest.mark.asyncio +async def test_a_stream_the_upstream_drops_mid_transcript_raises_a_litellm_error_to_the_caller() -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.RemoteProtocolError("peer closed connection")), + ) + ) + ), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError, match="peer closed connection"): + async for _ in stream: + pass + + +class _FailureHookCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[str, ...] = () + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.failures = (*self.failures, "sync_failure") + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures = (*self.failures, "async_failure") + + +@pytest.mark.asyncio +async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = _FailureHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")), + ) + ) + ), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert counter.failures == ("async_failure",) + + +class _FirstDeltaThenResetSync(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise httpx.ReadError("connection reset") + + +class _FailureCostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.costs: tuple[object, ...] = () + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + +def _silent_wav(seconds: int) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00\x00" * 16_000 * seconds) + return buffer.getvalue() + + +def _reset_mid_transcript(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")), + ) + + +def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync()) + + +def _per_second_rate(model: str) -> float: + rate: Final = litellm.model_cost[model]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +@pytest.mark.asyncio +async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe")) + + +def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = OpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)), + ) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe")) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..3964f472bd9 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -29,9 +29,11 @@ from litellm.llms.gemini.image_generation.cost_calculator import ( from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, + DataResidency, ImageObject, ImageResponse, ImageUsage, @@ -42,6 +44,62 @@ from litellm.types.utils import ( ) +def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map): + model: Final = "gpt-realtime-2.1-mini" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=400, + image_tokens=200, + cached_tokens=300, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) + cached_cost: Final = ( + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + ) + uncached_cost: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + ) + + assert breakdown.cache_read_cost == pytest.approx(cached_cost) + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + +def test_image_model_bills_cached_image_tokens_at_its_cached_image_rate(_local_model_cost_map): + model: Final = "gpt-image-2" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=200, + image_tokens=800, + cached_tokens=600, + cached_tokens_details=CachedTokensDetails(text_tokens=100, image_tokens=500), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + cached_cost: Final = 100 * rates["cache_read_input_token_cost"] + 500 * rates["cache_read_input_image_token_cost"] + uncached_cost: Final = 100 * rates["input_cost_per_token"] + 300 * rates["input_cost_per_image_token"] + + assert rates["cache_read_input_image_token_cost"] != rates["cache_read_input_token_cost"] + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -2234,6 +2292,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, "cache_hit_audio_tokens": 0, + "cached_text_tokens": 0, + "cached_audio_tokens": 0, + "cached_image_tokens": 0, + "has_cached_tokens_details": False, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -2518,6 +2580,51 @@ def test_data_residency_applies_uplift(data_residency, model, _local_model_cost_ assert regional[1] == pytest.approx(base[1] * 1.10, rel=1e-9) +def test_fedramp_data_residency_applies_configured_uplift() -> None: + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + model_info: Final[ModelInfo] = { + "key": "fedramp-fixture", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "litellm_provider": "openai", + "mode": "chat", + "supported_openai_params": None, + "regional_processing_uplift_multiplier_us": 1.5, + "fedramp_uplift_multiplier": 1.1, + } + + base: Final = generic_cost_per_token( + model="fedramp-fixture", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + fedramp: Final = generic_cost_per_token( + model="fedramp-fixture", + usage=usage, + custom_llm_provider="openai", + data_residency=DataResidency.FEDRAMP.value, + model_info=model_info, + ) + + us: Final = generic_cost_per_token( + model="fedramp-fixture", + usage=usage, + custom_llm_provider="openai", + data_residency=DataResidency.US.value, + model_info=model_info, + ) + + assert sum(base) > 0 + assert fedramp[0] == pytest.approx(base[0] * 1.1) + assert fedramp[1] == pytest.approx(base[1] * 1.1) + assert us[0] == pytest.approx(base[0] * 1.5) + assert us[1] == pytest.approx(base[1] * 1.5) + + @pytest.mark.parametrize("model", ["gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-pro", "gpt-4o", "gpt-4.1"]) def test_data_residency_no_uplift_for_pre_march_2026_models(model, _local_model_cost_map): """Models released before 2026-03-05 must not have the regional uplift.""" @@ -2658,6 +2765,22 @@ def test_vertex_uplift_invalid_multiplier_defaults_to_one(): ) +@pytest.mark.parametrize( + "model_info, data_residency", + [ + ({"fedramp_uplift_multiplier": "not-a-number"}, "fedramp"), + ({}, "fedramp"), + ({"fedramp_uplift_multiplier": 1.1}, "us"), + ({"fedramp_uplift_multiplier": 1.1}, None), + ], +) +def test_fedramp_uplift_defaults_to_one(model_info: ModelInfo, data_residency: str | None): + """A malformed or missing FedRAMP multiplier, or a non-FedRAMP host, bills base pricing and never raises.""" + from litellm.litellm_core_utils.llm_cost_calc.utils import get_fedramp_uplift_multiplier + + assert get_fedramp_uplift_multiplier(model_info, data_residency) == 1.0 + + def test_service_tier_suffixes_constant_in_sync_with_enum(): from litellm.litellm_core_utils.llm_cost_calc.utils import _SERVICE_TIER_SUFFIXES from litellm.types.utils import ServiceTier @@ -4187,6 +4310,37 @@ def test_azure_gpt_6_foundry_price_sheet(_local_model_cost_map, model_base): assert azure_eu_info[field] == pytest.approx(1.2 * base) +@pytest.mark.parametrize("model", ["azure/gpt-6-luna", "azure/gpt-6-luna-2026-09-22"]) +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "cache_read_rate", "output_rate"), + [ + (100_000, 5e-08, 5e-09, 2.5e-07), + (300_000, 1e-07, 1e-08, 3.75e-07), + ], +) +def test_generic_cost_per_token_azure_gpt_6_luna_flex_tiers( + _local_model_cost_map, model, prompt_tokens, input_rate, cache_read_rate, output_rate +): + """azure/gpt-6-luna bills Azure's Flex Global rates, half of Standard, with the long context step past 272K.""" + cached_tokens = 20_000 + completion_tokens = 1_000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="azure", + service_tier="flex", + ) + expected_prompt = (prompt_tokens - cached_tokens) * input_rate + cached_tokens * cache_read_rate + assert prompt_cost == pytest.approx(expected_prompt) + assert completion_cost == pytest.approx(completion_tokens * output_rate) + + @pytest.mark.parametrize("region_prefix", ["azure/", "azure/us/", "azure/eu/"]) def test_azure_gpt_5_6_alias_matches_sol_pricing(_local_model_cost_map, region_prefix): """The bare gpt-5.6 alias routes to GPT-5.6 Sol, so every Azure region must bill the diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 9da2632943c..06e673c3e68 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1230,3 +1230,27 @@ def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt( def test_built_in_tool_lookups_ignore_malformed_request_values(kwargs: dict[str, object]) -> None: assert StandardBuiltInToolCostTracking.get_web_search_options(kwargs) is None assert StandardBuiltInToolCostTracking.get_file_search_tool_call(kwargs) is None + + +def test_per_call_web_search_path_bills_the_count_the_usage_reports(local_model_cost_map): + """ + A provider with no web search calculator of its own (an OpenAI-compatible JSON provider) that + reports server_tool_use.web_search_requests is billed that many per-call searches, not one. + """ + from litellm.types.utils import ServerToolUse, Usage + + model = "empiriolabs/qwen3-7-plus" + per_search = litellm.model_cost[model]["search_context_cost_per_query"]["search_context_size_medium"] + response = ModelResponse(model="qwen3-7-plus") + usage = Usage( + prompt_tokens=4, completion_tokens=5, total_tokens=9, server_tool_use=ServerToolUse(web_search_requests=3) + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=usage, + custom_llm_provider="empiriolabs", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(3 * per_search) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index b6c18e0da86..1d2a5e4cc9b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -12,7 +12,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.llms.base import CachedTokensDetails -from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, PromptTokensDetailsWrapper, Usage TIER_MODEL: Final = "tier-priced-test-model" TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( @@ -48,6 +48,190 @@ AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( ) +@pytest.mark.parametrize("image_tokens", (60, 100)) +@pytest.mark.parametrize( + ("service_tier", "priority_rate", "image_rate"), + ((None, 11e-06, 7e-06), ("priority", 11e-06, 11e-06), ("flex", 11e-06, 7e-06), ("priority", 0.0, 0.0)), +) +def test_completion_cost_uses_priority_image_output_rate( + local_model_cost_map: None, service_tier: str | None, priority_rate: float, image_rate: float, image_tokens: int +) -> None: + model: Final = "gemini-synthetic-image-priority" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 5e-06, + "output_cost_per_image_token": 7e-06, + "output_cost_per_image_token_priority": priority_rate, + } + } + ) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=20, + completion_tokens=100, + total_tokens=120, + completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=image_tokens), + ), + ) + cost: Final = litellm.completion_cost( + completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier + ) + assert cost == pytest.approx(20 * 3e-06 + (100 - image_tokens) * 5e-06 + image_tokens * image_rate) + + +@pytest.mark.parametrize("prompt_tokens", (199_999, 200_000, 200_001)) +@pytest.mark.parametrize( + ("service_tier", "flat_rate", "context_rate"), + ((None, 7e-06, 11e-06), ("priority", 13e-06, 17e-06), ("flex", 19e-06, 23e-06)), +) +def test_completion_cost_uses_total_prompt_for_audio_context_tiers( + local_model_cost_map: None, prompt_tokens: int, service_tier: str | None, flat_rate: float, context_rate: float +) -> None: + model: Final = "gemini-synthetic-audio-context" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 5e-06, + "input_cost_per_audio_token": 7e-06, + "input_cost_per_audio_token_above_200k_tokens": 11e-06, + "input_cost_per_audio_token_priority": 13e-06, + "input_cost_per_audio_token_above_200k_tokens_priority": 17e-06, + "input_cost_per_audio_token_flex": 19e-06, + "input_cost_per_audio_token_above_200k_tokens_flex": 23e-06, + } + } + ) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=10, + total_tokens=prompt_tokens + 10, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100), + ), + ) + cost: Final = litellm.completion_cost( + completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier + ) + audio_rate: Final = context_rate if prompt_tokens > 200_000 else flat_rate + assert cost == pytest.approx((prompt_tokens - 100) * 3e-06 + 100 * audio_rate + 10 * 5e-06) + + +@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001)) +def test_batch_cost_uses_audio_context_and_image_output_tiers(local_model_cost_map: None, prompt_tokens: int) -> None: + from litellm.cost_calculator import batch_cost_calculator + + model: Final = "gemini-synthetic-batch-audio-context" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 6e-06, + "output_cost_per_token": 10e-06, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_200k_tokens_batches": 13e-06, + "output_cost_per_token_above_200k_tokens_batches": 17e-06, + "output_cost_per_image_token_batches": 19e-06, + "input_cost_per_audio_token_batches": 7e-06, + "input_cost_per_audio_token_above_200k_tokens_batches": 11e-06, + } + } + ) + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=20, + total_tokens=prompt_tokens + 20, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100), + completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=10), + ) + cost: Final = batch_cost_calculator(model=model, custom_llm_provider="vertex_ai", usage=usage) + audio_rate: Final = 11e-06 if prompt_tokens > 200_000 else 7e-06 + input_rate: Final = 13e-06 if prompt_tokens > 200_000 else 3e-06 + output_rate: Final = 17e-06 if prompt_tokens > 200_000 else 5e-06 + assert sum(cost) == pytest.approx( + (prompt_tokens - 100) * input_rate + 100 * audio_rate + 10 * output_rate + 10 * 19e-06 + ) + + +@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001)) +def test_completion_cost_uses_flex_context_input_and_output_rates( + local_model_cost_map: None, prompt_tokens: int +) -> None: + model: Final = "gemini-synthetic-flex-context" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 6e-06, + "output_cost_per_token": 10e-06, + "input_cost_per_token_flex": 3e-06, + "output_cost_per_token_flex": 5e-06, + "input_cost_per_token_above_200k_tokens_flex": 7e-06, + "output_cost_per_token_above_200k_tokens_flex": 11e-06, + } + } + ) + response: Final = ModelResponse( + model=model, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=10, total_tokens=prompt_tokens + 10) + ) + cost: Final = litellm.completion_cost( + completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="flex" + ) + input_rate: Final = 7e-06 if prompt_tokens > 200_000 else 3e-06 + output_rate: Final = 11e-06 if prompt_tokens > 200_000 else 5e-06 + assert cost == pytest.approx(prompt_tokens * input_rate + 10 * output_rate) + + +@pytest.mark.parametrize( + ("audio_fields", "audio_rate"), + ( + ({}, 3e-06), + ({"input_cost_per_audio_token": 7e-06}, 7e-06), + ({"input_cost_per_audio_token_above_200k_tokens": 11e-06}, 11e-06), + ({"input_cost_per_audio_token_above_200k_tokens_priority": 0.0}, 0.0), + ), +) +def test_audio_context_rates_preserve_fallbacks_and_explicit_zero( + local_model_cost_map: None, audio_fields: Mapping[str, float], audio_rate: float +) -> None: + model: Final = "gemini-synthetic-audio-fallback" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 5e-06, + **audio_fields, + } + } + ) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=200_001, + completion_tokens=10, + total_tokens=200_011, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100), + ), + ) + cost: Final = litellm.completion_cost( + completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="priority" + ) + assert cost == pytest.approx(199_901 * 3e-06 + 100 * audio_rate + 10 * 5e-06) + + def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float: return ( (PROMPT_TOKENS - CACHED_TOKENS) * prices[f"input_cost_per_token{column_suffix}"] diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_invalid_response_traceback.py b/tests/unit/litellm_core_utils/llm_response_utils/test_invalid_response_traceback.py new file mode 100644 index 00000000000..a2beb60665e --- /dev/null +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_invalid_response_traceback.py @@ -0,0 +1,54 @@ +""" +The "Invalid response object" error carries only the conversion's own traceback. A conversion that +runs while another error is being handled (a router retry inside the previous attempt's ``except``) +must not copy that error's text into its message, where exception mapping would read it. +""" + +import pytest + +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import convert_to_model_response_object # pyright: ignore[reportUnknownVariableType] # its untyped start_time, end_time and dict parameters +from litellm.types.utils import ModelResponse + +# A text_completion body handed to the chat conversion: ``choices[0]`` has no ``message``. +_MALFORMED_CHAT_BODY = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"text": "ok", "index": 0, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + +_OUTER_TEXT = "outer attempt failed: rate limit exceeded" + + +class _OuterAttemptError(Exception): + pass + + +def _convert_while_handling_another_error() -> Exception: + try: + raise _OuterAttemptError(_OUTER_TEXT) + except _OuterAttemptError: + with pytest.raises(Exception, match="Invalid response object") as exc_info: + _ = convert_to_model_response_object( + response_object=dict(_MALFORMED_CHAT_BODY), + model_response_object=ModelResponse(), + response_type="completion", + ) + return exc_info.value + + +def test_the_message_leaves_out_the_error_being_handled_when_the_conversion_ran() -> None: + message = str(_convert_while_handling_another_error()) + + assert _OUTER_TEXT not in message + assert "_OuterAttemptError" not in message + assert "During handling of the above exception" not in message + + +def test_the_message_names_the_conversion_error_itself() -> None: + message = str(_convert_while_handling_another_error()) + + assert "KeyError: 'message'" in message + assert "Traceback (most recent call last)" in message diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 146bb08a4fd..a17deedf7a9 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( schema_has_unresolvable_ref, schema_property_count, split_concatenated_json_objects, + split_embedding_media_for_tpm, split_media_for_tpm, strip_encrypted_reasoning_from_messages, strip_litellm_response_only_fields, @@ -2709,3 +2710,22 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] assert messages == messages_before + + +@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"]) +def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None: + import gc + import tracemalloc + + uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii") + split_embedding_media_for_tpm(f"data:{mime};base64,AAAA") + gc.collect() + tracemalloc.start() + try: + _, media = split_embedding_media_for_tpm(uri) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert media + assert peak < 1.5 * len(uri) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8a755d47817..d65f094971a 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1298,6 +1298,71 @@ def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_pro } +@pytest.mark.parametrize( + ("image_url", "expected_type"), + ( + ({"url": "https://example.com/download?id=7", "format": "application/pdf"}, "document"), + ({"url": "https://example.com/download?id=7", "format": "Application/PDF; charset=binary"}, "document"), + ({"url": "https://example.com/files/report.pdf", "format": "image/png"}, "image"), + ({"url": "https://example.com/pdf/photo.png"}, "image"), + ({"url": "https://example.com/pdf/photo.png", "format": 5}, "image"), + ({"url": "https://example.com/pdf/photo.png", "format": ["application/pdf"]}, "image"), + ), +) +def test_anthropic_messages_pt_picks_block_type_for_https_image_url( + image_url: dict[str, object], expected_type: str +) -> None: + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": image_url}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": expected_type, + "source": {"type": "url", "url": image_url["url"]}, + } + + +@pytest.mark.parametrize( + "image_url", + ( + {"url": "data:Application/PDF;base64,JVBERi0xLjQK"}, + {"url": "data:application/x-pdf;base64,JVBERi0xLjQK"}, + {"url": "data:application/octet-stream;base64,JVBERi0xLjQK", "format": "application/pdf; charset=binary"}, + ), +) +def test_anthropic_messages_pt_normalizes_a_pdf_media_type_with_case_or_parameters( + image_url: dict[str, str], +) -> None: + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": image_url}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + } + + def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None: image_data = "iVBORw0KGgo=" result: Final = anthropic_messages_pt( diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index aab57b87a90..d7696c00dee 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -327,7 +327,7 @@ EMPTY_ASSISTANT = pytest.mark.parametrize( ) -def test_flagged_placement_handles_a_long_run_of_unanswered_tool_search_uses_in_linear_time(): +def test_flagged_placement_turns_reminders_before_a_long_run_of_unanswered_tool_search_uses_into_user_turns(): search_uses: Final = tuple( { "role": "assistant", @@ -349,12 +349,9 @@ def test_flagged_placement_handles_a_long_run_of_unanswered_tool_search_uses_in_ {"role": "user", "content": "end"}, ] - started: Final = time.perf_counter() placed: Final = place_mid_conversation_system(messages, supports_mid_conversation_system=True) - elapsed: Final = time.perf_counter() - started assert set(_roles(placed)[:21]) == {"user"} - assert elapsed < 1.0 def test_flagged_placement_merges_every_system_run_that_shares_a_user_turn(): diff --git a/tests/unit/litellm_core_utils/test_batch_utils.py b/tests/unit/litellm_core_utils/test_batch_utils.py index cd37d635c43..45461b50761 100644 --- a/tests/unit/litellm_core_utils/test_batch_utils.py +++ b/tests/unit/litellm_core_utils/test_batch_utils.py @@ -5,14 +5,22 @@ from io import BytesIO from typing import Dict, Final, List import pytest +from openai.types.batch import BatchRequestCounts import litellm from litellm.litellm_core_utils.batch_utils import ( + POLLED_BATCH_COST_IS_FINAL_KWARG, + POLLED_BATCH_ID_KWARG, + ChargeOnceJob, InMemoryFile, + batch_cost_is_final, + defers_batch_cost, get_router_metadata_variable_name, parse_jsonl_with_embedded_newlines, replace_model_in_jsonl, ) +from litellm.types.utils import LiteLLMBatch +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG, encode_video_id_with_provider from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -382,30 +390,241 @@ def setup_and_teardown(): asyncio.set_event_loop(None) -@pytest.mark.parametrize( - ("call_type", "cost", "eligible"), - (("aretrieve_batch", 0.25, True), ("retrieve_batch", 0.25, True), ("acompletion", 0.25, False), - ("aretrieve_batch", 0, False), ("aretrieve_batch", -0.25, False)), -) -def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str, cost: float, eligible: bool) -> None: +def _batch(status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None): from openai.types import Batch - from litellm.litellm_core_utils.batch_utils import batch_id_to_charge_once - - batch: Final = Batch( - id="batch-budget", completion_window="24h", created_at=1, endpoint="/v1/chat/completions", - input_file_id="file-budget", object="batch", status="completed", + return Batch( + id="job-budget", completion_window="24h", created_at=1, endpoint="/v1/chat/completions", + input_file_id="file-budget", object="batch", status=status, output_file_id=output_file_id, + request_counts=counts, ) - assert batch_id_to_charge_once(call_type, batch, cost) == (batch.id if eligible else None) - assert batch_id_to_charge_once(call_type, {"id": batch.id}, cost) is None -def test_batch_charge_markers_keep_counters_and_batches_independent() -> None: +def _completed_batch(): + return _batch("completed", output_file_id="file-out") + + +def _completed_video(): + from litellm.types.videos.main import VideoObject + + return VideoObject(id="job-budget", object="video", status="completed") + + +@pytest.mark.parametrize( + ("call_type", "polled", "cost", "kind"), + ( + ("aretrieve_batch", _completed_batch, 0.25, "batch"), + ("retrieve_batch", _completed_batch, 0.25, "batch"), + ("avideo_status", _completed_video, 0.25, "video"), + ("video_status", _completed_video, 0.25, "video"), + ("acompletion", _completed_batch, 0.25, None), + ("avideo_status", _completed_batch, 0.25, None), + ("aretrieve_batch", _completed_video, 0.25, None), + ("avideo_generation", _completed_video, 0.25, None), + ("aretrieve_batch", _completed_batch, 0, None), + ("avideo_status", _completed_video, 0, None), + ("avideo_status", _completed_video, -0.25, None), + ), +) +def test_job_charge_identity_requires_a_cost_bearing_poll(call_type: str, polled, cost: float, kind) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + + expected: Final = None if kind is None else ChargeOnceJob(kind=kind, job_id="job-budget") + kwargs: Final = { + "call_type": call_type, + POLLED_VIDEO_STATUS_KWARG: "completed", + "standard_logging_object": {"response_cost": cost}, + } + assert job_to_charge_once(kwargs, polled(), cost) == expected + assert job_to_charge_once(kwargs, {"id": "job-budget"}, cost) is None + + +@pytest.mark.parametrize( + ("call_type", "polled", "guardrail_only_poll", "kind", "repeat_cost"), + ( + ("aretrieve_batch", _completed_batch, lambda: _batch("in_progress"), "batch", 0.0), + ("avideo_status", _completed_video, _completed_video, "video", 0.01), + ), +) +def test_only_the_jobs_own_cost_is_charged_once_since_a_guardrail_charges_every_poll( + call_type: str, polled, guardrail_only_poll, kind, repeat_cost +) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + + kwargs: Final = { + "call_type": call_type, + POLLED_VIDEO_STATUS_KWARG: "completed", + "standard_logging_object": {"guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}]}, + } + assert job_to_charge_once(kwargs, guardrail_only_poll(), 0.01) is None + assert job_to_charge_once(kwargs, polled(), 0.43) == ChargeOnceJob( + kind=kind, job_id="job-budget", repeat_poll_cost=repeat_cost + ) + + +@pytest.mark.parametrize("status", ("completed", "failed", "cancelled", "expired")) +def test_a_finished_batch_that_cost_nothing_itself_is_still_charged_once_for_its_retrieves_guardrail( + status: str, +) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + + kwargs: Final = { + "call_type": "aretrieve_batch", + "standard_logging_object": {"guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}]}, + } + finished_batch: Final = _batch(status, output_file_id="file-out") + assert job_to_charge_once(kwargs, finished_batch, 0.01) == ChargeOnceJob(kind="batch", job_id="job-budget") + + +def test_a_completed_batch_still_waiting_on_its_output_does_not_take_the_charge_its_real_cost_needs() -> None: + """Its first retrieve can report completed before the output file exists, and settling on that guardrail-only + retrieve would leave the batch's real cost nothing to charge once the output arrives.""" + from litellm.litellm_core_utils.batch_utils import job_to_charge_once + + kwargs: Final = { + "call_type": "aretrieve_batch", + "standard_logging_object": {"guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}]}, + } + assert job_to_charge_once(kwargs, _batch("completed"), 0.01) is None + assert defers_batch_cost(kwargs, _batch("completed")) + + +def test_a_video_is_charged_once_it_completes_under_the_id_the_client_polled() -> None: + """A provider may price a running video too, and a custom provider's own id can repeat across deployments.""" + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + from litellm.types.videos.main import VideoObject + + def poll(status: str, polled_id: str) -> ChargeOnceJob | None: + return job_to_charge_once( + {"call_type": "avideo_status", POLLED_VIDEO_STATUS_KWARG: status, "litellm_params": {"video_id": polled_id}}, + VideoObject(id="job-1", object="video", status=status), + 0.42, + ) + + assert poll("in_progress", "video_deployment_a") is None + assert poll("completed", "video_deployment_a") == ChargeOnceJob(kind="video", job_id="video_deployment_a") + assert poll("completed", "video_deployment_b") == ChargeOnceJob(kind="video", job_id="video_deployment_b") + + +@pytest.mark.parametrize("status", ["in_progress", "completed"]) +def test_a_video_poll_is_judged_by_its_logged_status_when_redaction_replaced_the_video(status: str) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + + kwargs: Final = { + "call_type": "avideo_status", + POLLED_VIDEO_STATUS_KWARG: status, + "litellm_params": {"video_id": "video_deployment_a"}, + } + expected: Final = ChargeOnceJob(kind="video", job_id="video_deployment_a") if status == "completed" else None + + assert job_to_charge_once(kwargs, {"text": "redacted-by-litellm"}, 0.42) == expected + + +@pytest.mark.parametrize( + ("cost_is_final", "deferred", "expected"), + [(False, True, None), (True, False, ChargeOnceJob(kind="batch", job_id="batch_abc"))], +) +def test_a_batch_retrieve_is_judged_by_its_logged_batch_when_redaction_replaced_it( + cost_is_final: bool, deferred: bool, expected: ChargeOnceJob | None +) -> None: + from litellm.litellm_core_utils.batch_utils import job_to_charge_once + + kwargs: Final = { + "call_type": "aretrieve_batch", + POLLED_BATCH_ID_KWARG: "batch_abc", + POLLED_BATCH_COST_IS_FINAL_KWARG: cost_is_final, + "standard_logging_object": {"guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}]}, + } + redacted: Final = {"text": "redacted-by-litellm"} + + assert defers_batch_cost(kwargs, redacted) is deferred + assert job_to_charge_once(kwargs, redacted, 0.01) == expected + + +def test_a_video_polled_with_or_without_its_base64_padding_is_charged_as_one_job() -> None: + from litellm.litellm_core_utils.batch_utils import job_to_charge_once + + padded: Final = encode_video_id_with_provider("job-1", "custom_video", "deployment-a") + assert padded.endswith("=") + + def charged_job(polled_id: str): + return job_to_charge_once( + {"call_type": "avideo_status", POLLED_VIDEO_STATUS_KWARG: "completed", "litellm_params": {"video_id": polled_id}}, + None, + 0.42, + ) + + assert charged_job(padded.rstrip("=")) == charged_job(padded) + assert charged_job(padded) is not None + + +def test_job_charge_markers_keep_counters_jobs_and_kinds_independent() -> None: from litellm.caching.in_memory_cache import InMemoryCache - from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_charged_once_marker_key + first_batch: Final = ChargeOnceJob(kind="batch", job_id="first-job") cache: Final = InMemoryCache() - cache.set_cache(batch_charged_once_marker_key("provider-spend", "first-batch"), 1) - assert cache.get_cache(batch_charged_once_marker_key("provider-spend", "first-batch")) == 1 - assert cache.get_cache(batch_charged_once_marker_key("deployment-spend", "first-batch")) is None - assert cache.get_cache(batch_charged_once_marker_key("provider-spend", "second-batch")) is None + cache.set_cache(job_charged_once_marker_key("provider-spend", first_batch), 1) + assert cache.get_cache(job_charged_once_marker_key("provider-spend", first_batch)) == 1 + assert cache.get_cache(job_charged_once_marker_key("deployment-spend", first_batch)) is None + assert cache.get_cache(job_charged_once_marker_key("provider-spend", ChargeOnceJob("batch", "second-job"))) is None + assert cache.get_cache(job_charged_once_marker_key("provider-spend", ChargeOnceJob("video", "first-job"))) is None + + +def test_batch_charge_marker_keeps_the_key_markers_already_written_use() -> None: + """Markers written before the video kind existed must still block a re-charge after an upgrade.""" + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_charged_once_marker_key + + assert job_charged_once_marker_key("provider-spend", ChargeOnceJob("batch", "first-job")) == ( + "provider-spend:batch:first-job" + ) + + +def _retrieved_batch( + status: str, output_file_id: str | None = None, counts: BatchRequestCounts | None = None +) -> LiteLLMBatch: + return LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + object="batch", + status="validating", + output_file_id=output_file_id, + request_counts=counts, + ).model_copy(update={"status": status}) + + +class TestBatchCostIsFinal: + """Every retrieve of one batch writes the same spend row, so the first retrieve + that prices it decides the row for good. A poll before the output exists must + therefore not count as final: pricing it recorded $0 and pinned it (LIT-7048).""" + + @pytest.mark.parametrize("status", ["validating", "in_progress", "finalizing", "cancelling"]) + def test_in_flight_batch_is_not_final(self, status): + assert batch_cost_is_final(_retrieved_batch(status)) is False + + @pytest.mark.parametrize("status", ["completed", "complete"]) + def test_completed_with_output_is_final(self, status): + assert batch_cost_is_final(_retrieved_batch(status, output_file_id="file-out")) is True + + def test_completed_without_output_and_unknown_counts_is_not_final(self): + assert batch_cost_is_final(_retrieved_batch("completed")) is False + + def test_completed_without_output_and_zero_counts_is_not_final(self): + counts = BatchRequestCounts(total=0, completed=0, failed=0) + assert batch_cost_is_final(_retrieved_batch("completed", counts=counts)) is False + + def test_completed_without_output_but_successful_lines_is_not_final(self): + counts = BatchRequestCounts(total=2, completed=2, failed=0) + assert batch_cost_is_final(_retrieved_batch("completed", counts=counts)) is False + + @pytest.mark.parametrize("status", ["completed", "complete"]) + def test_completed_without_output_and_every_line_failed_is_final(self, status): + counts = BatchRequestCounts(total=2, completed=0, failed=2) + assert batch_cost_is_final(_retrieved_batch(status, counts=counts)) is True + + @pytest.mark.parametrize("status", ["failed", "expired", "cancelled"]) + def test_other_terminal_statuses_are_final(self, status): + assert batch_cost_is_final(_retrieved_batch(status)) is True diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 6167e91c37f..d26da679246 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -21,6 +21,8 @@ they stay out of the body even without it. """ import datetime +import json +from collections.abc import Mapping, Sequence from typing import Any, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch @@ -28,11 +30,13 @@ import pytest from openai._legacy_response import LegacyAPIResponse import litellm +from litellm.caching.caching import Cache from litellm.integrations.code_interpreter_interception.handler import ( CodeInterpreterInterceptionLogger, ) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + _execute_chat_completion_agentic_plan, maybe_run_chat_completion_agentic_loop, ) from litellm.litellm_core_utils.litellm_logging import Logging @@ -43,6 +47,7 @@ from litellm.types.integrations.custom_logger import ( WEBSEARCH_INTERCEPTION_PREFIX, AgenticLoopPlan, AgenticLoopRequestPatch, + AgenticLoopSafetyError, ) from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -50,6 +55,7 @@ from litellm.types.utils import ( Function, Message, ModelResponse, + Usage, ) # The internal control fields that must never reach a provider request body. @@ -260,8 +266,6 @@ class _GateOnlyLogger(CustomLogger): class _RecordingGateLogger(_GateOnlyLogger): - """Also keeps every follow-up response the loop hands to the post hook.""" - def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: super().__init__(plan=plan, tool_calls=tool_calls) self.followups: tuple[ModelResponse, ...] = () @@ -441,72 +445,544 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ assert call_kwargs["api_base"] == "https://a" +_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} +_REFUSING_RAILS: Final = ( + {"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + { + "_agentic_loop_depth": 0, + "max_agentic_loops": 3, + "_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)], + }, +) + + +def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}")) + + +async def _run_refused_rerun( + response: ModelResponse, + rail_kwargs: Dict[str, object], + stream: bool = False, + gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS, + other_hooks_tool_calls: tuple[Dict[str, object], ...] = (), +) -> Any: + litellm.callbacks = [ + _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=tool_calls) + for tool_calls in (gate_tool_calls, *other_hooks_tool_calls) + ] + acompletion_mock = AsyncMock() + with patch.object(litellm, "acompletion", acompletion_mock): + try: + return await maybe_run_chat_completion_agentic_loop( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs=dict(rail_kwargs), + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=stream, + ) + finally: + acompletion_mock.assert_not_awaited() + + @pytest.mark.asyncio -async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops( - restore_callbacks, +@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"]) +async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call( + restore_callbacks, rail_kwargs: Dict[str, object] ): - """depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops, - before any follow-up call is attempted.""" - logger = _GateOnlyLogger( - plan=AgenticLoopPlan(run_agentic_loop=True), + result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs) + + assert isinstance(result, ModelResponse) + assert not result.choices[0].message.tool_calls + assert result.choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks): + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content="Running it.", + tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")], + ), + ) + ] + ) + + result = await _run_refused_rerun(response, _REFUSING_RAILS[0]) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.content == "Running it." + + +def _indexed_tool_call(call_id: str, name: str, index: int) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall( + id=call_id, type="function", function=Function(name=name, arguments="{}"), index=index + ) + + +def _turn_with_the_internal_call_between_the_clients() -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _indexed_tool_call("call_weather", "get_weather", 0), + _indexed_tool_call("call_abc", "litellm_code_execution", 1), + _indexed_tool_call("call_time", "get_time", 2), + ], + ), + ) + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"]) +async def test_refused_rerun_renumbers_the_kept_tool_calls_from_zero(restore_callbacks, rail_kwargs: Dict[str, object]): + result = await _run_refused_rerun(_turn_with_the_internal_call_between_the_clients(), rail_kwargs) + + assert [(call.id, call.index) for call in result.choices[0].message.tool_calls] == [ + ("call_weather", 0), + ("call_time", 1), + ] + + +@pytest.mark.asyncio +async def test_refused_rerun_on_a_converted_stream_replays_the_kept_tool_calls_from_index_zero( + restore_callbacks, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr( + litellm, + "callbacks", + [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=_REFUSED_GATE_TOOL_CALLS)], + ) + logging_obj: Final = Logging( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-refused-stream", + function_id="fn-refused-stream", + ) + + result: Final = await maybe_run_chat_completion_agentic_loop( + response=_turn_with_the_internal_call_between_the_clients(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs={**_REFUSING_RAILS[1], f"{CODE_INTERPRETER_INTERCEPTION_PREFIX}_converted_stream": True}, + logging_obj=logging_obj, + custom_llm_provider="openai", + stream=False, + ) + + assert isinstance(result, CustomStreamWrapper) + deltas: Final = [chunk.choices[0].delta async for chunk in result if chunk.choices] + streamed_calls: Final = [call for delta in deltas for call in delta.tool_calls or []] + assert [(call.id, call.index) for call in streamed_calls] == [("call_weather", 0), ("call_time", 1)] + + +@pytest.mark.asyncio +async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clients(restore_callbacks): + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_abc", "litellm_code_execution"), + _tool_call("call_search", "litellm_web_search"), + _tool_call("call_client", "lookup"), + ], + ), + ) + ] + ) + + result = await _run_refused_rerun( + response, + _REFUSING_RAILS[0], + other_hooks_tool_calls=({"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]},), + ) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + + +@pytest.mark.asyncio +async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks): + response: Final = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_abc", "litellm_code_execution"), + _tool_call("call_search", "litellm_web_search"), + _tool_call("call_client", "lookup"), + ], + ), + ) + ] + ) + + result: Final = await _run_refused_rerun( + response, + _REFUSING_RAILS[0], + gate_tool_calls={"tool_calls": [{"id": "call_abc"}]}, + other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},), + ) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response, gate_tool_calls", + [(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)], + ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"], +) +async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged( + restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]] +): + result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls) + + assert result.choices[0].model_dump() == response.choices[0].model_dump() + + +@pytest.mark.asyncio +async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks): + with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"): + await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True) + + +@pytest.mark.asyncio +async def test_dispatcher_surfaces_a_failed_follow_up_instead_of_the_tool_call_turn(): + logger: Final = _GateOnlyLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"} + ), + ), + tool_calls={"tool_calls": [{"id": "call_abc"}]}, + ) + + with pytest.raises(litellm.RateLimitError): + await _dispatch_tool_call_turn(_tool_call_model_response(), logger) + + assert logger.cleanup_calls == 1 + + +class _PlanRecordingLogger(_GateOnlyLogger): + def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: + super().__init__(plan=plan, tool_calls=tool_calls) + self.logging_objs: List[Any] = [] + + async def async_build_agentic_loop_plan( + self, + tools: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: Dict[str, Any], + logging_obj: Any, + stream: bool, + kwargs: Dict[str, Any], + ) -> AgenticLoopPlan: + self.logging_objs.append(logging_obj) + return self._plan + + +@pytest.mark.asyncio +async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(restore_callbacks): + logger: Final = _PlanRecordingLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"} + ), + ), tool_calls={"tool_calls": [{"id": "call_abc"}]}, ) litellm.callbacks = [logger] + first_call: Final = ModelResponse( + choices=_tool_call_model_response().choices, usage=Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8) + ) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(first_call)) - acompletion_mock = AsyncMock() - with patch.object(litellm, "acompletion", acompletion_mock): - with pytest.raises(ValueError, match="max_agentic_loops"): - await maybe_run_chat_completion_agentic_loop( - response=_tool_call_model_response(), - model="gpt-4o-mini", - messages=[{"role": "user", "content": "hi"}], - optional_params={}, - kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, - logging_obj=_LoggingStub(), - custom_llm_provider="openai", - stream=False, - ) + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) - acompletion_mock.assert_not_awaited() + (logging_obj,) = logger.logging_objs + price: Final = litellm.model_cost["gpt-4o-mini"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + 5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"] + ) + + +async def _agentic_turn_with_follow_up( + logger: _PlanRecordingLogger, monkeypatch: pytest.MonkeyPatch +) -> tuple[ModelResponse, Logging]: + monkeypatch.setattr(litellm, "callbacks", [logger]) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_model_response())) + response: Final = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) + assert isinstance(response, ModelResponse) + (logging_obj,) = logger.logging_objs + return response, logging_obj + + +def _follow_up_logger() -> _PlanRecordingLogger: + return _PlanRecordingLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": "42"}), + ), + tool_calls={"tool_calls": [{"id": "call_abc"}]}, + ) @pytest.mark.asyncio -async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks): - """A tool_calls fingerprint already present in _agentic_loop_fingerprints -> - ValueError about the repeated fingerprint (cycle guard), with no follow-up - call.""" - import json +async def test_acompletion_records_the_cost_breakdown_of_a_successful_follow_up(monkeypatch): + response, logging_obj = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch) - # The dispatcher fingerprints the whole value the gate returns as its second - # tuple element, so the seeded fingerprint must mirror that dict exactly. - gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} - fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str) + response_cost: Final = response._hidden_params["response_cost"] + assert response_cost > 0 + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["total_cost"] == pytest.approx(response_cost) - logger = _GateOnlyLogger( - plan=AgenticLoopPlan(run_agentic_loop=True), - tool_calls=gate_tool_calls, + +@pytest.mark.asyncio +async def test_acompletion_keeps_a_cached_follow_up_free(monkeypatch): + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + await litellm.acompletion(model="openai/gpt-4o-mini", messages=_patched_messages(), mock_response="42") + + response, _ = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch) + + assert response._hidden_params["cache_hit"] is True + assert response._hidden_params["response_cost"] == 0 + + +def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse: + return ModelResponse( + model="gpt-4o-mini", + choices=[ + Choices( + finish_reason="tool_calls", + message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]), + ) + ], + usage=usage, ) - litellm.callbacks = [logger] - acompletion_mock = AsyncMock() - with patch.object(litellm, "acompletion", acompletion_mock): - with pytest.raises(ValueError, match="fingerprint"): - await maybe_run_chat_completion_agentic_loop( - response=_tool_call_model_response(), + +class _TwoRoundLogger(CustomLogger): + def __init__(self, middle_round: ModelResponse) -> None: + super().__init__() + self._follow_ups: Final = (middle_round, "litellm.RateLimitError") + self.logging_objs: tuple[object, ...] = () + + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + assert isinstance(response, ModelResponse) + calls: Final = response.choices[0].message.tool_calls or [] + return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]} + + async def async_build_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + follow_up: Final = self._follow_ups[len(self.logging_objs)] + self.logging_objs = (*self.logging_objs, logging_obj) + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}), + ) + + +@pytest.mark.asyncio +async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails( + restore_callbacks, monkeypatch: pytest.MonkeyPatch +): + first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8) + middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15) + logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage)) + monkeypatch.setattr(litellm, "callbacks", [logger]) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage))) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) + + first_logging_obj, _ = logger.logging_objs + assert isinstance(first_logging_obj, Logging) + price: Final = litellm.model_cost["gpt-4o-mini"] + billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"] + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4) + assert first_logging_obj.model_call_details["response_cost"] == pytest.approx( + (5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool): + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + + sandbox_error: Final = RuntimeError("sandbox crashed") + + async def failing_follow_up() -> ModelResponse: + raise sandbox_error + + with pytest.raises(RuntimeError) as raised: + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=Logging( model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}], - optional_params={}, - kwargs={ - "_agentic_loop_depth": 0, - "max_agentic_loops": 3, - "_agentic_loop_fingerprints": [fingerprint], - }, - logging_obj=_LoggingStub(), - custom_llm_provider="openai", - stream=False, - ) + stream=stream, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-builtin-error", + function_id="fn-builtin-error", + ), + response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)), + stream=stream, + ) - acompletion_mock.assert_not_awaited() + assert raised.value is sandbox_error + + +class _RunOnlyLoopLogger(CustomLogger): + def __init__(self, result: ModelResponse) -> None: + super().__init__() + self._result = result + + async def async_should_run_agentic_loop( + self, + response: Any, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + custom_llm_provider: str, + kwargs: Dict[str, Any], + ) -> Tuple[bool, Dict[str, Any]]: + return True, {"tool_calls": [{"id": "call_abc"}]} + + async def async_run_agentic_loop( + self, + tools: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: Dict[str, Any], + logging_obj: Any, + stream: bool, + kwargs: Dict[str, Any], + ) -> Any: + return self._result + + +class _LoggingWithCallbacks(_LoggingStub): + def __init__(self, callbacks: Tuple[CustomLogger, ...]) -> None: + self.dynamic_success_callbacks = list(callbacks) + + +async def _dispatch_tool_call_turn( + response: ModelResponse, *callbacks: CustomLogger +) -> ModelResponse | CustomStreamWrapper | None: + return await maybe_run_chat_completion_agentic_loop( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={}, + logging_obj=_LoggingWithCallbacks(callbacks), + custom_llm_provider="openai", + stream=False, + ) + + +@pytest.mark.asyncio +async def test_dispatcher_returns_the_tool_call_turn_when_the_plan_terminates(): + response: Final = _tool_call_model_response() + terminating: Final = _GateOnlyLogger( + plan=AgenticLoopPlan(terminate=True), tool_calls={"tool_calls": [{"id": "call_abc"}]} + ) + + assert await _dispatch_tool_call_turn(response, terminating) is response + + +@pytest.mark.asyncio +async def test_dispatcher_moves_past_a_plan_that_declines_the_loop(): + override: Final = _plain_model_response("from the next callback") + tool_calls: Final = {"tool_calls": [{"id": "call_abc"}]} + declined: Final = _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=False), tool_calls=tool_calls) + overriding: Final = _GateOnlyLogger(plan=AgenticLoopPlan(response_override=override), tool_calls=tool_calls) + + assert await _dispatch_tool_call_turn(_tool_call_model_response(), declined, overriding) is override + + +@pytest.mark.asyncio +async def test_dispatcher_returns_a_run_only_loops_result(): + loop_result: Final = _plain_model_response() + + assert await _dispatch_tool_call_turn(_tool_call_model_response(), _RunOnlyLoopLogger(loop_result)) is loop_result @pytest.mark.asyncio @@ -562,3 +1038,55 @@ async def test_converted_stream_replays_the_final_answer_with_the_requested_usag if include_usage else [] ) + + +class TestAgenticFollowUpKeepsTheProviderPrefix: + @staticmethod + async def _run_followup( + model: str, custom_llm_provider: str, patched_model: str | None = None + ) -> ModelResponse: + response: Final = await _execute_chat_completion_agentic_plan( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "hi"}], model=patched_model + ), + ), + callback=CustomLogger(), + model=model, + optional_params={"mock_response": "ok from followup"}, + kwargs={}, + logging_obj=None, + custom_llm_provider=custom_llm_provider, + depth=0, + max_loops=2, + fingerprints=[], + fingerprint="fp", + ) + assert isinstance(response, ModelResponse) + return response + + @pytest.mark.asyncio + async def test_a_sub_path_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_an_ordinary_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o-mini", "openai") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5") + + assert response.model == "claude-sonnet-4-5" + assert response._hidden_params["custom_llm_provider"] == "anthropic" + + @pytest.mark.asyncio + async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None: + response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") + + assert response._hidden_params["custom_llm_provider"] == "openrouter" diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index b21e9eb8b5a..5d6d14a4ca8 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.core_helpers import ( response_cost_in_hidden_params, set_provider_response_headers_in_hidden_params, unbind_budget_reservation_from_callbacks, + without, ) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( @@ -852,3 +853,21 @@ def test_billed_usage_cost_keeps_the_upstream_charge_after_litellm_wrote_its_bil stamp_litellm_bill_on_usage(unreported, unreported.usage, 0.0015) assert unreported.usage.cost == 0.0015 assert billed_usage_cost(unreported, "openai") is None + + +def test_without_drops_the_named_keys_and_keeps_the_rest_in_order() -> None: + source: Final = {"a": 1, "b": 2, "c": 3, "d": 4} + + assert list(without(source, "b", "d").items()) == [("a", 1), ("c", 3)] + assert without(source, "missing") == source + assert without(source) == source + + +def test_without_returns_a_new_dict_and_leaves_the_source_unchanged() -> None: + source: Final = {"a": 1, "b": 2} + + result: Final = without(source, "a") + result["c"] = 3 + + assert source == {"a": 1, "b": 2} + assert result == {"b": 2, "c": 3} diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py index e9814b980d1..d9603230319 100644 --- a/tests/unit/litellm_core_utils/test_decisions_pricing.py +++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py @@ -62,6 +62,66 @@ def test_apply_decisions_pricing_leaves_rows_without_decisions_fields_unchanged( assert apply_decisions_pricing(plain_row) == plain_row +@pytest.mark.parametrize("rate", [0.0, 0.002]) +def test_flat_decisions_rate_overrides_catalog_without_changing_chat( + monkeypatch: pytest.MonkeyPatch, rate: float +) -> None: + model: Final = "parallel_ai/decision" + monkeypatch.setitem( + litellm.model_cost, + model, + {**_ROW, "litellm_provider": "parallel_ai", "input_cost_per_query_decisions": rate}, + ) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (rate, 0.0) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="completion") == pytest.approx( + (100 * 1e-07, 0.0) + ) + + +@pytest.mark.parametrize("query_field", ["input_cost_per_query", "input_cost_per_query_decisions"]) +def test_query_price_on_a_provider_that_bills_tokens_stays_unbilled( + monkeypatch: pytest.MonkeyPatch, query_field: str +) -> None: + model: Final = "fireworks_ai/accounts/example/models/query-priced" + monkeypatch.setitem( + litellm.model_cost, + model, + {query_field: 0.5, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + | {"litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (0.0, 0.0) + + +@pytest.mark.parametrize("call_type", ["adecisions", "asystemone"]) +@pytest.mark.parametrize( + ("token_rates", "expected_costs"), + [ + ({"input_cost_per_token": 1e-05, "output_cost_per_token": 0.0}, (100 * 1e-05, 0.0)), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 1e-05}, (0.0, 10 * 1e-05)), + ], +) +def test_plain_query_price_leaves_a_token_priced_row_billing_tokens( + monkeypatch: pytest.MonkeyPatch, + call_type: str, + token_rates: dict[str, float], + expected_costs: tuple[float, float], +) -> None: + model: Final = "fireworks_ai/accounts/example/models/priced-both" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_query": 0.5, **token_rates, "litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + + assert cost_calculator.cost_per_token( + model=model, + custom_llm_provider="fireworks_ai", + prompt_tokens=100, + completion_tokens=10, + call_type=cast(CallTypesLiteral, call_type), + ) == pytest.approx(expected_costs) + + _LUNA_USAGE: Final = Usage( prompt_tokens=100, completion_tokens=10, diff --git a/tests/unit/litellm_core_utils/test_error_normalization.py b/tests/unit/litellm_core_utils/test_error_normalization.py index 87b9463cb01..524dda41218 100644 --- a/tests/unit/litellm_core_utils/test_error_normalization.py +++ b/tests/unit/litellm_core_utils/test_error_normalization.py @@ -217,6 +217,22 @@ def test_unknown_exception_falls_back_to_status_then_unclassified() -> None: assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED" +def test_a_529_the_anthropic_routes_answered_is_an_overload() -> None: + paused = _proxy_exc("Model is paused", "service_unavailable_error", 529) + assert StandardLoggingPayloadSetup.get_error_information(paused)["normalized_error"] == "503_PROVIDER_OVERLOADED" + + +@pytest.mark.parametrize("body", ["", "bad gateway", '{"type":"error","error":{"type":"api_error"}}']) +def test_an_internal_server_error_answered_as_529_is_an_overload_whatever_its_body_says(body: str) -> None: + overloaded = litellm.InternalServerError(body, llm_provider="anthropic", model="claude", provider_status_code=529) + overloaded.status_code = 529 + internal = litellm.InternalServerError(body, llm_provider="anthropic", model="claude") + assert ( + StandardLoggingPayloadSetup.get_error_information(overloaded)["normalized_error"], + StandardLoggingPayloadSetup.get_error_information(internal)["normalized_error"], + ) == ("503_PROVIDER_OVERLOADED", "500_PROVIDER_INTERNAL_ERROR") + + def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None: exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key") assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED" diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 1b00a32d31d..32a99a37870 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1230,12 +1230,12 @@ def test_openai_incorrect_api_key_keeps_the_proxy_authentication_error_type() -> class _UpstreamHTTPError(Exception): - def __init__(self, status_code: int) -> None: - super().__init__("upstream failure") - self.message = "upstream failure" + def __init__(self, status_code: int, message: str = "upstream failure") -> None: + super().__init__(message) + self.message = message self.status_code = status_code self.request = httpx.Request("POST", "https://api.example.com/v1/chat/completions") - self.response = httpx.Response(status_code=status_code, request=self.request, text="upstream failure") + self.response = httpx.Response(status_code=status_code, request=self.request, text=message) UPSTREAM_STATUS_CODES = (400, 401, 402, 403, 404, 408, 422, 429, 500, 503) @@ -1433,6 +1433,48 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(provider, assert returned is already_mapped +@pytest.mark.parametrize( + "message", + ['{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}', "upstream failure"], +) +def test_an_anthropic_529_stays_an_internal_server_error_that_carries_the_529(message, quiet_exception_mapping): + with pytest.raises(litellm.InternalServerError) as raised: + exception_type( + model="claude-opus-5-5", + original_exception=_UpstreamHTTPError(status_code=529, message=message), + custom_llm_provider="anthropic", + ) + + assert (type(raised.value), raised.value.status_code, raised.value.provider_status_code) == ( + litellm.InternalServerError, + 500, + 529, + ) + + +@pytest.mark.parametrize("status_code", [400, 404]) +def test_an_anthropic_client_error_naming_overloaded_carries_no_overload_status(status_code, quiet_exception_mapping): + with pytest.raises(litellm.InternalServerError) as raised: + exception_type( + model="claude-opus-5-5", + original_exception=_UpstreamHTTPError(status_code=status_code, message="model: claude-Overloaded"), + custom_llm_provider="anthropic", + ) + + assert raised.value.provider_status_code is None + + +def test_an_anthropic_500_carries_its_own_status_as_the_provider_status(quiet_exception_mapping): + with pytest.raises(litellm.InternalServerError) as raised: + exception_type( + model="claude-opus-5-5", + original_exception=_UpstreamHTTPError(status_code=500), + custom_llm_provider="anthropic", + ) + + assert (raised.value.status_code, raised.value.provider_status_code) == (500, 500) + + @pytest.mark.parametrize("status_code", UPSTREAM_STATUS_CODES) @pytest.mark.parametrize("provider", PROVIDERS_WITHOUT_A_HANDLER) def test_a_provider_without_a_handler_maps_by_the_upstream_status(provider, status_code, quiet_exception_mapping): diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index 99fa849879c..d8e56c0ae51 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -9,7 +9,8 @@ resolution (get_model_info) including the shipped rules in the bundled cost map. import json import logging -from collections.abc import Callable +import re +from collections.abc import Callable, Mapping from pathlib import Path from typing import Final @@ -562,6 +563,23 @@ def test_shipped_rules_flag_unmapped_fable_as_always_on_thinking(shipped_cost_ma assert other.get("thinking_always_on") is None +@pytest.mark.parametrize( + ("model", "provider", "on_by_default"), + [ + ("claude-opus-5-9", "anthropic", True), + ("claude-newfamily-6", "anthropic", True), + ("us.anthropic.claude-sonnet-5-9", "bedrock", True), + ("claude-opus-4-9", "anthropic", None), + ], +) +def test_shipped_rules_flag_unmapped_claude_5_and_later_as_thinking_by_default( + shipped_cost_map, model, provider, on_by_default +): + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info.get("thinking_on_by_default") is on_by_default + + @pytest.mark.parametrize( "model,provider", [ @@ -915,6 +933,7 @@ def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map) assert info.get("supports_legacy_thinking") is None assert info.get("max_input_tokens") is None assert match_fill_missing_generalizations("claude-sonnet-4-6", "anthropic") == { + **_claude_baseline()["model_info"], "supports_adaptive_thinking": True, "supports_legacy_thinking": True, "supports_tool_search": True, @@ -961,3 +980,63 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None + + +CLAUDE_BASELINE_FILL_PROVIDERS: Final = ( + "anthropic", + "azure_ai", + "bedrock", + "bedrock_converse", + "vertex_ai-anthropic_models", +) + + +def _claude_baseline() -> Mapping[str, object]: + return next(rule for rule in get_fallback_generalization_rules() if rule["name"] == "claude-family-baseline") + + +def _mapped_claude_chat_entries() -> tuple[tuple[str, Mapping[str, object]], ...]: + pattern: Final = re.compile(_claude_baseline()["pattern"], re.IGNORECASE) + return tuple( + (key, entry) + for key, entry in litellm.model_cost.items() + if isinstance(entry, dict) and entry.get("mode") == "chat" and pattern.search(key) is not None + ) + + +def _fields_overriding_baseline(entry: Mapping[str, object], baseline: Mapping[str, object]) -> frozenset[str]: + return frozenset(field for field in entry.keys() & baseline.keys() if entry[field] != baseline[field]) + + +@pytest.mark.parametrize("provider", CLAUDE_BASELINE_FILL_PROVIDERS) +def test_shipped_claude_baseline_fills_only_missing_keys_on_mapped_entries(shipped_cost_map, provider): + """A mapped Claude entry from a first-party host gets each baseline key it leaves out, + while every baseline key the entry sets keeps the entry's own value.""" + baseline: Final = _claude_baseline()["model_info"] + entries: Final = tuple( + (key, entry) for key, entry in _mapped_claude_chat_entries() if entry["litellm_provider"] == provider + ) + assert any(baseline.keys() - entry.keys() for _, entry in entries) + assert any(_fields_overriding_baseline(entry, baseline) for _, entry in entries) + + for key, entry in entries: + info = litellm.get_model_info(key) + assert info["key"] == key + assert {field: info[field] for field in baseline} == { + field: entry.get(field, value) for field, value in baseline.items() + }, key + + +def test_shipped_claude_baseline_leaves_other_providers_entries_unfilled(shipped_cost_map): + baseline: Final = _claude_baseline()["model_info"] + entries: Final = tuple( + (key, entry) + for key, entry in _mapped_claude_chat_entries() + if entry["litellm_provider"] not in CLAUDE_BASELINE_FILL_PROVIDERS + ) + assert any("supports_parallel_function_calling" not in entry for _, entry in entries) + + for key, entry in entries: + info = litellm.get_model_info(key) + missing = baseline.keys() - entry.keys() + assert {field: info.get(field) for field in missing} == dict.fromkeys(missing), key diff --git a/tests/unit/litellm_core_utils/test_get_litellm_params.py b/tests/unit/litellm_core_utils/test_get_litellm_params.py index 738ab774b2f..a1184039102 100644 --- a/tests/unit/litellm_core_utils/test_get_litellm_params.py +++ b/tests/unit/litellm_core_utils/test_get_litellm_params.py @@ -340,6 +340,13 @@ class TestGetLitellmParamsDataResidency: ) assert result["data_residency"] == "us" + def test_fedramp_host_resolves_to_fedramp(self): + result = get_litellm_params( + custom_llm_provider="openai", + api_base="https://gov.api.openai.com/v1", + ) + assert result["data_residency"] == "fedramp" + def test_global_host_resolves_to_none(self): result = get_litellm_params( custom_llm_provider="openai", diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index d1bebde9917..2d4cd4b9a6d 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -1,23 +1,36 @@ import asyncio import base64 import copy +import threading import time import uuid +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final +from unittest.mock import MagicMock import pytest from httpx import Request, Response import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils import url_utils from litellm.litellm_core_utils.prompt_templates import image_handling from litellm.litellm_core_utils.prompt_templates.image_handling import ( MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, RemoteMedia, async_convert_url_to_base64, + async_format_pdf_links, async_inline_remote_media, + async_serves_pdf, convert_url_to_base64, + format_pdf_links, inline_remote_media, + is_unformatted_pdf_link_image, + serves_pdf, ) from litellm.litellm_core_utils.url_utils import SSRFError +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @pytest.fixture(autouse=True) @@ -695,3 +708,295 @@ async def test_async_inline_remote_media_raises_image_fetch_error_when_the_fetch with pytest.raises(litellm.ImageFetchError, match="Status code: 404"): await async_inline_remote_media([{"role": "user", "content": [{"type": "image_url", "image_url": url}]}]) + + +PDF_BODY: Final = b"%PDF-1.7\n" + b"0" * 4096 +PNG_BODY: Final = b"\x89PNG\r\n\x1a\n" + b"0" * 64 + + +@dataclass(frozen=True, slots=True) +class _Served: + status: int = 200 + content_type: str | None = "application/octet-stream" + body: bytes = PDF_BODY + honors_range: bool = True + barrier: threading.Barrier | None = None + declared_length: int | None = None + hold_open_until: threading.Event | None = None + respond_after: threading.Event | None = None + drip_every: float | None = None + + +@dataclass(frozen=True, slots=True) +class _ProbeServer: + base_url: str + routes: dict[str, _Served] + seen: list[tuple[str, str | None]] + + def serve(self, path: str, served: _Served) -> str: + self.routes[path] = served + return f"{self.base_url}{path}" + + +@pytest.fixture +def probe_server(monkeypatch): + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(image_handling, "safe_get", url_utils.safe_get) + monkeypatch.setattr(litellm, "module_level_client", HTTPHandler()) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + routes: Final[dict[str, _Served]] = {} + seen: Final[list[tuple[str, str | None]]] = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + served = routes[self.path] + requested_range = self.headers.get("Range") + seen.append((self.path, requested_range)) + if served.barrier is not None: + served.barrier.wait() + if served.respond_after is not None: + served.respond_after.wait(timeout=30) + ranged = served.honors_range and requested_range == "bytes=0-1023" + body = served.body[:1024] if ranged else served.body + self.send_response(206 if ranged and served.status == 200 else served.status) + if served.content_type is not None: + self.send_header("Content-Type", served.content_type) + self.send_header("Content-Length", str(served.declared_length or len(body))) + self.end_headers() + if served.drip_every is None: + self.wfile.write(body) + else: + for byte in body: + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(served.drip_every) + if served.hold_open_until is not None: + served.hold_open_until.wait(timeout=30) + + def log_message(self, *args): + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield _ProbeServer(f"http://127.0.0.1:{server.server_port}", routes, seen) + server.shutdown() + server.server_close() + + +async def _serves_pdf(mode: str, url: str, **kwargs) -> bool: + return serves_pdf(url, **kwargs) if mode == "sync" else await async_serves_pdf(url, **kwargs) + + +async def _format_pdf_links(mode, messages): + if mode == "sync": + return format_pdf_links(messages, should_probe=is_unformatted_pdf_link_image) + return await async_format_pdf_links(messages, should_probe=is_unformatted_pdf_link_image) + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_asks_for_only_the_leading_kilobyte(probe_server, mode): + url = probe_server.serve("/files/report.pdf", _Served()) + + assert await _serves_pdf(mode, url) is True + assert probe_server.seen == [("/files/report.pdf", "bytes=0-1023")] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize( + ("served", "expected"), + ( + (_Served(content_type="application/pdf", body=PNG_BODY), True), + (_Served(content_type="Application/X-PDF; charset=binary", body=b""), True), + (_Served(content_type="binary/octet-stream", body=PDF_BODY), True), + (_Served(content_type=None, body=PDF_BODY), True), + (_Served(content_type="application/octet-stream", body=b"\n" * 1000 + PDF_BODY), True), + (_Served(content_type="application/octet-stream", body=b"\n" * 1024 + PDF_BODY), False), + (_Served(content_type="binary/octet-stream", body=PNG_BODY), False), + (_Served(content_type="image/png", body=PDF_BODY), False), + (_Served(content_type="text/html", body=b"%PDF-"), False), + (_Served(status=404, content_type="application/pdf"), False), + (_Served(honors_range=False), True), + ), + ids=( + "served-pdf", + "served-x-pdf", + "generic-type-pdf-bytes", + "no-type-pdf-bytes", + "signature-inside-the-first-kilobyte", + "signature-past-the-first-kilobyte", + "generic-type-png-bytes", + "served-png", + "served-html", + "not-found", + "range-ignored", + ), +) +async def test_serves_pdf_trusts_a_specific_served_type_and_sniffs_a_generic_one(probe_server, mode, served, expected): + url = probe_server.serve(f"/{uuid.uuid4()}/report.pdf", served) + + assert await _serves_pdf(mode, url) is expected + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_makes_no_request_when_url_downloads_are_disabled(probe_server, monkeypatch, mode): + monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0) + url = probe_server.serve("/report.pdf", _Served(content_type="application/pdf")) + + assert await _serves_pdf(mode, url) is False + assert probe_server.seen == [] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_is_false_when_the_host_refuses_the_connection(probe_server, mode): + assert await _serves_pdf(mode, "http://127.0.0.1:1/report.pdf") is False + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_is_false_without_a_request_when_the_ssrf_check_blocks_the_link(probe_server, monkeypatch, mode): + monkeypatch.setattr(litellm, "user_url_validation", True) + url = probe_server.serve("/report.pdf", _Served(content_type="application/pdf")) + + assert await _serves_pdf(mode, url) is False + assert probe_server.seen == [] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_stops_reading_after_the_leading_kilobyte_when_the_host_ignores_range(probe_server, mode): + rest_of_body_sent = threading.Event() + url = probe_server.serve( + "/report.pdf", _Served(honors_range=False, declared_length=1024**3, hold_open_until=rest_of_body_sent) + ) + + try: + assert await _serves_pdf(mode, url) is True + finally: + rest_of_body_sent.set() + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_gives_up_within_its_timeout_when_the_host_sends_one_byte_at_a_time(probe_server, mode): + url = probe_server.serve("/report.pdf", _Served(body=b"0" * 1024, honors_range=False, drip_every=0.02)) + started = time.monotonic() + + assert await _serves_pdf(mode, url, timeout=0.5) is False + assert time.monotonic() - started < 3 + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_reuses_a_link_verdict_instead_of_probing_the_link_again(probe_server, mode): + pdf_link = probe_server.serve("/a/report.pdf", _Served()) + png_link = probe_server.serve("/a/page.pdf", _Served(content_type="image/png", body=PNG_BODY)) + + verdicts = [await _serves_pdf(mode, url) for url in (pdf_link, png_link, pdf_link, png_link)] + + assert verdicts == [True, False, True, False] + assert [path for path, _ in probe_server.seen] == ["/a/report.pdf", "/a/page.pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_remembers_a_timed_out_probe_so_the_next_turn_does_not_wait_again(probe_server, mode): + host_answers = threading.Event() + url = probe_server.serve("/report.pdf", _Served(respond_after=host_answers)) + + try: + first = await _serves_pdf(mode, url, timeout=0.3) + started = time.monotonic() + second = await _serves_pdf(mode, url, timeout=0.3) + second_took = time.monotonic() - started + finally: + host_answers.set() + + assert (first, second) == (False, False) + assert second_took < 0.1 + assert probe_server.seen == [("/report.pdf", "bytes=0-1023")] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("status", (404, 503)) +async def test_serves_pdf_probes_a_link_again_a_minute_after_its_host_answered_an_error( + probe_server, monkeypatch, mode, status +): + clock: Final = MagicMock(return_value=1_000.0) + monkeypatch.setattr(image_handling, "pdf_probe_verdicts", InMemoryCache(clock=clock)) + url = probe_server.serve("/report.pdf", _Served(status=status)) + + while_down = await _serves_pdf(mode, url) + probe_server.serve("/report.pdf", _Served()) + clock.return_value = 1_030.0 + within_the_minute = await _serves_pdf(mode, url) + clock.return_value = 1_061.0 + after_the_minute = await _serves_pdf(mode, url) + + assert (while_down, within_the_minute, after_the_minute) == (False, False, True) + assert len(probe_server.seen) == 2 + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_marks_only_links_that_serve_a_pdf(probe_server, mode): + pdf_link = probe_server.serve("/a/report.pdf", _Served()) + query_link = probe_server.serve("/a/download?file=summary.pdf", _Served(content_type="application/pdf")) + png_at_pdf_link = probe_server.serve("/a/page.pdf?fm=png", _Served(content_type="image/png", body=PNG_BODY)) + png_link = f"{probe_server.base_url}/a/photo.png" + messages = [ + {"role": "system", "content": "read the files"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + {"type": "image_url", "image_url": {"url": pdf_link, "detail": "high"}, "cache_control": {"type": "ephemeral"}}, + {"type": "image_url", "image_url": query_link}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + {"type": "image_url", "image_url": {"url": png_link}}, + {"type": "image_url", "image_url": {"url": pdf_link, "format": "image/png"}}, + ], + }, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}, + ] + original = copy.deepcopy(messages) + + formatted = await _format_pdf_links(mode, messages) + + assert messages == original + assert formatted == ( + messages[0], + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + { + "type": "image_url", + "image_url": {"url": pdf_link, "detail": "high", "format": "application/pdf"}, + "cache_control": {"type": "ephemeral"}, + }, + {"type": "image_url", "image_url": {"url": query_link, "format": "application/pdf"}}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + {"type": "image_url", "image_url": {"url": png_link}}, + {"type": "image_url", "image_url": {"url": pdf_link, "format": "image/png"}}, + ], + }, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link, "format": "application/pdf"}}]}, + ) + assert sorted(path for path, _ in probe_server.seen) == ["/a/download?file=summary.pdf", "/a/page.pdf?fm=png", "/a/report.pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_probes_every_link_at_once(probe_server, mode): + both_in_flight = threading.Barrier(2, timeout=5) + links = [probe_server.serve(f"/{name}.pdf", _Served(barrier=both_in_flight)) for name in ("first", "second")] + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": link}} for link in links]}] + + formatted = await _format_pdf_links(mode, messages) + + assert [part["image_url"]["format"] for part in formatted[0]["content"]] == ["application/pdf", "application/pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_leaves_messages_without_pdf_links_alone(probe_server, mode): + messages = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": f"{probe_server.base_url}/a.png"}}]}, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:application/pdf;base64,JVBERi0="}}]}, + ] + + assert await _format_pdf_links(mode, messages) == tuple(messages) + assert probe_server.seen == [] diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 13e181b2b92..e8ea0c04f6f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -14451,3 +14451,193 @@ def test_record_native_bedrock_guardrail_bills_each_call_its_own_trace_when_call (first_cost, second_cost) = costs assert second_cost == pytest.approx(2 * first_cost) assert shared_metadata == {"tag": "x"} + + +@pytest.mark.asyncio +async def test_a_redacted_video_poll_still_tells_the_spend_callbacks_the_video_finished( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, job_to_charge_once + from litellm.types.videos.main import VideoObject + + class Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: tuple[tuple[dict[str, object], object], ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.events = (*self.events, (dict(kwargs), response_obj)) + + recorder: Final = Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + logging_obj: Final = LitellmLogging( + model="custom-video", + messages=[], + stream=False, + call_type="avideo_status", + start_time=datetime.datetime.now(), + litellm_call_id="poll-1", + function_id="poll-1", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {}, "video_id": "video_abc"}, optional_params={} + ) + + await logging_obj.async_success_handler(result=VideoObject(id="video_abc", object="video", status="completed")) + + ((kwargs, response_obj),) = recorder.events + assert not isinstance(response_obj, VideoObject) + assert job_to_charge_once(kwargs, response_obj, 0.42) == ChargeOnceJob(kind="video", job_id="video_abc") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status", "output_file_id", "deferred", "charged"), + [("in_progress", None, True, False), ("completed", "file-out", False, True)], +) +async def test_a_redacted_batch_retrieve_still_tells_the_spend_callbacks_whether_the_batch_cost_is_final( + monkeypatch: pytest.MonkeyPatch, status: str, output_file_id: str | None, deferred: bool, charged: bool +) -> None: + from litellm.litellm_core_utils.batch_utils import ChargeOnceJob, defers_batch_cost, job_to_charge_once + from litellm.types.utils import LiteLLMBatch, Usage + + class Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: tuple[tuple[dict[str, object], object], ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.events = (*self.events, (dict(kwargs), response_obj)) + + recorder: Final = Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + logging_obj: Final = LitellmLogging( + model="gpt-4o-mini", + messages=[], + stream=False, + call_type="aretrieve_batch", + start_time=datetime.datetime.now(), + litellm_call_id="retrieve-1", + function_id="retrieve-1", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + + await logging_obj.async_success_handler( + result=LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + object="batch", + status=status, + output_file_id=output_file_id, + ), + batch_cost=0.42, + batch_usage=Usage(prompt_tokens=20, completion_tokens=18, total_tokens=38), + batch_models=["gpt-4o-mini"], + ) + + ((kwargs, response_obj),) = recorder.events + assert not isinstance(response_obj, LiteLLMBatch) + assert defers_batch_cost(kwargs, response_obj) is deferred + assert (job_to_charge_once(kwargs, response_obj, 0.42) == ChargeOnceJob(kind="batch", job_id="batch_abc")) is charged + + +_FAILED_ATTEMPT: Final = (Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), 0.5) + + +async def _success_row_cost_after_a_failed_attempt(stash: Callable[[LitellmLogging], None]) -> float: + logging_obj: Final = LitellmLogging( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="recovered-attempt", + function_id="recovered-attempt", + ) + logging_obj.update_environment_variables( + model="gpt-4o-mini", user="", optional_params={}, litellm_params={"metadata": {}}, custom_llm_provider="openai" + ) + stash(logging_obj) + logging_obj.carry_failed_attempt_usage() + now: Final = datetime.datetime.now() + await logging_obj.async_success_handler( + ModelResponse(model="gpt-4o-mini", usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)), + now, + now, + ) + return logging_obj.model_call_details["standard_logging_object"]["response_cost"] + + +@pytest.mark.asyncio +async def test_a_recovered_attempt_bills_a_failed_agentic_follow_up_but_not_a_failure_that_logged_its_own_row() -> None: + own_cost: Final = await _success_row_cost_after_a_failed_attempt(lambda logging_obj: None) + after_a_dropped_stream: Final = await _success_row_cost_after_a_failed_attempt( + lambda logging_obj: logging_obj.record_partial_usage_for_failure(*_FAILED_ATTEMPT) + ) + after_a_failed_follow_up: Final = await _success_row_cost_after_a_failed_attempt( + lambda logging_obj: logging_obj.record_served_rounds_for_failure((_FAILED_ATTEMPT,)) + ) + + assert own_cost > 0 + assert after_a_dropped_stream == pytest.approx(own_cost) + assert after_a_failed_follow_up == pytest.approx(own_cost + _FAILED_ATTEMPT[1]) + + +def test_standard_logging_payload_reports_the_per_call_prices_the_request_was_billed_from() -> None: + """A direct call that sets its own prices is logged at those prices, not at the catalog row's rates.""" + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + from litellm.litellm_core_utils.per_call_pricing import per_call_model_info + + rates: Final = {"input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06} + now: Final = datetime_unit_test(2026, 1, 1, 12, 0, 0) + logging_obj: Final = LitellmLogging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=now, + litellm_call_id="per-call-prices", + function_id="fn", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", user="", optional_params={}, litellm_params=rates, custom_llm_provider="openai" + ) + logging_obj.set_per_call_pricing(per_call_model_info("gpt-5.4-nano", "openai", rates, "chat")) + response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hi"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=6, total_tokens=16), + ) + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-5.4-nano", + "custom_llm_provider": "openai", + "litellm_params": rates, + "response_cost": logging_obj.response_cost_calculator(result=response), + }, + init_response_obj=response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["response_cost"] == pytest.approx(10 * 3e-06 + 6 * 7e-06) + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert (model_map_value["input_cost_per_token"], model_map_value["output_cost_per_token"]) == (3e-06, 7e-06) diff --git a/tests/unit/litellm_core_utils/test_model_param_helper.py b/tests/unit/litellm_core_utils/test_model_param_helper.py index 6293c748505..df37139bc7b 100644 --- a/tests/unit/litellm_core_utils/test_model_param_helper.py +++ b/tests/unit/litellm_core_utils/test_model_param_helper.py @@ -1,5 +1,9 @@ +from collections.abc import Sequence +from typing import Final +import pytest +from litellm.caching.caching import Cache from litellm.litellm_core_utils.model_param_helper import ModelParamHelper @@ -46,3 +50,18 @@ def test_langfuse_model_parameters_no_secret_leakage(): assert "secret_fields" not in sanitized assert "authorization" not in sanitized assert "headers" not in sanitized + + +@pytest.mark.parametrize( + "hint,first_value,second_value", + [("languages", ("en",), ("fr",)), ("keywords", ("LiteLLM",), ("Kubernetes",))], +) +def test_transcription_cache_key_separates_recognition_hints( + hint: str, first_value: Sequence[str], second_value: Sequence[str] +) -> None: + cache: Final = Cache() + request: Final = {"model": "gpt-4o-transcribe", "file": "meeting.wav", "response_format": "json"} + + assert cache.get_cache_key(**request, **{hint: first_value}) != cache.get_cache_key( + **request, **{hint: second_value} + ) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index ba76138a268..eec1dfcf459 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -30,6 +30,8 @@ from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTransc from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter + +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict @@ -512,6 +514,76 @@ async def test_backend_to_client_beta_receives_normalized_events(): assert sent["output_index"] == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("session.created", "session.updated")) +async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_session_shape(event_type: str) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + { + "type": event_type, + "event_id": "e1", + "session": { + "type": "realtime", + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "output_modalities": ["audio"], + "max_output_tokens": "inf", + "tools": [], + "audio": { + "input": { + "format": {"type": "audio/pcmu"}, + "transcription": {"model": "gpt-transcribe"}, + "noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "marin", "speed": 1.1}, + }, + }, + } + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.backend_to_client_send_messages() + + sent: Final = json.loads(client_ws.send_text.call_args_list[0].args[0]) + assert sent["type"] == event_type + assert sent["session"] == { + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "modalities": ["audio", "text"], + "max_response_output_tokens": "inf", + "tools": [], + "input_audio_format": "g711_ulaw", + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + "output_audio_format": "pcm16", + "voice": "marin", + "speed": 1.1, + }, ( + "Beta and GA session fields, checked 2026-10-20: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/session.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_session_create_response.py" + ) + + @pytest.mark.asyncio async def test_backend_to_client_stores_normalized_events_for_logging(): client_ws = MagicMock() @@ -648,6 +720,50 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): assert "audio" not in session +@pytest.mark.asyncio +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_session_update_omits_session_type(beta_client: bool) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "input_audio_format": "g711_ulaw", + "output_audio_format": "pcm16", + "audio": {"output": {"language": "fr"}}, + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + translation_session=True, + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + assert "type" not in sent_to_backend["session"] + assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + assert sent_to_backend["session"]["audio"]["input"]["format"]["type"] == "audio/pcmu", ( + "OpenAI Realtime audio formats, checked 2026-10-06: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_audio_formats.py" + ) + assert sent_to_backend["session"]["audio"]["output"]["format"]["type"] == "audio/pcm" + + def test_translate_event_to_beta_renames_delta_types(): ev = RealTimeStreaming._translate_event_to_beta( {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} @@ -1158,6 +1274,153 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): assert streaming._is_transcription_session is True +@pytest.mark.asyncio +async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + translation_session=True, + ) + + sent: Final = await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } + }, + }, + } + ) + ) + assert sent is False + backend_ws.send.assert_not_awaited() + assert streaming._is_transcription_session is False + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["type"] == "error" + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 + + +def _translation_transcription_update(transcription: Mapping[str, object]) -> str: + return json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"input": {"transcription": transcription}}}, + } + ) + + +@pytest.mark.asyncio +async def test_translation_session_update_authorizes_each_new_nested_transcription_model() -> None: + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth( + models=["gpt-realtime-translate", "gpt-realtime-whisper", "gpt-live-transcribe"] + ), + translation_session=True, + ) + + for transcription in ( + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"language": "de"}, + ): + await streaming._send_to_backend(_translation_transcription_update(transcription)) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"model": "gpt-live-transcribe", "language": "de"}, + ] + + +@pytest.mark.asyncio +async def test_translation_session_update_refuses_a_later_nested_transcription_model_the_key_cannot_call() -> None: + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + translation_session=True, + ) + + first_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-realtime-whisper", "language": "en"}) + ) + second_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-live-transcribe", "language": "fr"}) + ) + + assert (first_sent, second_sent) == (True, False) + assert backend_ws.send.await_count == 1 + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 + + +@pytest.mark.asyncio +async def test_translation_session_update_sends_the_deployment_model_for_a_nested_transcription_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + model_list: Final = [ + { + "model_name": "whisper-alias", + "litellm_params": { + "model": "azure/qa-whisper-deployment", + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "whisper-alias"]), + translation_session=True, + ) + + await streaming._send_to_backend(_translation_transcription_update({"model": "whisper-alias", "language": "en"})) + await streaming._send_to_backend(_translation_transcription_update({"language": "fr"})) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "qa-whisper-deployment", "language": "en"}, + {"model": "qa-whisper-deployment", "language": "fr"}, + ] + + @pytest.mark.asyncio async def test_normal_realtime_session_keeps_nested_transcription_model(): backend_ws = MagicMock() @@ -1847,8 +2110,7 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(monke @pytest.mark.asyncio async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pytest.MonkeyPatch): """ - Test that _has_realtime_guardrails returns True for a guardrail configured with - pre_call mode (not just realtime_input_transcription). + A pre_call guardrail gates typed user messages and tool output, never audio VAD. """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -1870,10 +2132,6 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pyt logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) - assert streaming._has_realtime_guardrails() is True, ( - "pre_call guardrail should be recognized as a realtime guardrail" - ) - # pre_call-only guardrails gate typed user messages / tool output, not audio VAD. assert streaming._has_audio_transcription_guardrails() is False, ( "pre_call-only guardrail must not disable server_vad auto-response" ) @@ -2024,7 +2282,6 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monke }, ) - assert streaming._has_realtime_guardrails() is True assert streaming._has_audio_transcription_guardrails() is False @@ -3561,6 +3818,444 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] +@pytest.mark.parametrize( + "event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"] +) +def test_translation_audio_duration_is_finalized_once(event_type: str): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + payload = base64.b64encode(bytes(48000)).decode() + streaming._capture_translation_output_audio({"type": event_type, "delta": payload}) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_bytes", (0, 48000)) +@pytest.mark.parametrize("format_event_type", ("session.created", "session.updated")) +@pytest.mark.parametrize( + "audio_format,bytes_per_second", + ( + ("pcm16", 48000), + ("g711_ulaw", 8000), + ("g711_alaw", 8000), + ({"type": "audio/pcm", "rate": 24000}, 48000), + ({"type": "audio/pcmu", "rate": 8000}, 8000), + ({"type": "audio/pcma", "rate": 8000}, 8000), + ({"type": "audio/pcmu"}, 8000), + ({"type": "audio/pcma"}, 8000), + ), +) +async def test_translation_disconnect_bills_sent_input_audio( + output_bytes: int, format_event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int +) -> None: + import base64 + + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), + backend_ws=backend, + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + format_event: Final = { + "type": format_event_type, + "session": {"audio": {"input": {"format": audio_format}}}, + } + streaming._capture_translation_output_audio(format_event) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(bytes(bytes_per_second * 2)).decode(), + } + ) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} + ) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + assert streaming.messages == [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": output_bytes / 48000}, + } + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("rate", (24000, 1, 0, -1, float("nan"), float("inf"))) +async def test_translation_client_format_updates_do_not_change_billed_duration(rate: int | float) -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"input": {"format": "g711_ulaw"}, "output": {"format": "g711_ulaw"}}}, + } + ) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "audio": { + "input": {"format": {"type": "audio/pcm", "rate": rate}}, + "output": {"format": {"type": "audio/pcm", "rate": rate}}, + } + }, + } + ) + ) + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.asyncio +async def test_translation_confirmed_format_changes_preserve_previous_audio_duration() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(48000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + { + "type": "session.updated", + "session": {"audio": {"input": {"format": "g711_alaw"}, "output": {"format": "g711_alaw"}}}, + } + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + ) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.parametrize("rate", (1, 0, -1, float("nan"), float("inf"), 10**400)) +def test_translation_invalid_provider_format_preserves_audio_duration(rate: int | float) -> None: + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=MagicMock(), logging_obj=_mock_logging(), translation_session=True + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + streaming._capture_translation_output_audio(output_event) + streaming._capture_translation_output_audio( + {"type": "session.updated", "session": {"audio": {"output": {"format": {"type": "audio/pcm", "rate": rate}}}}} + ) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [{"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}}] + + +@pytest.mark.asyncio +async def test_translation_failed_audio_send_is_not_billed() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock(side_effect=RuntimeError("send failed")) + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + + with pytest.raises(RuntimeError, match="send failed"): + await streaming._send_to_backend(json.dumps({"type": "session.input_audio_buffer.append", "audio": "AAAA"})) + streaming._finalize_translation_usage() + + assert streaming.messages == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("default_on,requested", ((False, False), (True, False), (False, True))) +@pytest.mark.parametrize( + "event_hook", + (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call), +) +async def test_translation_rejects_applicable_guardrails_before_forwarding_audio( + monkeypatch: pytest.MonkeyPatch, default_on: bool, requested: bool, event_hook: GuardrailEventHooks +) -> None: + guardrail: Final = CustomGuardrail(guardrail_name="speech-policy", event_hook=event_hook, default_on=default_on) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(4800)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(4800)).decode()} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[input_event, ConnectionClosed(None, None)]) + client.send_text = AsyncMock() + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(output_event), ConnectionClosed(None, None)]) + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=_mock_logging(), + translation_session=True, + request_data={"litellm_metadata": {"guardrails": ["speech-policy"] if requested else []}}, + ) + + await streaming.bidirectional_forward() + + messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + if default_on or requested: + assert messages[0]["error"]["type"] == "guardrail_error" + client.close.assert_awaited_once_with( + code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + ) + backend.send.assert_not_awaited() + else: + assert output_event in messages + backend.send.assert_awaited_once_with(input_event) + + +@pytest.mark.asyncio +async def test_translation_for_a_key_opted_out_of_a_legacy_callback_guardrail_gets_no_realtime_session_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", [CustomGuardrail(guardrail_name="legacy-callback")]) + session_update: Final = json.dumps({"type": "session.update", "session": {"audio": {"output": {"language": "es"}}}}) + session_created: Final = {"type": "session.created", "session": {"type": "translation"}} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[session_update, ConnectionClosed(None, None)]) + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(session_created), ConnectionClosed(None, None)]) + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=_mock_logging(), + translation_session=True, + request_data={"litellm_metadata": {"user_api_key_metadata": {"disable_global_guardrails": True}}}, + ) + + await streaming.bidirectional_forward() + + assert tuple(call.args[0] for call in backend.send.await_args_list) == (session_update,) + assert session_created in tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retain_close", (False, True)) +@pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) +async def test_translation_terminal_usage_fills_only_missing_input_duration( + monkeypatch: pytest.MonkeyPatch, retain_close: bool, reported_input: float | None, expected_input: float +) -> None: + import base64 + + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*" if retain_close else None) + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + close_event: Final = { + "type": "session.closed", + "usage": { + "type": "duration", + "output_seconds": 0.5, + **({"input_seconds": reported_input} if reported_input is not None else {}), + }, + } + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert sum(item.get("input_seconds") or 0.0 for item in usage) == expected_input + assert sum(item.get("output_seconds") or 0.0 for item in usage) == 0.5 + + +def test_translation_audio_duration_uses_session_output_format(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu", "rate": 8000}}}}, + } + ) + streaming._capture_translation_output_audio( + { + "type": "session.output_audio.delta", + "delta": base64.b64encode(bytes(8000)).decode(), + } + ) + streaming._finalize_translation_usage() + + assert streaming.messages[-1]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_does_not_duplicate_provider_duration_usage(): + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + + +def test_translation_prefers_provider_duration_over_audio_byte_estimate(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} + + +@pytest.mark.parametrize( + ("output_audio_bytes", "expected_usage"), + [ + (0, {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.0}), + (48000, {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0}), + ], +) +def test_translation_preserves_input_only_provider_usage( + output_audio_bytes: int, expected_usage: Mapping[str, str | float] +) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_audio_bytes)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == expected_usage + + +def test_translation_retained_input_only_close_event_bills_captured_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage_events: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert len(usage_events) == 2 + assert sum(usage.get("input_seconds", 0.0) for usage in usage_events) == 0.25 + assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 + + +@pytest.mark.parametrize("malformed_output", [-1.0, float("nan"), float("inf"), 10**1000]) +def test_translation_malformed_provider_duration_uses_captured_audio(malformed_output: float | int) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + + streaming._capture_translation_output_audio( + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}, + } + ) + streaming._finalize_translation_usage() + + close_events: Final = tuple(event for event in streaming.messages if event.get("type") == "session.closed") + assert len(close_events) == 1 + assert close_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() @@ -4200,6 +4895,48 @@ async def test_upstream_close_after_relayed_events_still_logs_the_session_as_suc client_ws.close.assert_awaited_once_with(code=1008, reason=_UPSTREAM_REFUSAL) +def _model_not_found_error_event() -> bytes: + return json.dumps( + { + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "model_not_found", + "message": "The model `gpt-realtime-translate-typo` does not exist", + }, + } + ).encode() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("close_code", (1000, 1008)) +async def test_upstream_error_before_any_session_logs_a_failure_not_a_success(close_code: int): + upstream_close: Final = ConnectionClosed(Close(close_code, ""), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), _backend_ws_closing_with(_model_not_found_error_event(), upstream_close) + ) + + await session.run() + + assert session.logging.logged_failures == (upstream_close,) + assert session.logging.logged_sessions == () + + +@pytest.mark.asyncio +async def test_upstream_error_after_session_created_still_logs_the_session_as_success(): + session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + upstream_close: Final = ConnectionClosed(Close(1000, ""), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), + _backend_ws_closing_with(session_created, _model_not_found_error_event(), upstream_close), + ) + + await session.run() + + assert len(session.logging.logged_sessions) == 1 + assert session.logging.logged_failures == () + + @pytest.mark.asyncio async def test_upstream_closing_while_a_client_message_is_forwarded_still_reaches_the_client(): upstream_close: Final = ConnectionClosed(Close(1008, _UPSTREAM_REFUSAL), None) @@ -5029,3 +5766,264 @@ async def test_blocked_transcript_is_removed_from_the_conversation_before_the_re sent: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] assert [event["type"] for event in sent] == expected_frames assert all(event["item_id"] == "item_blocked" for event in sent if event["type"] == "conversation.item.delete") + + +@pytest.mark.asyncio +async def test_beta_client_bridged_to_a_ga_backend_sends_assistant_text_items_as_ga_output_text() -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "The forecast is sunny"}], + }, + } + ), + json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Thanks"}]}, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj: Final = _mock_logging() + logging_obj.pre_call = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.client_ack_messages() + + sent_items: Final = [json.loads(call.args[0])["item"] for call in backend_ws.send.call_args_list] + assert [(item["role"], item["content"]) for item in sent_items] == [ + ("assistant", [{"type": "output_text", "text": "The forecast is sunny"}]), + ("user", [{"type": "input_text", "text": "Thanks"}]), + ], ( + "Beta and GA item content types, checked 2026-10-28: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/conversation_item_content.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_conversation_item_assistant_message.py" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("response.created", "response.done")) +async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_response_shape(event_type: str) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + { + "type": event_type, + "event_id": "e1", + "response": { + "object": "realtime.response", + "id": "resp_1", + "status": "completed", + "conversation_id": "conv_1", + "output_modalities": ["audio"], + "max_output_tokens": "inf", + "audio": {"output": {"format": {"type": "audio/pcmu"}, "voice": "marin"}}, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_audio", "transcript": "Sunny"}], + } + ], + }, + } + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.backend_to_client_send_messages() + + sent: Final = json.loads(client_ws.send_text.call_args_list[0].args[0]) + assert sent["type"] == event_type + assert sent["response"] == { + "object": "realtime.response", + "id": "resp_1", + "status": "completed", + "conversation_id": "conv_1", + "modalities": ["audio", "text"], + "max_output_tokens": "inf", + "output_audio_format": "g711_ulaw", + "voice": "marin", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "audio", "transcript": "Sunny"}]}], + }, ( + "Beta and GA response fields, checked 2026-10-28: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/realtime_response.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_response.py" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", (False, True)) +async def test_beta_client_bridged_to_a_ga_backend_sends_ga_session_and_response_fields(drop_params: bool) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "instructions": "Answer briefly", + "temperature": 0.7, + "max_response_output_tokens": 300, + "speed": 1.1, + "input_audio_noise_reduction": {"type": "near_field"}, + }, + } + ), + json.dumps( + { + "type": "response.create", + "response": { + "modalities": ["text", "audio"], + "voice": "marin", + "output_audio_format": "g711_ulaw", + "temperature": 0.7, + "max_response_output_tokens": 200, + "instructions": "Say hi", + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj: Final = _mock_logging() + logging_obj.pre_call = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + backend_uses_beta_protocol=False, + bridges_beta_client_to_ga=True, + drop_params=drop_params, + ) + + await streaming.client_ack_messages() + + sent: Final = { + event["type"]: event for event in (json.loads(call.args[0]) for call in backend_ws.send.call_args_list) + } + temperature: Final = {} if drop_params else {"temperature": 0.7} + assert sent["session.update"]["session"] == { + "type": "realtime", + "instructions": "Answer briefly", + "max_output_tokens": 300, + "audio": {"input": {"noise_reduction": {"type": "near_field"}}, "output": {"speed": 1.1}}, + **temperature, + } + assert sent["response.create"]["response"] == { + "output_modalities": ["audio"], + "audio": {"output": {"voice": "marin", "format": {"type": "audio/pcmu"}}}, + "max_output_tokens": 200, + "instructions": "Say hi", + **temperature, + } + + +@pytest.mark.asyncio +async def test_gemini_live_session_update_keeps_the_client_output_token_cap() -> None: + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = _ga_client_ws() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": {"temperature": 0.4, "max_response_output_tokens": 77, "modalities": ["text"]}, + } + ), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, + Exception("connection closed"), + ] + ) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + _mock_logging(), + provider_config=GeminiRealtimeConfig(), + model="gemini-2.0-flash-live-001", + ) + + await streaming.client_ack_messages() + + setup_frame, content_frame = (json.loads(call.args[0]) for call in backend_ws.send.call_args_list) + assert "clientContent" in content_frame + assert setup_frame["setup"]["generationConfig"] == { + "temperature": 0.4, + "maxOutputTokens": 77, + "responseModalities": ["TEXT"], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "event_normalizer", + (pytest.param(XAIRealtimeNormalizer(), id="xai"), pytest.param(None, id="openai-ga-client")), +) +async def test_unbridged_backend_receives_session_update_and_response_create_as_the_client_sent_them( + event_normalizer: XAIRealtimeNormalizer | None, +) -> None: + client_events: Final = ( + { + "type": "session.update", + "session": { + "type": "realtime", + "instructions": "Answer briefly", + "temperature": 0.7, + "max_response_output_tokens": 300, + "speed": 1.1, + "input_audio_noise_reduction": {"type": "near_field"}, + }, + }, + { + "type": "response.create", + "response": { + "modalities": ["text", "audio"], + "voice": "marin", + "output_audio_format": "g711_ulaw", + "temperature": 0.7, + "max_response_output_tokens": 200, + }, + }, + ) + client_ws: Final = _ga_client_ws() + client_ws.receive_text = AsyncMock( + side_effect=[*(json.dumps(event) for event in client_events), Exception("connection closed")] + ) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, _mock_logging(), event_normalizer=event_normalizer, drop_params=True + ) + + await streaming.client_ack_messages() + + assert tuple(json.loads(call.args[0]) for call in backend_ws.send.call_args_list) == client_events diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 0aee61d8365..491ea5315ff 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -54,9 +54,25 @@ SIGNATURE: Final = "opaque-thought-signature" def _provider_fields() -> dict[str, object]: return { "reasoning_content": SECRET, + "reasoning": SECRET, + "reasoning_text": SECRET, + "reasoning_details": [{"type": "reasoning.text", "text": SECRET, "index": 0, "format": "unknown"}], "thinking_blocks": [{"type": "thinking", "thinking": SECRET}], "reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}], + "reasoningContent": {"reasoningText": {"text": SECRET}}, + "annotations": [{"type": "file", "file": {"name": "notes.pdf", "content": [{"type": "text", "text": SECRET}]}}], "citations": [{"cited_text": SECRET}], + "citationsContent": [{"citations": [{"location": {}}], "content": [{"text": SECRET}]}], + "citation": {"type": "char_location", "cited_text": SECRET}, + "web_search_calls": [{"type": "web_search_call", "action": {"type": "search", "query": SECRET}}], + "web_search_results": [{"type": "web_search_tool_result", "content": [{"title": SECRET}]}], + "tool_results": [{"type": "web_fetch_tool_result", "content": {"text": SECRET}}], + "code_interpreter_results": [{"stdout": SECRET}], + "search_results": [{"object": "vector_store.search_results.page", "data": [{"content": [{"text": SECRET}]}]}], + "rerank_results": {"results": [{"document": {"text": SECRET}}]}, + "executed_tools": [{"type": "search", "search_results": {"results": [{"content": SECRET}]}}], + "custom_outputs": {"answer": SECRET}, + "tool_search_items": [{"type": "tool_search_output", "tools": [{"description": SECRET}]}], "compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}], "compaction_start": {"type": "compaction", "content": SECRET}, "compaction_delta": {"type": "compaction_delta", "content": SECRET}, @@ -112,6 +128,57 @@ def test_redaction_reuses_message_rules_for_request_body_and_provider_fields(sha assert result == original_result and params == original_params +def _reasoning_items() -> list[dict[str, object]]: + return [{"type": "reasoning", "id": "rs_redaction", "summary": [{"type": "summary_text", "text": SECRET}]}] + + +def _message_response() -> ModelResponse: + return ModelResponse( + choices=[{"index": 0, "message": {"role": "assistant", "content": "answer", "reasoning_items": _reasoning_items()}}] + ) + + +def _delta_choices() -> dict[str, object]: + return {"choices": [{"index": 0, "delta": {"content": "answer", "reasoning_items": _reasoning_items()}}]} + + +@pytest.mark.parametrize( + ("details", "result"), + [ + pytest.param({}, _message_response(), id="message_object"), + pytest.param({}, _message_response().model_dump(), id="message_dict"), + pytest.param({}, _delta_choices(), id="delta_dict"), + pytest.param( + {"stream": True, "complete_streaming_response": ModelResponseStream(**_delta_choices())}, + None, + id="stream_chunk", + ), + pytest.param({"standard_logging_object": {"response": _message_response().model_dump()}}, None, id="logged"), + pytest.param({"standard_logging_object": {"response": _delta_choices()}}, None, id="logged_delta"), + ], +) +def test_redaction_hides_reasoning_item_summaries(details: dict[str, object], result: object) -> None: + assert SECRET in str(details) + str(result) + + redacted: Final = perform_redaction(details, result) + + assert SECRET not in str(redacted) and SECRET not in str(details), (redacted, details) + + +def test_an_opted_out_callback_gets_a_text_completion_with_its_answer_redacted() -> None: + logprobs: Final = {"tokens": [SECRET], "token_logprobs": [-0.1], "text_offset": [0], "top_logprobs": [{SECRET: -0.1}]} + response: Final = TextCompletionResponse( + choices=[{"index": 0, "text": SECRET, "logprobs": logprobs, "finish_reason": "stop"}] + ) + + redacted: Final = redacted_response_obj_for_custom_logger(response, CustomLogger(message_logging=False)) + + assert isinstance(redacted, TextCompletionResponse) + assert redacted.choices[0].text == "redacted-by-litellm" + assert SECRET not in redacted.model_dump_json() + assert response.choices[0].text == SECRET, "the caller's own response is not redacted" + + def test_redaction_redacts_reasoning_copied_into_hidden_params_in_place() -> None: hidden_params: Final = {"model_id": "deployment", "provider_specific_fields": _provider_fields()} original_hidden_params: Final = copy.deepcopy(hidden_params) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 8ef118154d1..d0b837646b4 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1246,7 +1246,7 @@ def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_toke @pytest.mark.parametrize( "model", - ["together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "fireworks_ai/accounts/fireworks/models/deepseek-v2p5"], + ["deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "fireworks_ai/accounts/fireworks/models/deepseek-v2p5"], ) def test_token_counter_skips_the_deepseek_tokenizer_for_deepseek_models_built_on_other_vocabularies(model: str): assert litellm.token_counter(model=model, text=_CODE_SAMPLE) != _deepseek_token_count(_CODE_SAMPLE) @@ -2875,7 +2875,7 @@ def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deploym @pytest.mark.parametrize( - "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"] + "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat"] ) def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 @@ -3594,7 +3594,6 @@ def test_token_counter_counts_unlisted_names_of_newer_openai_releases_with_o200k "azure/my-deployment", "azure/ada", "azure/mistral-large-latest", - "azure/command-r-plus", "azure_ai/mistral-large-latest", ], ) diff --git a/tests/unit/litellm_core_utils/test_url_utils.py b/tests/unit/litellm_core_utils/test_url_utils.py index 9b29eac208e..958960af27a 100644 --- a/tests/unit/litellm_core_utils/test_url_utils.py +++ b/tests/unit/litellm_core_utils/test_url_utils.py @@ -16,11 +16,25 @@ from litellm.litellm_core_utils.url_utils import ( encode_url_path_segment, encode_url_path_segments, ensure_api_route, + is_https_url_with_host, join_api_base, validate_url, ) +@pytest.mark.parametrize( + ("url", "expected"), + ( + ("https://example.com/path", True), + ("http://example.com/path", False), + ("https:///path", False), + ("https://[", False), + ), +) +def test_is_https_url_with_host(url: str, expected: bool) -> None: + assert is_https_url_with_host(url) is expected + + @pytest.fixture def mock_dns_public(monkeypatch): """Resolve any hostname to 93.184.216.34 (public).""" diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index ce14adab4a0..e81ee2a4e02 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2,6 +2,7 @@ import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from itertools import chain +from collections.abc import Callable from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -13,7 +14,8 @@ from openai.types.chat import ChatCompletionChunk import litellm from litellm._uuid import uuid -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.prompt_templates import image_handling from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope @@ -253,6 +255,7 @@ def test_anthropic_tool_call_error_is_mapped_from_the_provider_response(): async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" + https_pdf_link = f"https://img.example/{uuid.uuid4()}/page.pdf?w=600" captured = {} def handle(request): @@ -282,6 +285,7 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop {"type": "text", "text": "What colour is this?"}, {"type": "image_url", "image_url": {"url": http_image_url}}, {"type": "image_url", "image_url": {"url": https_image_url}}, + {"type": "image_url", "image_url": {"url": https_pdf_link}}, ], } ], @@ -290,11 +294,193 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop ) assert response.choices[0].message.content == "Green" - assert async_only_image_fetch.fetched == [http_image_url] + assert async_only_image_fetch.fetched == [http_image_url, https_pdf_link] sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"] assert sources == [ {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}, {"type": "url", "url": https_image_url}, + {"type": "url", "url": https_pdf_link}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("download_limit_mb", (0, 50), ids=("downloads-disabled", "link-unreachable")) +async def test_anthropic_async_completion_sends_a_pdf_path_link_it_cannot_download_as_a_link( + monkeypatch: pytest.MonkeyPatch, download_limit_mb: int +) -> None: + pdf_link: Final = f"https://img.example/{uuid.uuid4()}/page.pdf?fm=png" + captured: Final[dict[str, object]] = {} + + async def unreachable(client: object, url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(404, request=httpx.Request("GET", url)) + + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A chart"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "async_safe_get", unreachable) + monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", download_limit_mb) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + + response: Final = await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}], + client=client, + api_key="sk-ant-unit-test", + ) + + assert response.choices[0].message.content == "A chart" + assert captured["body"]["messages"][0]["content"] == [{"type": "image", "source": {"type": "url", "url": pdf_link}}] + + +@pytest.mark.asyncio +async def test_anthropic_async_completion_gives_up_on_a_pdf_link_within_the_non_llm_connection_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://unreachable.example/{uuid.uuid4()}/report.pdf" + fetch_timeouts: Final[list[object]] = [] + captured: Final[dict[str, object]] = {} + + async def unreachable(client: object, url: str, **kwargs: object) -> httpx.Response: + fetch_timeouts.append(kwargs.get("timeout")) + raise httpx.ConnectTimeout("no route to host") + + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A chart"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "async_safe_get", unreachable) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + + await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [{"type": "image", "source": {"type": "url", "url": pdf_link}}] + assert fetch_timeouts == [NON_LLM_CONNECTION_TIMEOUT] + + +def _anthropic_reply(captured: dict[str, object]) -> Callable[[httpx.Request], httpx.Response]: + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A report"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + return handle + + +def _served_pdf_slice(url: str) -> httpx.Response: + return httpx.Response( + 206, content=b"%PDF-1.7\n", headers={"content-type": "binary/octet-stream"}, request=httpx.Request("GET", url) + ) + + +@pytest.mark.asyncio +async def test_anthropic_async_completion_sends_a_link_that_serves_a_pdf_as_a_url_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://files.example/{uuid.uuid4()}/download?file=report.pdf" + probes: Final[list[object]] = [] + captured: Final[dict[str, object]] = {} + + async def serve_pdf_slice(client: object, url: str, **kwargs: object) -> httpx.Response: + probes.append(kwargs.get("headers")) + return _served_pdf_slice(url) + + monkeypatch.setattr(image_handling, "async_safe_get", serve_pdf_slice) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(_anthropic_reply(captured))) + + await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": pdf_link}, "cache_control": {"type": "ephemeral"}} + ], + } + ], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [ + {"type": "document", "source": {"type": "url", "url": pdf_link}, "cache_control": {"type": "ephemeral"}} + ] + assert probes == [{"Range": "bytes=0-1023"}] + + +def test_anthropic_sync_completion_sends_a_link_that_serves_a_pdf_as_a_url_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://files.example/{uuid.uuid4()}/report.pdf" + png_at_pdf_link: Final = f"https://files.example/{uuid.uuid4()}/page.pdf?fm=png" + captured: Final[dict[str, object]] = {} + + def serve(client: object, url: str, **kwargs: object) -> httpx.Response: + if url == pdf_link: + return _served_pdf_slice(url) + return httpx.Response(200, content=b"\x89PNG", headers={"content-type": "image/png"}, request=httpx.Request("GET", url)) + + monkeypatch.setattr(image_handling, "safe_get", serve) + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_anthropic_reply(captured)))) + + litellm.completion( + model="anthropic/claude-haiku-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": pdf_link}}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + ], + } + ], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [ + {"type": "document", "source": {"type": "url", "url": pdf_link}}, + {"type": "image", "source": {"type": "url", "url": png_at_pdf_link}}, ] @@ -2954,6 +3140,118 @@ def test_streamed_json_mode_truncated_output_is_passed_through_untouched(): assert "".join(contents) == '{"name": "json_tool_call", "parameters": {"tier": 3' +def _block_events( + index: int, block: dict[str, object], deltas: tuple[dict[str, object], ...] +) -> tuple[dict[str, object], ...]: + return ( + {"type": "content_block_start", "index": index, "content_block": block}, + *({"type": "content_block_delta", "index": index, "delta": delta} for delta in deltas), + {"type": "content_block_stop", "index": index}, + ) + + +def _message_events(*blocks: tuple[dict[str, object], tuple[dict[str, object], ...]]) -> tuple[dict[str, object], ...]: + starts_and_deltas: Final = tuple( + chain.from_iterable(_block_events(index, block, deltas) for index, (block, deltas) in enumerate(blocks)) + ) + return ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + *starts_and_deltas, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 40}, + }, + ) + + +_THINKING_BLOCK: Final = ( + {"type": "thinking", "thinking": "", "signature": ""}, + ({"type": "thinking_delta", "thinking": "7 * 142 = 994"}, {"type": "signature_delta", "signature": "sig"}), +) +_PREAMBLE_BLOCK: Final = ( + {"type": "text", "text": ""}, + ({"type": "text_delta", "text": "**Working it out:** "}, {"type": "text_delta", "text": "the count is **120**"}), +) + + +def _json_tool_block(answer: dict[str, object]) -> tuple[dict[str, object], tuple[dict[str, object], ...]]: + encoded: Final = json.dumps(answer) + return ( + {"type": "tool_use", "id": "toolu_1", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {}}, + tuple({"type": "input_json_delta", "partial_json": part} for part in (encoded[:5], encoded[5:])), + ) + + +def _parse_stream( + events: tuple[dict[str, object], ...], schema: dict[str, object] | None = _STREAMED_PERSON_SCHEMA +) -> tuple[ModelResponseStream, ...]: + iterator: Final = ModelResponseIterator( + streaming_response=MagicMock(), + sync_stream=True, + json_mode=True, + response_format_schema=None if schema is None else response_format_schema(schema), + ) + return tuple(iterator.chunk_parser(event) for event in events) + + +def _joined_content(chunks: tuple[ModelResponseStream, ...]) -> str: + return "".join(chunk.choices[0].delta.content or "" for chunk in chunks) + + +def test_streamed_json_mode_drops_prose_written_before_the_json_tool(): + chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK, _json_tool_block(_STREAMED_PERSON))) + + assert json.loads(_joined_content(chunks)) == _STREAMED_PERSON + assert "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) == ( + "7 * 142 = 994" + ) + assert all(chunk.choices[0].delta.tool_calls is None for chunk in chunks) + + +def test_streamed_json_mode_returns_prose_when_claude_skips_the_json_tool(): + chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK)) + + assert _joined_content(chunks) == "**Working it out:** the count is **120**" + + +def test_streamed_json_mode_sends_prose_ahead_of_a_caller_tool_call(): + caller_tool: Final = ( + {"type": "tool_use", "id": "toolu_2", "name": "get_weather", "input": {}}, + ({"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},), + ) + + chunks: Final = _parse_stream(_message_events(_PREAMBLE_BLOCK, caller_tool)) + tool_chunk_index: Final = next(i for i, chunk in enumerate(chunks) if chunk.choices[0].delta.tool_calls) + + assert _joined_content(chunks[: tool_chunk_index + 1]) == "**Working it out:** the count is **120**" + assert chunks[tool_chunk_index].choices[0].delta.tool_calls[0].function.name == "get_weather" + + +def test_streamed_native_output_text_is_not_held_back(): + answer: Final = json.dumps(_STREAMED_PERSON) + text_block: Final = ( + {"type": "text", "text": ""}, + tuple({"type": "text_delta", "text": part} for part in (answer[:6], answer[6:])), + ) + + chunks: Final = _parse_stream(_message_events(text_block), schema=None) + + assert tuple(chunk.choices[0].delta.content for chunk in chunks[2:4]) == (answer[:6], answer[6:]) + + def test_streamed_json_mode_tool_name_envelope_is_unwrapped_when_name_is_a_schema_property(): envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_CONTACT}) @@ -3379,7 +3677,7 @@ def test_streamed_json_mode_partial_objects_that_repeat_a_field_keep_the_order_t assert "".join(contents) == "".join(blocks) -def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer(): +def test_streamed_json_mode_text_after_the_answer_is_dropped_like_the_non_streamed_reply(): answer = json.dumps(_STREAMED_RISK_OBJECT) message_start, *_, message_delta = _json_tool_events(()) text_block = ( @@ -3390,7 +3688,7 @@ def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer(): chunks = _json_mode_stream_chunks((message_start, *_json_tool_block_events(0, answer), *text_block, message_delta)) - assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer + " Done." + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer def test_streamed_json_mode_envelope_whose_label_is_a_property_name_is_still_unwrapped(): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 13ff11e8a3f..80845b1f12d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,4 +1,5 @@ import copy +import itertools import json import logging from collections.abc import Iterator @@ -21,6 +22,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt @@ -1423,6 +1425,33 @@ def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(m assert "json_tool_call" not in [tool["name"] for tool in optional_params.get("tools", [])] +def test_dict_fields_take_the_unforced_json_tool_on_a_default_thinking_model(local_model_cost_map): + optional_params: Final = AnthropicConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": { + "type": "object", + "properties": {"field": {"type": "object", "additionalProperties": {"type": "integer"}}}, + "required": ["field"], + }, + }, + } + }, + optional_params={}, + model="claude-opus-5", + drop_params=False, + ) + + assert "output_format" not in optional_params + assert "tool_choice" not in optional_params + assert [ + (tool["name"], tool["input_schema"]["properties"]["field"]) for tool in optional_params["tools"] + ] == [("json_tool_call", {"type": "object", "additionalProperties": {"type": "integer"}})] + + def test_native_structured_output_uses_bundled_capability_when_remote_map_lags( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -2630,6 +2659,46 @@ def test_get_config_with_model_uses_dynamic_max_tokens(): assert config_claude37["max_tokens"] == 64000 +def test_transform_request_uses_deployment_max_output_tokens_before_catalog() -> None: + model: Final = "claude-sonnet-4-6" + catalog_max_output_tokens: Final = litellm.get_model_info(model)["max_output_tokens"] + config: Final = AnthropicConfig() + + configured: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={"model_info": {"max_output_tokens": 987}}, + headers={}, + ) + catalog_default: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + non_mapping_model_info: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={"model_info": "invalid"}, + headers={}, + ) + requested: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hello"}], + optional_params={"max_tokens": 654}, + litellm_params={"model_info": {"max_output_tokens": 987}}, + headers={}, + ) + + assert configured["max_tokens"] == 987 + assert catalog_default["max_tokens"] == catalog_max_output_tokens + assert non_mapping_model_info["max_tokens"] == catalog_max_output_tokens + assert requested["max_tokens"] == 654 + + def test_get_config_without_model_uses_fallback(): """ Test that get_config without model parameter uses 128000 fallback. @@ -4422,6 +4491,191 @@ def test_transform_request_includes_cache_control(): assert result["cache_control"] == {"type": "ephemeral"} +def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: list | None = None) -> dict: + return AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=messages, + optional_params={"max_tokens": 100, **({"tools": tools} if tools else {})}, + litellm_params=litellm_params or {}, + headers={}, + ) + + +def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control(): + body: Final = _anthropic_body( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}, + { + "type": "text", + "text": "Bye", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ], + }, + ] + ) + + assert body["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}] + assert body["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Bye", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + ] + + +@pytest.mark.parametrize( + "marker, expected_block", + [ + ({"mode": "explicit", "ttl": "30m"}, {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}), + ({"mode": "implicit"}, {"type": "text", "text": "Hello"}), + ], +) +def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block): + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]} + ] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _anthropic_body(copy.deepcopy(messages)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}) + assert body["messages"] == [{"role": "user", "content": [expected_block]}] + + +def test_transform_request_names_the_marker_ttl_rule_for_a_ttl_only_claude_takes(): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}} + ], + } + ] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="optional ttl of '30m' or '12h'"): + _anthropic_body(messages) + + +def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}} + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(1, 5) + ] + messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}] + tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + _anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools) + assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + None, + {"type": "ephemeral"}, + ] + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap(): + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "max_tokens": 100, + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + optional_params={}, + model="claude-sonnet-5-5", + drop_params=False, + ) + marked_turns: Final = [ + { + "role": "user", + "content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + for index in range(4) + ] + + def body(litellm_params: dict) -> dict: + return config.transform_request( + model="claude-sonnet-5-5", + messages=copy.deepcopy(marked_turns), + optional_params=copy.deepcopy(optional_params), + litellm_params=litellm_params, + headers={}, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + body({}) + + capped: Final = body({"drop_params": True}) + assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"} + sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"]) + assert [part.get("cache_control") for part in sent_parts] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + {"type": "ephemeral"}, + ] + + +@pytest.mark.parametrize( + "unsent_marks", + [ + pytest.param( + [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Noted.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + ], + id="assistant_part", + ), + pytest.param( + [{"role": "user", "content": "Also this.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + id="message_level", + ), + ], +) +def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]): + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4) + ] + + body: Final = _anthropic_body( + [{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}] + ) + + sent_marks: Final = [ + part.get("cache_control") + for part in itertools.chain.from_iterable(message["content"] for message in body["messages"]) + if isinstance(part, dict) + ] + assert sent_marks.count({"type": "ephemeral"}) == 4 + + def test_compaction_block_empty_list_not_added(): """ Test that empty compaction_blocks list is not added to provider_specific_fields. @@ -6730,6 +6984,63 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc assert "tool_choice" not in result +@pytest.mark.parametrize( + ("model", "thinking_params", "forced"), + [ + ("claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False), + ("claude-opus-5", {}, False), + ("claude-sonnet-5", {}, False), + ("claude-haiku-5-5", {}, False), + ("claude-fable-5", {"reasoning_effort": "none"}, False), + ("claude-opus-4-8", {}, True), + ("claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ("claude-haiku-5-5", {"reasoning_effort": "none"}, True), + ], + ids=[ + "opus-4-8-adaptive", + "opus-5-default-thinking", + "sonnet-5-default-thinking", + "haiku-5-5-default-thinking", + "fable-5-always-on-reasoning-effort-none", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + "haiku-5-5-reasoning-effort-none", + ], +) +def test_response_format_tool_is_forced_only_when_the_model_will_not_think( + set_native_structured_output, model, thinking_params, forced +): + set_native_structured_output(model, None) + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + **thinking_params, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "capital", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, + }, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + request: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert [tool["name"] for tool in request["tools"]] == ["json_tool_call"] + assert (request.get("tool_choice") == {"name": "json_tool_call", "type": "tool"}) is forced + assert request["tools"][0].get("description") == (None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION) + + def _eager_chat_function(**extra: object) -> dict[str, object]: return { "name": "write_file", @@ -10070,7 +10381,9 @@ def test_chat_response_without_input_transformations_adds_no_key() -> None: def test_json_mode_chat_response_keeps_input_transformations() -> None: completion_response: Final = { **_input_transformations_response(), - "content": [{"type": "tool_use", "id": "toolu_json", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {"answer": 42}}], + "content": [ + {"type": "tool_use", "id": "toolu_json", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {"answer": 42}} + ], "stop_reason": "tool_use", } result: Final = AnthropicConfig().transform_response( @@ -10510,7 +10823,9 @@ def test_chat_transform_request_never_blames_a_numeric_assistant_tool_call_id_fo { "role": "assistant", "content": None, - "tool_calls": [{"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + "tool_calls": [ + {"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], }, {"role": "tool", "tool_call_id": "5", "content": "rain"}, ], @@ -10547,9 +10862,15 @@ def test_chat_transform_request_names_no_field_for_a_failure_outside_roles_and_t def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ - {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "system", + "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, - {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": "answer 2"}, {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, diff --git a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 90a6ae3a574..7db81b7841e 100644 --- a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest - +import litellm from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, @@ -21,6 +21,10 @@ from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( _handle_message_start, _parse_sse_events, ) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging # --------------------------------------------------------------------------- @@ -88,6 +92,14 @@ def _build_simple_text_stream() -> List[bytes]: return chunks +def _sse_frames(wire: str) -> list[tuple[str, str]]: + return [ + (frame.split("\n")[0].removeprefix("event: "), frame.split("data: ", 1)[1]) + for frame in wire.split("\n\n") + if frame.startswith("event: ") and "data: " in frame + ] + + def _build_tool_use_stream() -> List[bytes]: """Produce SSE bytes for a response with a tool_use block.""" chunks = [] @@ -727,8 +739,7 @@ class TestAgenticStreamingIteratorPhase2: class TestAgenticStreamingIteratorErrorHandling: @pytest.mark.asyncio - async def test_should_swallow_hook_errors(self): - """Errors in hook processing should be swallowed; Phase 1 chunks are still yielded.""" + async def test_a_hook_failure_reaches_the_client_after_the_streamed_message(self): chunks = _build_simple_text_stream() mock_stream = MockAsyncStream(chunks) @@ -751,11 +762,15 @@ class TestAgenticStreamingIteratorErrorHandling: ) collected = [] - async for chunk in iterator: - collected.append(chunk) - # All Phase 1 chunks should still have been yielded - assert len(collected) == len(chunks) + async def _drain(): + async for chunk in iterator: + collected.append(chunk) + + with pytest.raises(RuntimeError, match="hook exploded"): + await _drain() + + assert collected == chunks @pytest.mark.asyncio async def test_should_handle_empty_stream(self): @@ -924,21 +939,54 @@ class TestAgenticStreamingIteratorHoldBack: assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks @pytest.mark.asyncio - async def test_should_error_instead_of_replaying_server_fulfilled_tool_use_when_hook_crashes(self): - """A hook crash must not replay the buffered retrieval tool_use: that is the unknown-tool bug.""" + async def test_a_rate_limited_follow_up_reaches_the_client_as_its_anthropic_error_event(self): chunks = _build_tool_use_stream() mock_handler = MagicMock() - mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=RuntimeError("hook exploded")) + mock_handler._call_agentic_completion_hooks = AsyncMock( + side_effect=litellm.RateLimitError( + message="follow-up rate limited", llm_provider="anthropic", model="claude-sonnet-4-20250514" + ) + ) - iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler) + wire = "".join( + [ + frame if isinstance(frame, str) else frame.decode() + async for frame in ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=_build_hold_back_iterator(MockAsyncStream(chunks), mock_handler), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "claude-sonnet-4-20250514"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + ) + ] + ) - collected = [] - async for chunk in iterator: - collected.append(chunk) + error_events = [json.loads(data) for event, data in _sse_frames(wire) if event == "error"] + assert [event["error"]["type"] for event in error_events] == ["rate_limit_error"] + assert "follow-up rate limited" in error_events[0]["error"]["message"] + assert "litellm_content_retrieve" not in wire - assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES] - assert b"litellm_content_retrieve" not in b"".join(collected) + @pytest.mark.asyncio + async def test_a_capped_loop_tells_the_client_to_raise_max_agentic_loops_instead_of_retrying(self): + mock_handler = MagicMock() + mock_handler._call_agentic_completion_hooks = AsyncMock( + side_effect=lambda **_: BaseLLMHTTPHandler._check_agentic_loop_safety( + tool_calls=[{"name": "litellm_content_retrieve"}], + fingerprints=[], + depth=1, + max_loops=1, + model="claude-sonnet-4-20250514", + ) + ) + + iterator = _build_hold_back_iterator(MockAsyncStream(_build_tool_use_stream()), mock_handler) + wire = b"".join([chunk async for chunk in iterator if chunk != STREAM_SSE_KEEPALIVE_PING_BYTES]).decode() + + error_messages = [json.loads(data)["error"]["message"] for event, data in _sse_frames(wire) if event == "error"] + assert len(error_messages) == 1 + assert "max_agentic_loops" in error_messages[0] + assert "retry" not in error_messages[0].lower() + assert "litellm_content_retrieve" not in wire @pytest.mark.asyncio async def test_should_error_instead_of_replaying_when_no_hook_fires_on_tool_use(self): diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index cf0b8a39865..99e6438c723 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -10,6 +10,7 @@ Verifies that: - ANTHROPIC_API_KEY / ANTHROPIC_API_BASE take precedence over their aliases. """ +import itertools import json import logging import os @@ -4604,6 +4605,7 @@ def test_anthropic_error_frame_exception_classes_an_overloaded_frame_as_internal error: Final = anthropic_error_frame_exception("overloaded_error", "Overloaded", 503, "claude-sonnet-4-5") assert type(error) is litellm.InternalServerError + assert error.provider_status_code == 529 _NON_NUMERIC_BUDGETS: Final = (True, False, None, "", "lots", [4096]) @@ -4728,3 +4730,142 @@ def test_tools_without_strict_drops_only_the_strict_key_and_counts_the_strict_to assert tools_without_strict(tools) == ToolsWithoutStrict( tools=({"name": "a"}, {"name": "b"}, {"name": "c"}), dropped_count=1 ) + + +@pytest.mark.parametrize("registration_order", [("caching-off", "unflagged"), ("unflagged", "caching-off")]) +def test_auto_injection_reads_a_deployments_own_prompt_caching_flag_not_a_siblings( + registration_order: tuple[str, ...], +) -> None: + from litellm import Router + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + model: Final = "anthropic.claude-qa-cache-model" + litellm.register_model({model: {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_caching": True}}) + model_infos: Final = { + "caching-off": {"id": "caching-off-id", "supports_prompt_caching": False}, + "unflagged": {"id": "unflagged-id"}, + } + Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"bedrock/{model}", "aws_region_name": "us-east-1"}, + "model_info": model_infos[name], + } + for name in registration_order + ] + ) + + assert {name: supports_anthropic_cache_control(model, "bedrock", model_infos[name]) for name in registration_order} == { + "caching-off": False, + "unflagged": True, + } + + +def test_auto_injection_leaves_an_unmapped_claude_deployment_uncached_instead_of_raising() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + assert supports_anthropic_cache_control("claude-qa-unmapped-model", "anthropic", {"id": "unmapped-id"}) is False + + +def test_auto_injection_honors_a_register_model_opt_out_made_under_the_provider_prefixed_name() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ) + litellm.register_model({f"anthropic/{model}": {"supports_prompt_caching": False}}) + + assert supports_anthropic_cache_control(model, "anthropic", {"id": "plain-id"}) is False + + +@pytest.mark.parametrize("route_prefix", ["bedrock/", "bedrock/converse/"]) +def test_auto_injection_honors_a_register_model_opt_out_made_under_a_bedrock_route_prefixed_name(route_prefix: str) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "bedrock_converse" + and entry.get("supports_prompt_caching") is True + and key.startswith("us.anthropic.claude") + ) + litellm.register_model({f"{route_prefix}{model}": {"supports_prompt_caching": False}}) + + assert ( + supports_anthropic_cache_control(model, "bedrock", {"id": "plain-id"}), + supports_anthropic_cache_control(model, "bedrock"), + ) == (False, False) + + +@pytest.mark.parametrize( + ("declared", "catalog_caching", "expected"), + [("true", False, True), (1, False, True), ("false", True, False), (0, True, False), ("maybe", True, True)], +) +def test_auto_injection_reads_a_string_or_numeric_deployment_prompt_caching_flag_as_the_bool_it_spells( + declared: object, catalog_caching: bool, expected: bool +) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + model: Final = f"claude-qa-lax-flag-model-{catalog_caching}" + litellm.register_model( + {model: {"litellm_provider": "anthropic", "mode": "chat", "supports_prompt_caching": catalog_caching}} + ) + + assert ( + supports_anthropic_cache_control(model, "anthropic", {"id": "lax-id", "supports_prompt_caching": declared}) + is expected + ) + + +@pytest.mark.parametrize("reload_cost_map", [False, True], ids=["before-reload", "after-reload"]) +@pytest.mark.parametrize( + ("registrations", "expected"), + [ + ((("", True), ("anthropic/", False)), False), + ((("anthropic/", False), ("", True)), True), + ((("", False), ("anthropic/", True), ("", False)), False), + ], + ids=["prefixed-opt-out-last", "bare-opt-in-last", "bare-opt-out-registered-again-last"], +) +def test_the_later_of_a_bare_and_a_prefixed_register_model_entry_wins_for_routed_and_sdk_calls( + registrations: tuple[tuple[str, bool], ...], expected: bool, reload_cost_map: bool +) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations, reapply_runtime_model_cost_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ) + for prefix, flag in registrations: + litellm.register_model({f"{prefix}{model}": {"supports_prompt_caching": flag}}) + if reload_cost_map: + reapply_runtime_model_cost_registrations() + + assert ( + supports_anthropic_cache_control(model, "anthropic", {"id": "plain-id"}), + supports_anthropic_cache_control(model, "anthropic"), + ) == (expected, expected) + + +def test_a_register_model_opt_out_for_one_claude_model_leaves_every_other_routed_models_catalog_flag() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + opted_out, untouched = itertools.islice( + ( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ), + 2, + ) + litellm.register_model({f"anthropic/{opted_out}": {"supports_prompt_caching": False}}) + + assert supports_anthropic_cache_control(untouched, "anthropic", {"id": "plain-id"}) is True diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index aa7fa2727b4..13c17ebd2bf 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -50,20 +50,30 @@ COMPACTED_HISTORY = [ "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) ) @pytest.mark.parametrize( - ("image_url", "source"), + ("image_url", "block"), ( - ("data:image/png;base64,aW1hZ2U=", {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}), - ({"url": "data:image/png;base64,aW1hZ2U="}, {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}), + ( + "data:image/png;base64,aW1hZ2U=", + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}}, + ), + ( + {"url": "data:image/png;base64,aW1hZ2U="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}}, + ), ( {"url": "data:image/png;base64,aW1hZ2U=", "format": "image/jpeg", "detail": "high"}, - {"type": "base64", "media_type": "image/jpeg", "data": "aW1hZ2U="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "aW1hZ2U="}}, + ), + ( + "data:application/pdf;base64,JVBERi0=", + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="}}, ), ), ) def test_count_translates_openai_images_without_mutating_input( config_type: type[AnthropicCountTokensConfig], image_url: str | dict[str, JsonValue], - source: dict[str, JsonValue], + block: dict[str, JsonValue], ) -> None: cache_control: Final[dict[str, JsonValue]] = {"type": "ephemeral"} messages: Final[list[dict[str, JsonValue]]] = [{ @@ -81,7 +91,7 @@ def test_count_translates_openai_images_without_mutating_input( "model": "claude-opus-5-5", "messages": [{ "role": "user", "content": [ {"type": "text", "text": "Count this image"}, - {"type": "image", "source": source, "cache_control": cache_control}, + {**block, "cache_control": cache_control}, ], }], } diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index dd29ee83bc6..f93968eb2e6 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -917,7 +917,7 @@ def test_azure_sends_part_breakpoints_as_written_without_drop_params( [ ("unit-breakpoint-ignorer", {"drop_params": True}), ("unit-breakpoint-reader", {"drop_params": True, "base_model": "azure/unit-breakpoint-ignorer"}), - ("my-azure-alias", {"drop_params": True, "model_info": {"base_model": "azure/unit-breakpoint-ignorer"}}), + ("my-azure-alias", {"drop_params": True, "model_info": {"base_model": "azure/gpt-4.1"}}), ], ids=("known-non-reader", "base-model-known-non-reader", "model-info-base-model-known-non-reader"), ) diff --git a/tests/unit/llms/azure/decisions/test_transformation.py b/tests/unit/llms/azure/decisions/test_transformation.py index b6de6084b9c..3f5e97ee000 100644 --- a/tests/unit/llms/azure/decisions/test_transformation.py +++ b/tests/unit/llms/azure/decisions/test_transformation.py @@ -125,6 +125,29 @@ async def test_default_credential_refreshes_without_opt_in( ) +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.parametrize("systemone", (False, True)) +async def test_entra_rejecting_the_default_credential_answers_401_with_only_the_aadsts_reason( + monkeypatch: pytest.MonkeyPatch, is_async: bool, systemone: bool +) -> None: + from azure.core.exceptions import ClientAuthenticationError + + app_client_id: Final = "0d08e61f-6184-4980-9cef-a2e9128f4cb2" + credential = Mock(spec=["get_token"]) + credential.get_token.side_effect = ClientAuthenticationError( + message="Authentication failed: AADSTS7000215: Invalid client secret provided. Ensure the secret being " + f"sent in the request is the client secret value, for a secret added to app '{app_client_id}'." + ) + monkeypatch.setattr("azure.identity.DefaultAzureCredential", Mock(return_value=credential)) + + with pytest.raises(litellm.AuthenticationError) as excinfo: + await call_decisions(is_async, systemone, {}) + + assert excinfo.value.status_code == 401 + assert "AADSTS7000215" in str(excinfo.value) + assert app_client_id not in str(excinfo.value) + + @pytest.mark.parametrize("kind", ("service-principal", "oidc", "provider")) async def test_azure_decisions_reuses_existing_azure_credential_resolution( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, kind: str diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index f1db76c1e21..6745421b8f8 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -1,4 +1,7 @@ import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -259,6 +262,21 @@ async def test_construct_url_v1_protocol(): assert url.count("/realtime") == 1 +def test_construct_url_translation_protocol(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + url = AzureOpenAIRealtime()._construct_url( + api_base="https://my-endpoint.openai.azure.com", + model="translate-deployment", + api_version=None, + realtime_protocol="GA", + query_params={"model": "translate-deployment"}, + realtime_mode="translation", + ) + + assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment" + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"]) async def test_construct_url_case_insensitive_protocol(protocol): @@ -752,3 +770,32 @@ def test_construct_url_with_a_trailing_slash_api_base_matches_the_url_without_it ) assert url_for("https://gw.example.com/azure/") == url_for("https://gw.example.com/azure") + + +@dataclass(frozen=True, slots=True) +class _ClientWebSocketWithHeaders: + scope: Mapping[str, object] + + +@pytest.mark.parametrize("pinned_protocol", ("GA", "ga", "v1")) +@pytest.mark.parametrize( + ("client_headers", "bridges_beta_client_to_ga"), + ( + pytest.param(((b"openai-beta", b"realtime=v1"),), True, id="beta-client"), + pytest.param((), False, id="ga-client"), + ), +) +def test_a_deployment_pinned_to_ga_bridges_only_a_beta_client( + pinned_protocol: str, client_headers: tuple[tuple[bytes, bytes], ...], bridges_beta_client_to_ga: bool +) -> None: + from litellm.llms.azure.realtime.handler import AzureRealtimeProtocol, azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + pinned_protocol, + model="my-realtime-deployment", + realtime_mode="realtime", + query_params=None, + websocket=_ClientWebSocketWithHeaders(scope={"headers": client_headers}), + ) + + assert protocol == AzureRealtimeProtocol(name=pinned_protocol, bridges_beta_client_to_ga=bridges_beta_client_to_ga) diff --git a/tests/unit/llms/azure/realtime/test_http_transformation.py b/tests/unit/llms/azure/realtime/test_http_transformation.py index 714ce9558cf..dae8ed10d1d 100644 --- a/tests/unit/llms/azure/realtime/test_http_transformation.py +++ b/tests/unit/llms/azure/realtime/test_http_transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import httpx @@ -7,7 +8,11 @@ import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview" -_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai") +_API_BASES: Final = ( + "https://my.openai.azure.com", + "https://my.openai.azure.com/", + "https://my.openai.azure.com/openai", +) class _AzureThatRecordsTheRequest: @@ -55,3 +60,62 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls" assert request.headers.get("authorization") == "Bearer ek_upstream" assert "api-key" not in request.headers + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "url"), + ( + ("https://gw.example.com/azure/v1", "https://gw.example.com/azure/v1/openai/realtime/transcription_sessions"), + ("https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/realtime/transcription_sessions"), + ("https://my.openai.azure.com/openai/v1", "https://my.openai.azure.com/openai/realtime/transcription_sessions"), + ), +) +async def test_azure_transcription_session_strips_only_a_trailing_openai_path(api_base: str, url: str) -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_transcription_session( + model="azure/gpt-realtime-whisper-deployment", + api_base=api_base, + api_key="azure-key", + api_version=_DATED_PREVIEW_API_VERSION, + client=azure.client(), + ) + (request,) = azure.requests + assert str(request.url) == f"{url}?api-version={_DATED_PREVIEW_API_VERSION}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller_headers", "sent_headers"), + ( + pytest.param( + {"Authorization": "Bearer caller-token"}, + {"authorization": "Bearer caller-token", "api-key": None}, + id="caller-authorization", + ), + pytest.param( + {"authorization": "Bearer caller-token"}, + {"authorization": "Bearer caller-token", "api-key": None}, + id="caller-lowercase-authorization", + ), + pytest.param({"api-key": "caller-key"}, {"authorization": None, "api-key": "caller-key"}, id="caller-api-key"), + pytest.param({}, {"authorization": "Bearer entra-token", "api-key": None}, id="no-caller-auth"), + ), +) +async def test_azure_entra_fallback_never_overrides_the_callers_own_auth_header( + caller_headers: Mapping[str, str], + sent_headers: Mapping[str, str | None], + monkeypatch: pytest.MonkeyPatch, + no_ambient_azure_credentials: None, +) -> None: + monkeypatch.delenv("AZURE_API_KEY", raising=False) + monkeypatch.setenv("AZURE_AD_TOKEN", "entra-token") + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-deployment", + api_base="https://my.openai.azure.com", + extra_headers=caller_headers, + client=azure.client(), + ) + (request,) = azure.requests + assert {name: request.headers.get(name) for name in sent_headers} == sent_headers diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 29006901877..05c0817e19b 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -7,15 +7,18 @@ from unittest.mock import MagicMock, patch import httpx import pytest import respx +from openai.types.responses import ResponseReasoningItem +from pydantic import JsonValue, TypeAdapter import litellm from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.azure.responses.o_series_transformation import ( AzureOpenAIOSeriesResponsesAPIConfig, ) from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure_ai.responses.transformation import AzureAIAgentResponsesAPIConfig, AzureAIResponsesAPIConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams @@ -1547,3 +1550,267 @@ def test_openai_gpt5_resolves_to_responses_config_with_temperature() -> None: config: Final = ProviderConfigManager.get_provider_responses_api_config(provider=LlmProviders.OPENAI, model="gpt-5") assert type(config) is OpenAIResponsesAPIConfig assert "temperature" in config.get_supported_openai_params("gpt-5") + + +@pytest.mark.parametrize( + ("config", "model"), + [ + (AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureOpenAIOSeriesResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureAIResponsesAPIConfig(), "gpt-6.1-sol"), + (AzureAIAgentResponsesAPIConfig(), "agents/replay-test"), + ], + ids=["azure", "o_series", "azure_ai", "azure_ai_agent"], +) +@pytest.mark.parametrize("as_model", [False, True], ids=["dict", "pydantic"]) +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "no-drop-params"]) +def test_azure_reasoning_replay_removes_output_content_only_under_drop_params_without_changing_history( + config: AzureOpenAIResponsesAPIConfig, model: str, as_model: bool, drop_params: bool +) -> None: + replayable_reasoning: Final = { + "id": "rs_saved_reasoning", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Check the file before changing it."}], + "encrypted_content": "opaque-saved-reasoning", + } + saved_reasoning: Final = { + **replayable_reasoning, + "content": [{"type": "reasoning_text", "text": "I should inspect the file first."}], + } + tool_call: Final = { + "type": "function_call", + "id": "fc_read", + "call_id": "call_read", + "name": "read_file", + "arguments": '{"path":"main.py"}', + } + tool_result: Final = {"type": "function_call_output", "call_id": "call_read", "output": "print(1)"} + answer: Final = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "The file prints 1.", "annotations": []}], + } + prompt: Final = {"role": "user", "content": "Continue."} + history: Final = [ + ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning, + tool_call, + tool_result, + answer, + prompt, + ] + original: Final = deepcopy(history) + + request: Final = config.transform_responses_api_request( + model=model, + input=history, + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + sent_reasoning: Final = replayable_reasoning if drop_params else saved_reasoning + assert request["input"] == [sent_reasoning, tool_call, tool_result, answer, prompt] + assert history == original + + +@pytest.mark.parametrize("as_model", [False, True], ids=["dict", "pydantic"]) +def test_openai_replay_without_drop_params_keeps_reasoning_content_and_message_status(as_model: bool) -> None: + saved_reasoning: Final = { + "id": "rs_saved_reasoning", + "type": "reasoning", + "summary": [], + "content": [{"type": "reasoning_text", "text": "Inspect the file."}], + "encrypted_content": "opaque-saved-reasoning", + } + saved_message: Final = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "The file is ready.", "annotations": []}], + } + history: Final = [ + ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning, + saved_message, + ] + original: Final = deepcopy(history) + + request: Final = OpenAIResponsesAPIConfig().transform_responses_api_request( + model="gpt-6.1-sol", + input=history, + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request["input"] == [saved_reasoning, saved_message] + assert history == original + + +@pytest.mark.parametrize( + "config", + [AzureOpenAIResponsesAPIConfig(), AzureOpenAIOSeriesResponsesAPIConfig(), AzureAIResponsesAPIConfig()], + ids=["azure", "o_series", "azure_ai"], +) +def test_azure_family_forwards_input_items_that_are_not_objects_for_the_provider_to_judge( + config: AzureOpenAIResponsesAPIConfig, +) -> None: + saved_reasoning: Final = {"type": "reasoning", "summary": [], "status": "completed", "encrypted_content": "opaque"} + + request: Final = config.transform_responses_api_request( + model="gpt-6.1-sol", + input=["Continue.", saved_reasoning, ["nested"]], + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request["input"] == [ + "Continue.", + {"type": "reasoning", "summary": [], "encrypted_content": "opaque"}, + ["nested"], + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("async_call", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize(("reasoning_index", "include_id"), [(51, True), (10, False)], ids=["with-id", "without-id"]) +async def test_azure_responses_replays_reasoning_history_without_fields_azure_rejects( + respx_mock: respx.MockRouter, stream: bool, async_call: bool, reasoning_index: int, include_id: bool +) -> None: + prefix: Final = [{"role": "user", "content": f"Earlier context {index}."} for index in range(reasoning_index)] + replayable_reasoning: Final = { + "type": "reasoning", + "summary": [], + "encrypted_content": "opaque-saved-reasoning", + **({"id": "rs_saved_reasoning"} if include_id else {}), + } + saved_reasoning: Final = { + **replayable_reasoning, + "status": "completed", + "content": [{"type": "reasoning_text", "text": "Inspect the file before editing."}], + } + answer: Final = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Ready to continue.", "annotations": []}], + } + prompt: Final = {"role": "user", "content": "Continue."} + replayable_followup: Final = { + "type": "reasoning", + "summary": [], + "encrypted_content": "opaque-followup-reasoning", + **({"id": "rs_followup_reasoning"} if include_id else {}), + } + saved_followup: Final = { + **replayable_followup, + "status": "completed", + "content": [{"type": "reasoning_text", "text": "Check the next file."}], + } + history: Final = [*prefix, saved_reasoning, answer, saved_followup, prompt] + original: Final = deepcopy(history) + + def respond(request: httpx.Request) -> httpx.Response: + body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(request.content) + items: Final = body["input"] + assert isinstance(items, list) + reasoning: Final = items[reasoning_index] + assert isinstance(reasoning, dict) + if "status" in reasoning: + return httpx.Response( + 400, + json={ + "error": { + "message": f"Unknown parameter: 'input[{reasoning_index}].status'.", + "type": "invalid_request_error", + "param": f"input[{reasoning_index}].status", + "code": "unknown_parameter", + } + }, + ) + if reasoning.get("content"): + return httpx.Response( + 400, + json={ + "error": { + "message": f"Invalid 'input[{reasoning_index}].content': array too long. " + "Expected an array with maximum length 0, " + "but got an array with length 1 instead.", + "type": "invalid_request_error", + "param": f"input[{reasoning_index}].content", + "code": "array_above_max_length", + } + }, + ) + return httpx.Response( + 200, + text=_COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE), + headers={"content-type": "text/event-stream" if stream else "application/json"}, + ) + + route: Final = respx_mock.post(host="example-resource.openai.azure.com").mock(side_effect=respond) + request_params: Final = { + "model": "azure/gpt-6.1-sol", + "input": history, + "api_base": _AZURE_RESOURCE, + "api_key": "azure-test-key", + "stream": stream, + "store": False, + "include": ["reasoning.encrypted_content"], + "drop_params": True, + } + response: Final = await litellm.aresponses(**request_params) if async_call else litellm.responses(**request_params) + deltas: Final = ( + ( + [event.delta async for event in response if event.type == "response.output_text.delta"] + if async_call + else [event.delta for event in response if event.type == "response.output_text.delta"] + ) + if stream + else [] + ) + text: Final = "".join(deltas) if stream else response.output[0].content[0].text + + assert text == "Done." + assert route.call_count == 1 + assert TypeAdapter(dict[str, JsonValue]).validate_json(route.calls[0].request.content)["input"] == [ + *prefix, + replayable_reasoning, + answer, + replayable_followup, + prompt, + ] + assert history == original + + +@pytest.mark.parametrize( + ("as_model", "include_id"), + [(False, True), (False, False), (True, True)], + ids=["dictionary-with-id", "dictionary-without-id", "pydantic"], +) +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "no-drop-params"]) +def test_azure_reasoning_replay_drops_status_always_and_content_under_drop_params( + as_model: bool, include_id: bool, drop_params: bool +) -> None: + replayable: Final = { + "type": "reasoning", + "summary": [], + "encrypted_content": "opaque-saved-reasoning", + **({"id": "rs_saved_reasoning"} if include_id else {}), + } + saved_content: Final = [{"type": "reasoning_text", "text": "Inspect the file first."}] + saved_reasoning: Final = {**replayable, "status": "completed", "content": saved_content} + history: Final = [ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning] + original: Final = deepcopy(history) + + request: Final = AzureOpenAIResponsesAPIConfig().transform_responses_api_request( + model="gpt-6.1-sol", + input=history, + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request["input"] == [replayable if drop_params else {**replayable, "content": saved_content}] + assert history == original diff --git a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py index dfd1474211f..53f426d8505 100644 --- a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py +++ b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py @@ -466,6 +466,12 @@ def _mock_response(): response.content = json.dumps(MOCK_BING_GROUNDING_RESPONSE).encode() return response +async def _search_what_is_litellm(is_async: bool) -> object: + if is_async: + return await litellm.asearch(query="what is litellm", search_provider="bing_grounding") + return litellm.search(query="what is litellm", search_provider="bing_grounding") + + @pytest.mark.usefixtures("_vcr_outcome_gate") class TestBingGroundingSearchTransformation: """ @@ -546,6 +552,36 @@ class TestBingGroundingSearchTransformation: } ] + @pytest.mark.parametrize("is_async", (False, True)) + async def test_entra_rejecting_the_default_credential_answers_401_with_only_the_aadsts_reason( + self, monkeypatch: pytest.MonkeyPatch, is_async: bool + ): + from azure.core.exceptions import ClientAuthenticationError + + from litellm.llms.azure import common_utils + + app_client_id: Final = "0d08e61f-6184-4980-9cef-a2e9128f4cb2" + credential = Mock(spec=["get_token"]) + credential.get_token.side_effect = ClientAuthenticationError( + message="Authentication failed: AADSTS7000215: Invalid client secret provided. Ensure the secret being " + f"sent in the request is the client secret value, for a secret added to app '{app_client_id}'." + ) + monkeypatch.delenv("BING_GROUNDING_TOKEN") + for name in ("AZURE_CLIENT_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_SECRET", "AZURE_USERNAME", "AZURE_PASSWORD"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr("azure.identity.DefaultAzureCredential", Mock(return_value=credential)) + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + try: + with pytest.raises(litellm.AuthenticationError) as excinfo: + await _search_what_is_litellm(is_async) + finally: + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + assert excinfo.value.status_code == 401 + assert "AADSTS7000215" in str(excinfo.value) + assert app_client_id not in str(excinfo.value) + @pytest.mark.asyncio async def test_bing_grounding_asearch(self): with patch( # test-quality-ok: litellm.asearch has no client injection seam diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index c5d403b3c38..f86fae494ef 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -4,6 +4,7 @@ from typing import Final import httpx import pytest +import respx from openai import AsyncAzureOpenAI, AzureOpenAI import litellm @@ -160,3 +161,70 @@ async def test_azure_transcription_parses_response_formats( ) assert response.text == expected_text + + +@pytest.mark.parametrize("api_version", ["2024-06-01", "2025-03-01-preview", "v1", "latest", "preview"]) +async def test_azure_router_transcription_sends_the_default_api_version( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_version: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("AZURE_API_VERSION", raising=False) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": { + "model": "azure/gpt-transcribe", + "api_key": "test-key", + "api_base": "https://transcribe-route.openai.azure.com", + "api_version": api_version, + }, + } + ] + ) + + response: Final = await router.atranscription(model="transcriber", file=_audio_file()) + + sent_url: Final = route.calls.last.request.url + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" + assert sent_url.params.get("api-version") == litellm.AZURE_DEFAULT_API_VERSION + assert response.text == "hello" + + +@pytest.mark.parametrize( + ("api_version", "expected_api_version"), + [ + ("v1", litellm.AZURE_DEFAULT_API_VERSION), + ("latest", litellm.AZURE_DEFAULT_API_VERSION), + ("preview", litellm.AZURE_DEFAULT_API_VERSION), + ("2025-03-01-preview", "2025-03-01-preview"), + ], +) +def test_azure_sync_transcription_uses_the_deployments_route( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_version: str, + expected_api_version: str, +) -> None: + monkeypatch.delenv("AZURE_API_VERSION", raising=False) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + response: Final = litellm.transcription( + model="azure/gpt-transcribe", + file=_audio_file(), + api_key="test-key", + api_base="https://transcribe-route.openai.azure.com", + api_version=api_version, + ) + + sent_url: Final = route.calls.last.request.url + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" + assert sent_url.params.get("api-version") == expected_api_version + assert response.text == "hello" diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 3f9da8be2d4..d66ba3f324e 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1,16 +1,23 @@ +import asyncio +import io import json import os +import sys import threading import traceback import uuid +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from queue import SimpleQueue -from typing import Callable, Final, Optional +from typing import Awaitable, Callable, Final, Optional from urllib.parse import parse_qs from unittest.mock import ANY, MagicMock, patch import httpx +import openai import pytest +import requests +from azure.core.pipeline.transport import RequestsTransport import litellm from litellm.llms.azure.common_utils import ( @@ -491,6 +498,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "avector_store_search", "acreate_skill", "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_calls", + "arealtime_translation_calls", ] ], ) @@ -2495,3 +2507,487 @@ def test_an_azure_client_litellm_built_its_own_http_client_for_is_still_closed(m closer.reap() assert wrapper.is_closed() is True + + +_ENTRA_APP_CLIENT_ID: Final = "0d08e61f-6184-4980-9cef-a2e9128f4cb2" +_ENTRA_TRACE_ID: Final = "f0c149d3-ef20-4194-8b81-c81a5e002500" +_ENTRA_REJECTIONS: Final = { + "wrong_secret": ( + "AADSTS7000215", + "Authentication failed: AADSTS7000215: Invalid client secret provided. Ensure the secret being sent in the " + "request is the client secret value, not the client secret ID, for a secret added to app " + f"'{_ENTRA_APP_CLIENT_ID}'. Trace ID: {_ENTRA_TRACE_ID} Correlation ID: ecfd2c73-ec0a-474c-bba7-205be903159d " + "Timestamp: 2026-10-29 07:19:40Z", + ), + "expired_secret": ( + "AADSTS7000222", + f"Authentication failed: AADSTS7000222: The provided client secret keys for app '{_ENTRA_APP_CLIENT_ID}' " + f"are expired. Visit the Azure portal to create new keys for your app. Trace ID: {_ENTRA_TRACE_ID} " + "Correlation ID: 354f9b57-5fed-47e4-a67d-89a7c56f6ddc Timestamp: 2026-10-29 07:19:50Z", + ), +} + + +def _raising_token_provider(error: Exception) -> Callable[[], str]: + def token_provider() -> str: + raise error + + return token_provider + + +def _azure_chat(token_provider: Callable[[], str]) -> object: + return litellm.completion( + model="azure/gpt-4.1", + api_base="https://litellm-qa.openai.azure.com", + api_version="2025-04-01-preview", + messages=[{"role": "user", "content": "hi"}], + azure_ad_token_provider=token_provider, + max_retries=0, + ) + + +def _azure_async_chat(token_provider: Callable[[], str]) -> object: + return asyncio.run( + litellm.acompletion( + model="azure/gpt-4.1", + api_base="https://litellm-qa.openai.azure.com", + api_version="2025-04-01-preview", + messages=[{"role": "user", "content": "hi"}], + azure_ad_token_provider=token_provider, + ) + ) + + +def _azure_responses(token_provider: Callable[[], str]) -> object: + return litellm.responses( + model="azure/gpt-4.1", + api_base="https://litellm-qa.openai.azure.com", + api_version="2025-04-01-preview", + input="hi", + azure_ad_token_provider=token_provider, + ) + + +def _azure_ai_decisions(token_provider: Callable[[], str]) -> object: + return litellm.systemone( + model="azure_ai/decision-1-qa", + api_base="https://litellm-qa.services.ai.azure.com", + state={"ticket": "The login page returns a 500 error after deploy"}, + questions={"is_bug": {"type": "noul", "instructions": "Is this a software bug report?"}}, + azure_ad_token_provider=token_provider, + ) + + +_AZURE_CALLS: Final = { + "chat": _azure_chat, + "async_chat": _azure_async_chat, + "responses": _azure_responses, + "decisions": _azure_ai_decisions, +} + + +@pytest.fixture +def no_azure_api_key(monkeypatch): + for env_var in ( + "AZURE_API_KEY", + "AZURE_OPENAI_API_KEY", + "AZURE_AI_API_KEY", + "AZURE_AD_TOKEN", + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_USERNAME", + "AZURE_PASSWORD", + ): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "azure_key", None) + + +@pytest.mark.parametrize("call", _AZURE_CALLS.values(), ids=_AZURE_CALLS.keys()) +@pytest.mark.parametrize(("aadsts_code", "entra_message"), _ENTRA_REJECTIONS.values(), ids=_ENTRA_REJECTIONS.keys()) +def test_entra_rejecting_the_credential_answers_401_with_only_the_aadsts_reason( + no_azure_api_key, call, aadsts_code, entra_message +): + from azure.core.exceptions import ClientAuthenticationError + + with pytest.raises(litellm.AuthenticationError) as excinfo: + call(_raising_token_provider(ClientAuthenticationError(message=entra_message))) + + assert excinfo.value.status_code == 401 + assert aadsts_code in str(excinfo.value) + assert _ENTRA_APP_CLIENT_ID not in str(excinfo.value) + assert _ENTRA_TRACE_ID not in str(excinfo.value) + + +@pytest.mark.parametrize("api_version", ["2025-04-01-preview", "v1"]) +def test_entra_rejecting_an_async_token_provider_answers_401(no_azure_api_key, api_version): + from azure.core.exceptions import ClientAuthenticationError + + aadsts_code, entra_message = _ENTRA_REJECTIONS["wrong_secret"] + + async def token_provider() -> str: + raise ClientAuthenticationError(message=entra_message) + + async_token_provider: Final[Callable[[], Awaitable[str]]] = token_provider + + with pytest.raises(litellm.AuthenticationError) as excinfo: + asyncio.run( + litellm.acompletion( + model="azure/gpt-4.1", + api_base="https://litellm-qa.openai.azure.com", + api_version=api_version, + messages=[{"role": "user", "content": "hi"}], + azure_ad_token_provider=async_token_provider, + ) + ) + + assert excinfo.value.status_code == 401 + assert aadsts_code in str(excinfo.value) + assert _ENTRA_APP_CLIENT_ID not in str(excinfo.value) + assert _ENTRA_TRACE_ID not in str(excinfo.value) + + +def _azure_identity_wrapped_token_provider(inner_error: Exception) -> Callable[[], str]: + from azure.core.exceptions import ClientAuthenticationError + + def token_provider() -> str: + try: + raise inner_error + except Exception as error: + raise ClientAuthenticationError(message=f"Authentication failed: {error}") from error + + return token_provider + + +@pytest.mark.parametrize("call", _AZURE_CALLS.values(), ids=_AZURE_CALLS.keys()) +@pytest.mark.parametrize("failed_leg", ["request", "response"]) +def test_unreachable_entra_endpoint_stays_a_retryable_server_error(no_azure_api_key, call, failed_leg): + from azure.core.exceptions import ServiceRequestError, ServiceResponseError + + transport_error: Final = {"request": ServiceRequestError, "response": ServiceResponseError}[failed_leg]( + "('Unable to connect to proxy', NewConnectionError(\"Failed to establish a new connection: " + "[Errno 111] Connection refused\"))" + ) + + with pytest.raises(openai.APIError) as excinfo: + call(_azure_identity_wrapped_token_provider(transport_error)) + + assert not isinstance(excinfo.value, litellm.AuthenticationError) + assert excinfo.value.status_code != 401 + + +_ENTRA_TENANT_ID: Final = "5b3e1c7a-2f4d-4e8b-9a61-0c2d7f8e9b14" +_ENTRA_SCOPE: Final = "https://cognitiveservices.azure.com/.default" +_ENTRA_OPENID_CONFIGURATION: Final = { + "token_endpoint": f"https://login.microsoftonline.com/{_ENTRA_TENANT_ID}/oauth2/v2.0/token", + "authorization_endpoint": f"https://login.microsoftonline.com/{_ENTRA_TENANT_ID}/oauth2/v2.0/authorize", + "issuer": f"https://login.microsoftonline.com/{_ENTRA_TENANT_ID}/v2.0", +} + + +class _FakeEntraAdapter(requests.adapters.BaseAdapter): + def __init__(self, status_code: int | None, body: Mapping[str, object] | str) -> None: + super().__init__() + self.status_code: Final = status_code + self.body: Final = body + + def send(self, request: requests.PreparedRequest, *args: object, **kwargs: object) -> requests.Response: + if "openid-configuration" in str(request.url): + return self._answer(request, 200, _ENTRA_OPENID_CONFIGURATION) + if self.status_code is None: + raise requests.exceptions.ConnectionError("Failed to establish a new connection: Connection refused") + return self._answer(request, self.status_code, self.body) + + def close(self) -> None: + pass + + @staticmethod + def _answer( + request: requests.PreparedRequest, status_code: int, body: Mapping[str, object] | str + ) -> requests.Response: + response: Final = requests.Response() + response.status_code = status_code + response.raw = io.BytesIO((body if isinstance(body, str) else json.dumps(body)).encode()) + response.headers = requests.structures.CaseInsensitiveDict( + {"Content-Type": "application/json", "Retry-After": "0"} + ) + response.url = str(request.url) + response.request = request + return response + + +def _entra_answering(status_code: int | None, body: Mapping[str, object] | str = "") -> RequestsTransport: + session: Final = requests.Session() + session.mount("https://", _FakeEntraAdapter(status_code, body)) + return RequestsTransport(session=session, session_owner=False) + + +def _entra_error(aadsts_code: str, description: str) -> Mapping[str, object]: + return { + "error": "invalid_client", + "error_description": f"{aadsts_code}: {description} Trace ID: {_ENTRA_TRACE_ID}", + "error_codes": [int(aadsts_code.removeprefix("AADSTS"))], + } + + +def _real_credential_token_provider(credential_kind: str, entra: RequestsTransport) -> Callable[[], str]: + from azure.identity import ( + ChainedTokenCredential, + ClientAssertionCredential, + ClientSecretCredential, + get_bearer_token_provider, + ) + + client_secret: Final = ClientSecretCredential( + _ENTRA_TENANT_ID, _ENTRA_APP_CLIENT_ID, "wrong-secret", transport=entra, disable_instance_discovery=True + ) + credentials: Final = { + "client_secret": client_secret, + "default_azure_credential_chain": ChainedTokenCredential(client_secret), + "workload_identity": ClientAssertionCredential( + _ENTRA_TENANT_ID, _ENTRA_APP_CLIENT_ID, lambda: "federated-token", transport=entra, retry_total=0 + ), + } + return get_bearer_token_provider(credentials[credential_kind], _ENTRA_SCOPE) + + +_REAL_CREDENTIAL_REJECTIONS: Final = { + "wrong_secret": ("client_secret", "AADSTS7000215", "Invalid client secret provided."), + "wrong_secret_through_a_credential_chain": ( + "default_azure_credential_chain", + "AADSTS7000215", + "Invalid client secret provided.", + ), + "unknown_app_through_workload_identity": ( + "workload_identity", + "AADSTS700016", + f"Application with identifier '{_ENTRA_APP_CLIENT_ID}' was not found in the directory.", + ), + "unknown_federated_subject_through_workload_identity": ( + "workload_identity", + "AADSTS700213", + "No matching federated identity record found for presented assertion subject 'system:serviceaccount:qa:proxy'.", + ), +} + + +@pytest.mark.parametrize( + ("credential_kind", "aadsts_code", "description"), + _REAL_CREDENTIAL_REJECTIONS.values(), + ids=_REAL_CREDENTIAL_REJECTIONS.keys(), +) +def test_entra_rejection_through_a_real_azure_identity_credential_answers_401( + no_azure_api_key, credential_kind, aadsts_code, description +): + entra: Final = _entra_answering(401, _entra_error(aadsts_code, description)) + + with pytest.raises(litellm.AuthenticationError) as excinfo: + _azure_chat(_real_credential_token_provider(credential_kind, entra)) + + assert excinfo.value.status_code == 401 + assert aadsts_code in str(excinfo.value) + assert _ENTRA_APP_CLIENT_ID not in str(excinfo.value) + assert _ENTRA_TRACE_ID not in str(excinfo.value) + + +_REAL_CREDENTIAL_NON_REJECTIONS: Final = { + "unreachable_through_a_credential_chain": ("default_azure_credential_chain", None, ""), + "html_503_through_a_credential_chain": ( + "default_azure_credential_chain", + 503, + "Service Unavailable", + ), + "json_503_through_a_credential_chain": ( + "default_azure_credential_chain", + 503, + _entra_error("AADSTS90033", "A transient error has occurred."), + ), + "throttled_client_secret": ( + "client_secret", + 429, + _entra_error("AADSTS50196", "The server terminated an operation because it encountered a client request loop."), + ), + "tenant_throttled_client_secret": ( + "client_secret", + 429, + _entra_error("AADSTS90055", "There are too many incoming requests."), + ), + "request_budget_exceeded_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90024", "A transient error has occurred. Try again."), + ), + "retryable_error_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS50033", "A transient error has occurred."), + ), + "external_server_retryable_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90006", "The service is temporarily unavailable."), + ), + "graph_retryable_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90090", "The service is temporarily unavailable."), + ), + "token_issuance_error_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS50000", "There was an error issuing a token or an issue with our sign-in service."), + ), + "throttled_without_a_code_through_a_credential_chain": ( + "default_azure_credential_chain", + 429, + {"error": "temporarily_unavailable", "error_description": "Too many requests."}, + ), + "json_503_through_workload_identity": ( + "workload_identity", + 503, + _entra_error("AADSTS90033", "A transient error has occurred."), + ), + "throttled_workload_identity_naming_a_rejection_code": ( + "workload_identity", + 429, + _entra_error("AADSTS7000215", "Invalid client secret provided."), + ), +} + + +@pytest.mark.parametrize( + ("credential_kind", "status_code", "body"), + _REAL_CREDENTIAL_NON_REJECTIONS.values(), + ids=_REAL_CREDENTIAL_NON_REJECTIONS.keys(), +) +def test_entra_outage_or_throttle_through_a_real_azure_identity_credential_stays_retryable( + no_azure_api_key, credential_kind, status_code, body +): + with pytest.raises(openai.APIError) as excinfo: + _azure_chat(_real_credential_token_provider(credential_kind, _entra_answering(status_code, body))) + + assert not isinstance(excinfo.value, litellm.AuthenticationError) + assert excinfo.value.status_code != 401 + + +def _azure_chat_with_oidc_token( + respx_mock, + monkeypatch, + entra_status_code: int, + entra_body: Mapping[str, object], + entra_headers: Mapping[str, str] | None = None, +): + monkeypatch.setenv("LITELLM_QA_FEDERATED_TOKEN", "federated-token") + monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) + respx_mock.post(f"https://login.microsoftonline.com/{_ENTRA_TENANT_ID}/oauth2/v2.0/token").mock( + return_value=httpx.Response(entra_status_code, json=entra_body, headers=entra_headers) + ) + return litellm.completion( + model="azure/gpt-4.1", + api_base="https://litellm-qa.openai.azure.com", + api_version="2025-04-01-preview", + messages=[{"role": "user", "content": "hi"}], + azure_ad_token="oidc/env/LITELLM_QA_FEDERATED_TOKEN", + tenant_id=_ENTRA_TENANT_ID, + client_id=_ENTRA_APP_CLIENT_ID, + max_retries=0, + ) + + +def test_entra_rejecting_an_oidc_token_answers_401_with_only_the_aadsts_reason(no_azure_api_key, respx_mock, monkeypatch): + rejection: Final = _entra_error( + "AADSTS700213", f"No matching federated identity record found for app '{_ENTRA_APP_CLIENT_ID}'." + ) + + with pytest.raises(litellm.AuthenticationError) as excinfo: + _azure_chat_with_oidc_token(respx_mock, monkeypatch, 400, rejection) + + assert excinfo.value.status_code == 401 + assert "AADSTS700213" in str(excinfo.value) + assert _ENTRA_APP_CLIENT_ID not in str(excinfo.value) + assert _ENTRA_TRACE_ID not in str(excinfo.value) + + +def test_entra_answering_an_oidc_exchange_with_a_transient_code_keeps_its_status( + no_azure_api_key, respx_mock, monkeypatch +): + with pytest.raises(openai.APIError) as excinfo: + _azure_chat_with_oidc_token( + respx_mock, monkeypatch, 400, _entra_error("AADSTS90024", "A transient error has occurred. Try again.") + ) + + assert excinfo.value.status_code == 400 + assert not isinstance(excinfo.value, litellm.AuthenticationError) + + +def test_entra_throttling_an_oidc_exchange_keeps_its_retry_after(no_azure_api_key, respx_mock, monkeypatch): + with pytest.raises(litellm.RateLimitError) as excinfo: + _azure_chat_with_oidc_token( + respx_mock, + monkeypatch, + 429, + _entra_error("AADSTS90055", "TenantThrottlingError: There are too many pending requests."), + {"Retry-After": "7"}, + ) + + assert excinfo.value.response.headers.get("retry-after") == "7" + + +class _EntraServiceUnavailableAnswer: + status_code: Final = 503 + reason: Final = "Service Unavailable" + + def text(self, encoding: str | None = None) -> str: + return "Service Unavailable" + + +@pytest.mark.parametrize("call", _AZURE_CALLS.values(), ids=_AZURE_CALLS.keys()) +@pytest.mark.parametrize("credential_kind", ["client_secret", "managed_identity"]) +def test_entra_answering_a_server_error_stays_a_retryable_server_error(no_azure_api_key, call, credential_kind): + from azure.core.exceptions import ClientAuthenticationError + from msal.exceptions import MsalServiceError + + token_providers: Final = { + "client_secret": _azure_identity_wrapped_token_provider( + MsalServiceError("HTTP Error: 503", error=None, error_description=None) + ), + "managed_identity": _raising_token_provider( + ClientAuthenticationError(message="No token received.", response=_EntraServiceUnavailableAnswer()) + ), + } + + with pytest.raises(openai.APIError) as excinfo: + call(token_providers[credential_kind]) + + assert not isinstance(excinfo.value, litellm.AuthenticationError) + assert excinfo.value.status_code != 401 + + +def test_token_provider_error_without_azure_core_installed_keeps_the_generic_token_error(monkeypatch): + monkeypatch.setitem(sys.modules, "azure.core.exceptions", None) + + with pytest.raises(RuntimeError, match="Failed to get Azure AD token: token service down"): + get_azure_ad_token( + GenericLiteLLMParams(azure_ad_token_provider=_raising_token_provider(ValueError("token service down"))) + ) + + +def test_image_generation_with_an_api_key_ignores_a_non_callable_azure_ad_token_provider(): + import respx + + with respx.mock: + image_route: Final = respx.post(url__startswith="https://litellm-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"created": 1, "data": [{"b64_json": "aGk="}]}) + ) + response: Final = litellm.image_generation( + model="azure/gpt-image-1", + prompt="a red dot", + api_base="https://litellm-qa.openai.azure.com", + api_version="2025-04-01-preview", + api_key="configured-key", + azure_ad_token_provider="not-a-callable", + ) + + assert image_route.calls.last.request.headers["api-key"] == "configured-key" + assert response.data[0].b64_json == "aGk=" diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index faa33e559ca..7ce5d8b6a02 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -171,6 +171,79 @@ async def test_upstream_error_status_becomes_an_error_response_with_that_status_ ) +@pytest.mark.asyncio +async def test_entra_rejecting_the_deployment_oidc_token_becomes_a_401_error_response_without_counting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LITELLM_QA_FEDERATED_TOKEN", "federated-token") + monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) + with respx.mock: + respx.post("https://login.microsoftonline.com/qa-tenant/oauth2/v2.0/token").mock( + return_value=httpx.Response( + 401, + json={ + "error": "invalid_client", + "error_description": "AADSTS700211: No matching federated identity record found for presented " + "assertion issuer 'https://oidc.litellm-qa.invalid'. Trace ID: 0f5c2b1e", + "error_codes": [700211], + }, + ) + ) + count_route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 6})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={ + "litellm_params": { + "api_base": API_BASE, + "tenant_id": "qa-tenant", + "client_id": "qa-client", + "azure_ad_token": "oidc/env/LITELLM_QA_FEDERATED_TOKEN", + } + }, + request_model="azure_ai/claude-sonnet-4-5", + ) + + assert result is not None + assert (result.error, result.status_code) == (True, 401) + assert "AADSTS700211" in (result.error_message or "") + assert not count_route.called + + +@pytest.mark.asyncio +async def test_entra_throttling_the_deployment_oidc_token_exchange_keeps_its_429( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LITELLM_QA_FEDERATED_TOKEN", "federated-token") + monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) + with respx.mock: + respx.post("https://login.microsoftonline.com/throttled-tenant/oauth2/v2.0/token").mock( + return_value=httpx.Response(429, json={"error": "temporarily_unavailable", "error_codes": [90055]}) + ) + count_route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 6})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={ + "litellm_params": { + "api_base": API_BASE, + "tenant_id": "throttled-tenant", + "client_id": "qa-client", + "azure_ad_token": "oidc/env/LITELLM_QA_FEDERATED_TOKEN", + } + }, + request_model="azure_ai/claude-sonnet-4-5", + ) + + assert result is not None + assert (result.error, result.status_code) == (True, 429) + assert not count_route.called + + @pytest.mark.asyncio async def test_undecodable_count_body_becomes_a_500_error_response_carrying_the_decoder_message( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index b1d52c9dd51..6b34589d1ee 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1360,7 +1360,7 @@ def test_azure_ai_services_handler(api_base, expected_url): with patch.object(client, "post") as mock_client: try: response = litellm.completion( - model="azure_ai/Meta-Llama-3.1-70B-Instruct", + model="azure_ai/Llama-3.3-70B-Instruct", messages=[{"role": "user", "content": "Hello, how are you?"}], api_key="my-fake-api-key", api_base=api_base, @@ -1385,7 +1385,7 @@ def test_azure_ai_services_with_api_version(): with patch.object(client, "post") as mock_client: try: response = litellm.completion( - model="azure_ai/Meta-Llama-3.1-70B-Instruct", + model="azure_ai/Llama-3.3-70B-Instruct", messages=[{"role": "user", "content": "Hello, how are you?"}], api_key="my-fake-api-key", api_version="2024-05-01-preview", diff --git a/tests/unit/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py b/tests/unit/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py index 3de27199e2e..80e37655b60 100644 --- a/tests/unit/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py +++ b/tests/unit/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py @@ -9,7 +9,7 @@ from litellm.llms.azure_ai.rerank.transformation import AzureAIRerankConfig class TestAzureAIRerankConfigGetCompleteUrl: def setup_method(self): self.config = AzureAIRerankConfig() - self.model = "azure_ai/cohere-rerank-v3-english" + self.model = "azure_ai/cohere-rerank-v4.0-pro" def test_api_base_required(self): with pytest.raises(ValueError, match='Azure AI API Base is required\\. api_base=None\\. Set in') as exc_info: @@ -99,7 +99,7 @@ class TestAzureAIRerankConfigValidateEnvironment: def test_uses_api_key_when_set(self): headers = AzureAIRerankConfig().validate_environment( headers={}, - model="azure_ai/cohere-rerank-v3-english", + model="azure_ai/cohere-rerank-v4.0-pro", api_key="my-key", ) @@ -111,7 +111,7 @@ class TestAzureAIRerankConfigValidateEnvironment: headers = AzureAIRerankConfig().validate_environment( headers={}, - model="azure_ai/cohere-rerank-v3-english", + model="azure_ai/cohere-rerank-v4.0-pro", litellm_params={"azure_ad_token": "entra-token"}, ) diff --git a/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py b/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py index 908e727bc4c..a8958b301db 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py @@ -5,9 +5,15 @@ Every azure_ai route must authenticate with an Entra ID token when no API key is instead of requiring an API key. """ +import json +import queue +from collections.abc import Mapping +from typing import Final from unittest.mock import patch +import httpx import pytest +import respx import litellm from litellm.llms.azure_ai.common_utils import ( @@ -16,6 +22,8 @@ from litellm.llms.azure_ai.common_utils import ( has_azure_entra_params, resolve_azure_ai_agent_auth_header, ) +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import HTTPHandler ENTRA_PARAMS = {"azure_ad_token": "entra-token"} @@ -246,6 +254,57 @@ def test_agent_entra_values_resolve_os_environ_references(monkeypatch): assert get_azure_ai_agent_entra_token({"azure_ad_token": "os.environ/FOUNDRY_AGENT_AD_TOKEN"}) == "token-from-env" +_AGENT_APP_CLIENT_ID: Final = "0d08e61f-6184-4980-9cef-a2e9128f4cb2" +_AGENT_CREDENTIALS: Final = { + "service_principal": ( + "litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id", + {"tenant_id": "tenant", "client_id": _AGENT_APP_CLIENT_ID, "client_secret": "wrong-secret"}, + ), + "username_password": ( + "litellm.llms.azure.common_utils.get_azure_ad_token_from_username_password", + {"client_id": _AGENT_APP_CLIENT_ID, "azure_username": "agent@example.com", "azure_password": "wrong"}, + ), +} + + +def _raising(error: Exception): + def token() -> str: + raise error + + return token + + +@pytest.mark.parametrize(("token_fetch", "agent_params"), _AGENT_CREDENTIALS.values(), ids=_AGENT_CREDENTIALS.keys()) +def test_agent_entra_rejection_raises_401_with_only_the_aadsts_reason(token_fetch, agent_params): + from azure.core.exceptions import ClientAuthenticationError + + rejection: Final = ClientAuthenticationError( + message="Authentication failed: AADSTS7000215: Invalid client secret provided. Ensure the secret being sent " + f"in the request is the client secret value, for a secret added to app '{_AGENT_APP_CLIENT_ID}'." + ) + + with patch(token_fetch, return_value=_raising(rejection)): # test-quality-ok: stands in for the Entra token endpoint answering a wrong credential, the shape azure-identity raises + with pytest.raises(litellm.AuthenticationError) as excinfo: + get_azure_ai_agent_entra_token(agent_params) + + assert excinfo.value.status_code == 401 + assert "AADSTS7000215" in str(excinfo.value) + assert _AGENT_APP_CLIENT_ID not in str(excinfo.value) + + +def test_agent_entra_endpoint_unreachable_is_not_an_auth_error(): + from azure.core.exceptions import ClientAuthenticationError + + unreachable: Final = ClientAuthenticationError(message="Authentication failed: Connection refused") + token_fetch, agent_params = _AGENT_CREDENTIALS["service_principal"] + + with patch(token_fetch, return_value=_raising(unreachable)): # test-quality-ok: stands in for an unreachable Entra token endpoint, the shape azure-identity raises + with pytest.raises(ClientAuthenticationError) as excinfo: + get_azure_ai_agent_entra_token(agent_params) + + assert not isinstance(excinfo.value, litellm.AuthenticationError) + + def test_agent_entra_token_failure_names_the_credential_fields(): with pytest.raises(ValueError, match="client_secret"): get_azure_ai_agent_entra_token({"azure_scope": "https://ai.azure.com/.default"}) @@ -290,3 +349,159 @@ async def test_agent_auth_header_is_the_entra_bearer(): headers = await resolve_azure_ai_agent_auth_header({"azure_ad_token": "entra-token"}) assert headers == {"Authorization": "Bearer entra-token"} + + +OPENAI_CHAT_COMPLETION: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +OPENAI_EMBEDDING: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "cohere-embed-v3-english", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, +} +ANTHROPIC_MESSAGE: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "OK"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} + + +class _PreCallOptionalParamsCapture(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.optional_params: tuple[dict[str, object], ...] = () + + def log_pre_api_call(self, model: str, messages: list[object], kwargs: dict[str, dict[str, object]]) -> None: + self.optional_params = (*self.optional_params, kwargs["optional_params"]) + + +@pytest.mark.parametrize( + ("model", "provider_response"), + [ + pytest.param("azure_ai/gpt-6.1-sol", OPENAI_CHAT_COMPLETION, id="openai-shaped"), + pytest.param("azure_ai/claude-sonnet-5-5", ANTHROPIC_MESSAGE, id="claude"), + ], +) +def test_completion_sends_the_entra_token_only_as_the_bearer_header( + model: str, provider_response: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +): + capture: Final = _PreCallOptionalParamsCapture() + monkeypatch.setattr(litellm, "callbacks", [capture]) + sent: Final[queue.Queue[httpx.Request]] = queue.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + sent.put_nowait(request) + return httpx.Response(200, json=provider_response) + + response: Final = litellm.completion( + model=model, + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + messages=[{"role": "user", "content": "hi"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + ) + + request: Final = sent.get_nowait() + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + assert response.choices[0].message.content == "OK" + assert capture.optional_params + assert "entra-token" not in json.dumps(capture.optional_params, default=str) + + +@pytest.mark.respx(assert_all_called=True) +def test_embedding_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=OPENAI_EMBEDDING) + ) + + litellm.embedding( + model="azure_ai/cohere-embed-v3-english", + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + input=["hello"], + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + + +IMAGE_RESPONSE: Final = {"created": 1, "data": [{"b64_json": "aGk="}]} +PNG_BYTES: Final = b"\x89PNG\r\n\x1a\n" + b"\x00" * 32 + + +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize( + "extra_kwargs", + [{}, {"extra_body": {"azure_ad_token": "entra-token", "client_secret": "entra-token"}}], + ids=["kwarg", "extra_body"], +) +def test_image_edit_sends_the_entra_token_only_as_the_bearer_header( + respx_mock: respx.MockRouter, extra_kwargs: dict[str, dict[str, str]] +): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=IMAGE_RESPONSE) + ) + + litellm.image_edit( + model="azure_ai/FLUX.1-Kontext-pro", + api_base="https://example-resource.services.ai.azure.com", + api_version="2025-04-01-preview", + azure_ad_token="entra-token", + image=PNG_BYTES, + prompt="add a hat", + **extra_kwargs, + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert b"entra-token" not in request.content + + +@pytest.mark.respx(assert_all_called=True) +def test_image_generation_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=IMAGE_RESPONSE) + ) + + litellm.image_generation( + model="azure_ai/FLUX-1.1-pro", + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + prompt="a red circle", + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + + +@pytest.mark.respx(assert_all_called=True) +def test_transcription_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hi"}) + ) + + litellm.transcription( + model="azure_ai/whisper", + api_base="https://example-resource.services.ai.azure.com", + api_version="2024-06-01", + azure_ad_token="entra-token", + file=("hello.wav", b"RIFF" + b"\x00" * 32), + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert b"entra-token" not in request.content diff --git a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 9b20192c3f2..a5b66efe7cd 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -22,6 +22,7 @@ TOKEN_PRICED_NAMES: Final = ( "cohere-command-a", "grok-4-20-reasoning", "grok-4-20-non-reasoning", + "claude-mythos-5-1", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 041c625eaef..2ef202762ff 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -12,6 +12,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsInputLimits, + DecisionsResponseValidationError, ir_to_systemone_request, ir_to_systemone_response, parse_systemone_response, @@ -21,6 +22,7 @@ from litellm.llms.openai.decisions.transformation import openai_request_to_ir from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig from litellm.types.decisions import ( MAX_DECISION_QUESTIONS, + DecisionsIRChoiceAnswer, DecisionsIRRefusal, DecisionsIRRequest, DecisionsIRScoreAnswer, @@ -347,6 +349,49 @@ def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None: assert response.model_dump(mode="json") == payload +@pytest.mark.parametrize( + ("question", "answer"), + [ + ( + {"type": "choice", "criteria": {"yes": "Yes", "no": "No"}}, + {"type": "choice", "choice": "yes", "probabilities": {"yes": 0.9, "no": 0.1}}, + ), + ( + {"type": "score", "criteria": ["low", "high"]}, + {"type": "score", "score": 0.5, "legend": {"0": "low", "1": "high"}, "probabilities": {"0": 0.5, "1": 0.5}}, + ), + ], +) +@pytest.mark.parametrize("confidence", ({}, {"confidence": None})) +@pytest.mark.parametrize( + ("provider", "optional"), (("nace", False), ("fireworks_ai", False), ("parallel_ai", True)) +) +def test_only_a_provider_without_confidence_may_omit_it( + question: Mapping[str, object], + answer: Mapping[str, object], + confidence: Mapping[str, object], + provider: str, + optional: bool, +) -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders(provider)) + assert config is not None + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate({"state": {"text": "Example"}, "questions": {"asked": question}}) + ) + upstream: Final = httpx.Response( + 200, json={"mode": "fast", "model": "decision", "answers": {"asked": {**answer, **confidence}}} + ) + + if optional: + parsed: Final = config.transform_decisions_response("decision", provider, upstream, request) + assert isinstance(parsed.answers[0], (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) + assert parsed.answers[0].confidence is None + return + with pytest.raises(DecisionsResponseValidationError) as raised: + config.transform_decisions_response("decision", provider, upstream, request) + assert raised.value.status_code == 500 + + @pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe")) @pytest.mark.parametrize("legend", (["low", "high"], {"0": "low", "1": "high"})) def test_score_legends_preserve_labels_probabilities_and_metadata(provider: str, legend: object) -> None: diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 05d755b1215..93f6e237fc7 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -18,7 +18,7 @@ from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.unit.conftest import NativeStructuredOutputKey from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -562,19 +562,27 @@ def _invoke_params_for_schema( @pytest.mark.parametrize( - ("native_key", "thinking_params", "native"), + ("model", "native_key", "thinking_params", "native"), [ - ("legacy", {}, True), - ("bedrock", {}, False), - ("bedrock", {"thinking": {"type": "adaptive"}}, True), - ("bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "legacy", {}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {}, False), + ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), + ], + ids=[ + "legacy-key", + "bedrock-key-thinking-by-default", + "bedrock-key-thinking-off-by-default", + "bedrock-key-adaptive", + "bedrock-key-effort", + "bedrock-key-thinking-disabled", ], - ids=["legacy-key", "bedrock-key-no-thinking", "bedrock-key-adaptive", "bedrock-key-effort"], ) -def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( - set_native_structured_output, native_key, thinking_params, native +def test_bedrock_native_output_key_applies_only_when_the_model_will_think( + set_native_structured_output, model, native_key, thinking_params, native ): - model = "us.anthropic.claude-haiku-5-5" set_native_structured_output(model, native_key) schema = {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]} @@ -914,9 +922,7 @@ def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_th assert result["output_config"] == {"effort": "high"} -def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(set_native_structured_output): - """Anthropic accepts a forced tool_choice next to adaptive thinking, and dropping - it lets the model answer in prose instead of the requested JSON.""" +def test_bedrock_chat_invoke_adaptive_thinking_leaves_the_response_format_tool_unforced(set_native_structured_output): set_native_structured_output("us.anthropic.claude-opus-5", None) result = AmazonAnthropicClaudeConfig().map_openai_params( non_default_params={ @@ -935,7 +941,8 @@ def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_ drop_params=False, ) - assert result["tool_choice"] == {"type": "tool", "name": result["tools"][0]["name"]} + assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] + assert "tool_choice" not in result assert result["thinking"] == {"type": "adaptive"} @@ -1052,6 +1059,64 @@ async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sourc } in captured["body"]["messages"][0]["content"] +def test_bedrock_invoke_claude_sync_completion_downloads_a_pdf_link_once_and_sends_it_as_a_base64_document(monkeypatch): + from litellm.litellm_core_utils.prompt_templates import image_handling + + pdf_url = f"https://docs.example/{uuid.uuid4()}/report.pdf" + pdf_bytes = b"%PDF-1.7\n" + b"0" * 64 + fetched = [] + captured = {} + + def serve_pdf(client, url, **kwargs): + fetched.append(url) + return httpx.Response( + 200, content=pdf_bytes, headers={"content-type": "application/pdf"}, request=httpx.Request("GET", url) + ) + + def handle(request): + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "us.anthropic.claude-sonnet-5", + "content": [{"type": "text", "text": "A lease"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "safe_get", serve_pdf) + client = HTTPHandler() + client.client = httpx.Client(transport=httpx.MockTransport(handle)) + + response = litellm.completion( + model="bedrock/invoke/us.anthropic.claude-sonnet-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this document?"}, + {"type": "image_url", "image_url": {"url": pdf_url}}, + ], + } + ], + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="fake-secret", + aws_region_name="us-east-1", + client=client, + ) + + assert response.choices[0].message.content == "A lease" + assert fetched == [pdf_url] + assert captured["body"]["messages"][0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": base64.b64encode(pdf_bytes).decode()}, + } + + @pytest.mark.parametrize( "model, expected_betas", [ @@ -2026,7 +2091,7 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co (False, True, None, {"thinking": {"type": "adaptive"}}, True), (False, False, None, {}, False), (False, None, {"supports_bedrock_native_structured_output": False}, {}, False), - (True, True, None, {}, False), + (True, True, None, {"thinking": {"type": "disabled"}}, False), ], ids=[ "cannot-force-tool", diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 8cab1018d25..b8eb5858fda 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -15,6 +15,7 @@ import respx import litellm from litellm._logging import verbose_logger from litellm import ModelResponse, Router, completion +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, _bedrock_tools_pt, @@ -1440,18 +1441,27 @@ def test_thinking_request_adds_output_tokens_details_response_path(): assert result["additionalModelResponseFieldPaths"] == ("/usage/output_tokens_details",) -def test_request_without_thinking_omits_response_field_paths(): - config = AmazonConverseConfig() - - result = config._transform_request( - model="bedrock/converse/us.anthropic.claude-opus-4-7", +@pytest.mark.parametrize( + ("model", "thinks_by_default"), + [ + ("bedrock/converse/us.anthropic.claude-opus-4-7", False), + ("bedrock/converse/us.anthropic.claude-opus-5", True), + ("us.anthropic.claude-haiku-5-5", True), + ], + ids=["opus-4-7-thinking-off-by-default", "opus-5-thinking-on-by-default", "haiku-5-5-thinking-on-by-default"], +) +def test_request_without_thinking_asks_for_thinking_tokens_only_when_the_model_thinks_by_default( + local_model_cost_map: None, model: str, thinks_by_default: bool +): + result = AmazonConverseConfig()._transform_request( + model=model, messages=[{"role": "user", "content": "hi"}], optional_params={"maxTokens": 256}, litellm_params={}, headers={}, ) - assert "additionalModelResponseFieldPaths" not in result + assert (result.get("additionalModelResponseFieldPaths") == ("/usage/output_tokens_details",)) is thinks_by_default def test_transform_usage_prefers_provider_reasoning_tokens(): @@ -4937,7 +4947,6 @@ def test_translate_response_format_native_output_config(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, - is_thinking_enabled=False, ) # Should have outputConfig, NOT tools @@ -4965,20 +4974,20 @@ def test_translate_response_format_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -CAPITAL_RESPONSE_FORMAT: Final = { - "type": "json_schema", - "json_schema": { - "name": "capital", - "schema": { - "type": "object", - "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, - "required": ["city", "country"], - "additionalProperties": False, - }, +CAPITAL_JSON_SCHEMA: Final = { + "name": "capital", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, }, } +CAPITAL_RESPONSE_FORMAT: Final = {"type": "json_schema", "json_schema": CAPITAL_JSON_SCHEMA} + + ADAPTIVE_THINKING: Final = {"type": "adaptive"} @@ -5030,24 +5039,124 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag assert "fake_stream" not in native_params assert "outputConfig" not in tool_request assert [tool["toolSpec"]["name"] for tool in tool_request["toolConfig"]["tools"]] == ["json_tool_call"] - assert tool_request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert "toolChoice" not in tool_request["toolConfig"] assert tool_params.get("fake_stream", False) is stream +_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}} + + @pytest.mark.parametrize( - ("native_key", "thinking_params", "native"), + ("model", "native_key", "model_info", "thinking_params", "forced"), [ - ("legacy", {}, True), - ("bedrock", {}, False), - ("bedrock", {"thinking": ADAPTIVE_THINKING}, True), - ("bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), - ("bedrock", {"reasoning_effort": "low"}, True), - ("bedrock", {"reasoning_effort": "none"}, False), - ("bedrock", {"thinking": {"type": "disabled"}}, False), + ( + "us.anthropic.claude-opus-4-8", + "bedrock", + {"supports_bedrock_native_structured_output": False}, + {"thinking": ADAPTIVE_THINKING}, + False, + ), + ("us.anthropic.claude-opus-5", None, None, {}, False), + ("global.anthropic.claude-sonnet-5", None, None, {}, False), + ("us.anthropic.claude-haiku-5-5", None, None, {}, False), + ("us.anthropic.claude-fable-5", None, None, {}, False), + ("us.anthropic.claude-fable-5", None, None, {"reasoning_effort": "none"}, False), + ("us.anthropic.claude-opus-4-8", None, None, {}, True), + ("us.anthropic.claude-opus-5", None, None, {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-haiku-5-5", None, None, {"reasoning_effort": "none"}, True), + ], + ids=[ + "opus-4-8-adaptive-native-output-fallback", + "opus-5-default-thinking", + "sonnet-5-default-thinking", + "haiku-5-5-default-thinking", + "fable-5-always-on", + "fable-5-always-on-reasoning-effort-none", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + "haiku-5-5-reasoning-effort-none", + ], +) +def test_json_tool_is_forced_only_when_the_model_will_not_think( + set_native_structured_output, model, native_key, model_info, thinking_params, forced +): + set_native_structured_output(model, native_key) + config: Final = AmazonConverseConfig() + + response_format: Final = { + "type": "json_schema", + "json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"}, + } + + optional_params: Final = config.map_openai_params( + non_default_params={"response_format": response_format, **thinking_params}, + optional_params={}, + model=model, + drop_params=False, + model_info=model_info, + ) + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert (request["toolConfig"].get("toolChoice") == _JSON_TOOL_CHOICE) is forced + assert request["toolConfig"]["tools"][0]["toolSpec"]["description"] == ( + "The capital of a country" + if forced + else f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} The capital of a country" + ) + + +def test_unforced_json_tool_keeps_the_schema_description_on_a_non_claude_model(set_native_structured_output): + model: Final = "us.meta.llama3-3-70b-instruct-v1:0" + set_native_structured_output(model, None) + config: Final = AmazonConverseConfig() + + optional_params: Final = config.map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"}, + } + }, + optional_params={}, + model=model, + drop_params=False, + ) + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "toolChoice" not in request["toolConfig"] + assert [tool["toolSpec"]["description"] for tool in request["toolConfig"]["tools"]] == ["The capital of a country"] + + +@pytest.mark.parametrize( + ("model", "native_key", "thinking_params", "native"), + [ + ("us.anthropic.claude-haiku-5-5", "legacy", {}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {}, False), + ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": ADAPTIVE_THINKING}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"reasoning_effort": "none"}, False), + ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), ], ids=[ "legacy-key", - "bedrock-key-no-thinking", + "bedrock-key-thinking-by-default", + "bedrock-key-thinking-off-by-default", "bedrock-key-adaptive", "bedrock-key-enabled", "bedrock-key-effort", @@ -5055,10 +5164,9 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag "bedrock-key-thinking-disabled", ], ) -def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( - set_native_structured_output, native_key, thinking_params, native +def test_bedrock_native_output_key_applies_only_when_the_model_will_think( + set_native_structured_output, model, native_key, thinking_params, native ): - model: Final = "us.anthropic.claude-haiku-5-5" set_native_structured_output(model, native_key) _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5078,7 +5186,7 @@ _FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}} (False, True, None, {"thinking": {"type": "adaptive"}}, True, None), (False, False, None, None, False, None), (False, None, {"supports_bedrock_native_structured_output": False}, None, False, None), - (True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE), + (True, True, None, {"thinking": {"type": "disabled"}}, False, _FORCED_JSON_TOOL_CHOICE), ], ids=[ "cannot-force-tool", @@ -5596,7 +5704,6 @@ def test_translate_response_format_fallback_tool_call(): model="anthropic.claude-3-haiku-20240307-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, - is_thinking_enabled=False, ) # Should use tool-call approach, NOT outputConfig (model doesn't support native structured outputs) @@ -5633,7 +5740,6 @@ def test_native_structured_output_no_fake_stream(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format, "stream": True}, - is_thinking_enabled=False, ) assert "outputConfig" in result @@ -5954,7 +6060,6 @@ def test_json_object_no_schema_skips_tool_injection(monkeypatch): model="anthropic.claude-sonnet-4-5-20250929-v1:0", optional_params=optional_params, non_default_params=non_default_params, - is_thinking_enabled=False, ) # Should NOT use native outputConfig (no schema provided) @@ -10073,6 +10178,461 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): assert "cachePoint" not in json.dumps(transformed_messages) +def test_converse_carries_prompt_cache_breakpoint_to_cache_point(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params={}, + litellm_params={}, + ) + + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert request["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + + +def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(5) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + transform({}) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert transform({"drop_params": True})["messages"] == [ + { + "role": "user", + "content": [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + {"text": "part 4"}, + cache_point, + ], + } + ] + + +_CONVERSE_CAP_TOOL: Final[dict[str, object]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize( + "tool_cache_params", + [ + {"tools": [{**_CONVERSE_CAP_TOOL, "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [_CONVERSE_CAP_TOOL], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["client_tool_cache_control", "tool_config_injection_point"], +) +def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params=copy.deepcopy(tool_cache_params), + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + ] + assert request["toolConfig"]["tools"][-1] == cache_point + + + +@pytest.mark.parametrize( + "unsent_tool_params", + [ + { + "tools": [ + _CONVERSE_CAP_TOOL, + {"type": "tool_search_tool_regex_20251119", "name": "find", "cache_control": {"type": "ephemeral"}}, + ] + }, + {"tools": [_CONVERSE_CAP_TOOL, {"type": "web_search", "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [{"type": "web_search"}], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"], +) +@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"]) +def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( + unsent_tool_params: dict[str, object], litellm_params: dict[str, object] +): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ], + optional_params=copy.deepcopy(unsent_tool_params), + litellm_params=dict(litellm_params), + ) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + cache_point, + ] + assert cache_point not in request.get("toolConfig", {}).get("tools", []) + +def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(result_parts)}, + ] + + +def _marked_text_parts(count: int) -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + +def _marked_tool_result_parts() -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"result {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(2) + ] + + +def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + {"role": "user", "content": _marked_text_parts(3)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ], + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params={}, + ) + + assert len(_collect_cache_points(request["messages"])) == 4 + + +def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): + messages: Final = [ + {"role": "user", "content": _marked_text_parts(4)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + ] + assert len(_collect_cache_points(request["messages"][1:])) == 1 + + +def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result(): + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + {"role": "user", "content": "Look it up."}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="us.amazon.nova-lite-v1:0", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="tool result"): + transform({}) + + request: Final = transform({"drop_params": True}) + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert _collect_cache_points(request["messages"]) == [] + + +def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_when_flagged(local_model_cost_map): + from litellm import Router + + flagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + unflagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/mistral" + Router( + model_list=[ + { + "model_name": "claude-profile", + "litellm_params": {"model": f"bedrock/{flagged_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "mistral-profile", + "litellm_params": {"model": f"bedrock/{unflagged_arn}", "aws_region_name": "us-east-1"}, + }, + ] + ) + + def user_content(model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + assert user_content(flagged_arn, {}) == [{"text": "Hello"}, {"cachePoint": {"type": "default"}}] + assert user_content(unflagged_arn, {}) == [{"text": "Hello"}] + assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] + + + +def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): + from litellm import Router + + opted_in_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-next" + opted_out_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-opted-out" + Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": {"model": f"bedrock/{opted_in_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "opted-out", + "litellm_params": {"model": f"bedrock/{opted_out_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": False}, + }, + ] + ) + + def user_content(model: str, base_model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={"base_model": base_model}, + )["messages"][0]["content"] + + assert user_content(opted_in_arn, "anthropic.claude-unmapped-v1:0", {}) == [ + {"text": "Hello"}, + {"cachePoint": {"type": "default"}}, + ] + assert user_content(opted_out_arn, "anthropic.claude-sonnet-5-5", {"ttl": "30m"}) == [{"text": "Hello"}] + + +def test_converse_translates_prompt_cache_breakpoint_only_for_families_that_take_cache_points(local_model_cost_map): + litellm.register_model( + {"zai.glm-next-v1:0": {"litellm_provider": "bedrock_converse", "mode": "chat", "supports_prompt_caching": True}} + ) + + def user_content(model: str, mark: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", **mark}]}], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + marker: Final = {"prompt_cache_breakpoint": {"mode": "explicit"}} + client_mark: Final = {"cache_control": {"type": "ephemeral"}} + cache_point: Final = {"cachePoint": {"type": "default"}} + assert user_content("zai.glm-next-v1:0", marker) == [{"text": "Hello"}] + assert user_content("zai.glm-next-v1:0", client_mark) == [{"text": "Hello"}, cache_point] + assert user_content("us.anthropic.claude-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + assert user_content("us.amazon.nova-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + + +def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_point(local_model_cost_map): + guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}} + + def request(system_marks: int) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(system_marks) + ), + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + + assert request(0)["messages"][0]["content"] == [ + {"guardContent": {"text": {"text": "Hello"}}}, + {"cachePoint": {"type": "default"}}, + ] + with pytest.raises(litellm.UnsupportedParamsError): + request(4) + + client_marked: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(4) + ), + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + assert sum("cachePoint" in block for block in client_marked["system"]) == 4 + assert client_marked["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, + litellm_params={}, + ) + + assert request["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def _converse_request_with_30m_breakpoint(model: str, litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}} + ], + } + ], + optional_params={}, + litellm_params=litellm_params, + ) + + +def test_converse_rejects_30m_breakpoint_ttl_only_where_a_cache_point_is_written(): + claude: Final = "anthropic.claude-sonnet-5-5" + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _converse_request_with_30m_breakpoint(claude, {}) + + assert _converse_request_with_30m_breakpoint(claude, {"drop_params": True})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + assert _converse_request_with_30m_breakpoint("meta.llama3-1-8b-instruct-v1:0", {})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}]} + ] + + @pytest.mark.parametrize( "model, expected_supports_tool_call", [ diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 996cfec6e22..4424f7640a3 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1149,6 +1149,108 @@ async def test_converse_streaming_completion_yields_text_and_finish_reason( _assert_converse_stream_text_and_finish_reason(sync_chunks) +_CONVERSE_PROSE: Final = "Let me work this out. " + + +async def _stream_chunks( + request: Mapping[str, object], transport: httpx.MockTransport, async_mode: bool +) -> tuple[ModelResponseStream, ...]: + if not async_mode: + return tuple(litellm.completion(**request, client=HTTPHandler(client=httpx.Client(transport=transport)))) + async_stream: Final = await litellm.acompletion(**request, client=AsyncHTTPHandler(transport=transport)) + return tuple([chunk async for chunk in async_stream]) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize( + ("tool_block", "stop_reason", "expected_content", "expected_tools", "expected_finish_reason"), + [ + pytest.param( + {"toolUse": {"toolUseId": "tooluse_1", "name": "json_tool_call", "input": {"answer": 120}}}, + "tool_use", + '{"answer": 120}', + (), + "stop", + id="prose_then_json_tool", + ), + pytest.param( + {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}}, + "tool_use", + _CONVERSE_PROSE, + (("get_weather", '{"city": "Paris"}'),), + "tool_calls", + id="prose_then_caller_tool", + ), + pytest.param(None, "end_turn", _CONVERSE_PROSE, (), "stop", id="json_tool_skipped"), + ], +) +@pytest.mark.asyncio +async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_skips_the_json_tool( + _aws_test_credentials: None, + async_mode: bool, + tool_block: Mapping[str, object] | None, + stop_reason: str, + expected_content: str, + expected_tools: tuple[tuple[str, str], ...], + expected_finish_reason: str, +) -> None: + converse_reply: Final = { + "output": { + "message": { + "role": "assistant", + "content": [{"text": _CONVERSE_PROSE}, *((tool_block,) if tool_block is not None else ())], + } + }, + "stopReason": stop_reason, + "usage": {"inputTokens": 30, "outputTokens": 20, "totalTokens": 50}, + "metrics": {"latencyMs": 1}, + } + + def handle_request(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + assert "toolChoice" not in body["toolConfig"] + assert "json_tool_call" in [tool["toolSpec"]["name"] for tool in body["toolConfig"]["tools"]] + return httpx.Response(200, json=converse_reply) + + transport: Final = httpx.MockTransport(handle_request) + request: Final = { + "model": "bedrock/us.anthropic.claude-opus-5", + "messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"]}, + }, + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + "stream": True, + } + chunks: Final = await _stream_chunks(request, transport, async_mode) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == expected_content + tool_calls: Final = tuple( + itertools.chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in chunks) + ) + named_calls: Final = tuple( + (call.function.name, "".join(c.function.arguments or "" for c in tool_calls)) + for call in tool_calls + if call.function.name + ) + assert named_calls == expected_tools + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == [ + expected_finish_reason + ] + + def test_mistral_invoke_stream_yields_text_and_finish_reason( _aws_test_credentials: None, ) -> None: diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 889faf784bf..904ca3bb8ca 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -61,6 +61,7 @@ def test_bedrock_embedding_with_api_key_bearer_token(model, input_type, embed_re with patch.object(client, "post") as mock_post: mock_response = Mock() mock_response.status_code = 200 + mock_response.headers = httpx.Headers() mock_response.text = json.dumps(embed_response) mock_response.json = lambda: json.loads(mock_response.text) mock_post.return_value = mock_response @@ -774,6 +775,7 @@ def test_bedrock_embedding_header_forwarding(model, embed_response): with patch.object(client, "post") as mock_post: mock_response = Mock() mock_response.status_code = 200 + mock_response.headers = httpx.Headers() mock_response.text = json.dumps(embed_response) mock_response.json = lambda: json.loads(mock_response.text) mock_post.return_value = mock_response @@ -909,6 +911,7 @@ def test_bedrock_cohere_v4_embedding_response_parsing(): with patch.object(client, "post") as mock_post: mock_response = Mock() mock_response.status_code = 200 + mock_response.headers = httpx.Headers() mock_response.text = json.dumps(cohere_v4_response) mock_response.json = lambda: json.loads(mock_response.text) mock_post.return_value = mock_response @@ -1317,6 +1320,7 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list( with patch.object(client, "post") as mock_post: mock_response = Mock() mock_response.status_code = 200 + mock_response.headers = httpx.Headers() mock_response.text = json.dumps(cohere_embedding_response) mock_response.json = lambda: json.loads(mock_response.text) mock_post.return_value = mock_response @@ -1848,6 +1852,7 @@ def test_bedrock_embedding_models(model, input_type, embed_response): with patch.object(client, "post") as mock_post: mock_response = Mock() mock_response.status_code = 200 + mock_response.headers = httpx.Headers() mock_response.text = json.dumps(embed_response) mock_response.json = lambda: json.loads(mock_response.text) mock_post.return_value = mock_response diff --git a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py index aec1672b335..1c0ce5a334a 100644 --- a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py +++ b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py @@ -1394,6 +1394,7 @@ class _ScriptedGuardrail(CustomGuardrail): self.skip_system_message_in_guardrail = skip_system self.scan_only_tool_results = scan_only_tool_results self.received: tuple[GenericGuardrailAPIInputs, ...] = () + self.request_data: tuple[Mapping[str, object], ...] = () async def apply_guardrail( self, @@ -1403,6 +1404,7 @@ class _ScriptedGuardrail(CustomGuardrail): logging_obj: LiteLLMLoggingObj | None = None, ) -> Mapping[str, object]: self.received = (*self.received, copy.deepcopy(inputs)) + self.request_data = (*self.request_data, request_data) return self.reply(inputs) @@ -1429,16 +1431,30 @@ def _converse_request(body: dict) -> dict: async def _sent_to_bedrock(data: dict) -> dict: - """The Converse body Bedrock receives when the passthrough route sends ``data``.""" + """The body Bedrock receives when the passthrough route sends ``data``.""" received: list[httpx.Request] = [] + action: Final = data["endpoint"].rsplit("/", 1)[-1] + def respond(request: httpx.Request) -> httpx.Response: received.append(request) - reply = { - "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, - "stopReason": "end_turn", - "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, - } + reply: Final = ( + { + "id": "msg_bdrk_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + if action == "invoke" + else { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + } + ) return httpx.Response(200, json=reply, request=request) router = litellm.Router( @@ -1457,7 +1473,7 @@ async def _sent_to_bedrock(data: dict) -> dict: await router.allm_passthrough_route( model="bedrock-model", method="POST", - endpoint="/model/bedrock-model/converse", + endpoint=f"/model/bedrock-model/{action}", data=data["data"], client=AsyncHTTPHandler(transport=httpx.MockTransport(respond)), ) @@ -3843,6 +3859,260 @@ class TestInvokeStreamGuardrailRelay: assert rewritten_key == received_key +def _anthropic_invoke_data( + action: str = "invoke", +) -> dict[str, object]: # mutable-ok: tests write into the request the handler gets + return { + "endpoint": f"model/anthropic.claude-sonnet/{action}", + "custom_llm_provider": "bedrock", + "model": "anthropic.claude-sonnet", + "litellm_call_id": "call-1", + "data": { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 512, + "system": "You are a travel assistant. Card on file 4111 1111 1111 1111", + "tools": [ + { + "name": "find_trips", + "description": "Search trips", + "input_schema": {"type": "object", "properties": {"port": {"type": "string"}}}, + } + ], + "messages": [ + {"role": "user", "content": "Pay with 4111 1111 1111 1111"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "find_trips", "input": {"port": "Miami"}} + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "Card 4111 1111 1111 1111 charged"} + ], + }, + ], + }, + } + + +def _masked_anthropic_invoke_body() -> Mapping[str, object]: + body: Final = _anthropic_invoke_data()["data"] + return { + **body, + "system": "You are a travel assistant. Card on file [CARD] 1111 1111 1111", + "messages": [ + {"role": "user", "content": "Pay with [CARD] 1111 1111 1111"}, + body["messages"][1], + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "Card [CARD] 1111 1111 1111 charged"} + ], + }, + ], + } + + +def _rows_masked(inputs: GenericGuardrailAPIInputs) -> Mapping[str, object]: + rows = [ + {**row, "content": row["content"].replace("4111", "[CARD]")} if isinstance(row.get("content"), str) else row + for row in inputs["structured_messages"] + ] + return {"structured_messages": rows} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) +async def test_anthropic_invoke_masks_are_written_back_into_the_body(action: str) -> None: + guardrail = _ScriptedGuardrail(lambda inputs: {"texts": _masked_texts(inputs)}) + + result = await BedrockPassthroughGuardrailHandler().process_input_messages( + _anthropic_invoke_data(action), guardrail + ) + + assert result["data"] == _masked_anthropic_invoke_body() + + +@pytest.mark.asyncio +async def test_anthropic_invoke_masks_reach_bedrock() -> None: + data: Final = _anthropic_invoke_data() + + await BedrockPassthroughGuardrailHandler().process_input_messages( + data, _ScriptedGuardrail(lambda inputs: {"texts": _masked_texts(inputs)}) + ) + + assert await _sent_to_bedrock(data) == _masked_anthropic_invoke_body() + + +@pytest.mark.asyncio +async def test_anthropic_invoke_guardrail_gets_roles_tools_and_the_body_at_the_top_level_of_request_data() -> None: + data: Final = _anthropic_invoke_data() + guardrail = _replying({}) + + await BedrockPassthroughGuardrailHandler().process_input_messages(data, guardrail) + + inputs = guardrail.received[-1] + assert [row["role"] for row in inputs["structured_messages"]] == ["system", "user", "assistant", "tool"] + assert [tool["function"]["name"] for tool in inputs["tools"]] == ["find_trips"] + assert inputs["model"] == data["model"] + request_data: Final = guardrail.request_data[-1] + assert request_data["messages"] is data["data"]["messages"] + assert request_data["endpoint"] == data["endpoint"] + assert request_data["litellm_call_id"] == data["litellm_call_id"] + assert "data" not in request_data + + +@pytest.mark.asyncio +async def test_anthropic_invoke_structured_message_rewrites_are_written_back() -> None: + result = await BedrockPassthroughGuardrailHandler().process_input_messages( + _anthropic_invoke_data(), _ScriptedGuardrail(_rows_masked) + ) + + assert result["data"]["messages"][0] == { + "role": "user", + "content": [{"type": "text", "text": "Pay with [CARD] 1111 1111 1111"}], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["anthropic.claude-sonnet", "prod-model", ""]) +async def test_anthropic_invoke_structured_rewrite_keeps_cache_control_whatever_the_model_is_called( + model: str, +) -> None: + cached: Final = {"type": "ephemeral"} + data: Final = _anthropic_invoke_data() + data["model"] = model + data["data"]["system"] = [{"type": "text", "text": "Card on file 4111 1111 1111 1111", "cache_control": cached}] + data["data"]["tools"] = [ + { + "name": "find_trips", + "description": "Search trips", + "input_schema": {"type": "object", "properties": {}}, + "cache_control": cached, + } + ] + data["data"]["messages"] = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Pay with 4111 1111 1111 1111", "cache_control": {**cached, "ttl": "1h"}} + ], + } + ] + + result = await BedrockPassthroughGuardrailHandler().process_input_messages( + data, + _ScriptedGuardrail( + lambda inputs: { + "structured_messages": json.loads(json.dumps(inputs["structured_messages"]).replace("4111", "[CARD]")), + "tools": json.loads(json.dumps(inputs["tools"])), + } + ), + ) + + assert result["data"]["system"] == [ + {"type": "text", "text": "Card on file [CARD] 1111 1111 1111", "cache_control": cached} + ] + assert result["data"]["messages"][0]["content"] == [ + {"type": "text", "text": "Pay with [CARD] 1111 1111 1111", "cache_control": {**cached, "ttl": "1h"}} + ] + assert result["data"]["tools"][0]["cache_control"] == cached + + +@pytest.mark.asyncio +async def test_invoke_body_without_anthropic_version_is_scanned_whole() -> None: + data: Final = {**_anthropic_invoke_data(), "data": {"messages": [{"role": "user", "content": "hello"}]}} + guardrail = _replying({}) + + await BedrockPassthroughGuardrailHandler().process_input_messages(data, guardrail) + + assert json.loads(guardrail.received[-1]["texts"][0])["data"] == data["data"] + assert "structured_messages" not in guardrail.received[-1] + + +_PLAIN_TEXT_DOCUMENT: Final = { + "type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Card 4111 1111 1111 1111"}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "content", + [ + pytest.param([{"type": "text", "text": "Summarize this"}, _PLAIN_TEXT_DOCUMENT], id="document"), + pytest.param( + [ + { + "type": "search_result", + "source": "https://example.com/cards", + "title": "Cards", + "content": [{"type": "text", "text": "Card 4111 1111 1111 1111"}], + } + ], + id="search_result", + ), + pytest.param( + [{"type": "tool_result", "tool_use_id": "toolu_1", "content": [_PLAIN_TEXT_DOCUMENT]}], + id="document_in_tool_result", + ), + ], +) +async def test_anthropic_invoke_body_with_blocks_the_messages_handler_does_not_scan_is_scanned_whole(content: Sequence[object]) -> None: + invoke: Final = _anthropic_invoke_data() + data: Final = {**invoke, "data": {**invoke["data"], "messages": [{"role": "user", "content": content}]}} + guardrail = _replying({}) + + await BedrockPassthroughGuardrailHandler().process_input_messages(data, guardrail) + + assert json.loads(guardrail.received[-1]["texts"][0])["data"] == data["data"] + assert "structured_messages" not in guardrail.received[-1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"content": "hi"}], id="message_without_role"), + pytest.param([{"role": "user", "content": ["hi"]}], id="bare_string_in_content"), + pytest.param([{"role": "user", "content": None}], id="null_content"), + pytest.param(["hi"], id="message_is_a_string"), + ], +) +async def test_anthropic_invoke_malformed_messages_are_scanned_whole_and_left_for_bedrock_to_reject( + messages: Sequence[object], +) -> None: + invoke: Final = _anthropic_invoke_data() + data: Final = {**invoke, "data": {**invoke["data"], "messages": messages}} + guardrail = _replying({}) + + result: Final = await BedrockPassthroughGuardrailHandler().process_input_messages(data, guardrail) + + assert json.loads(guardrail.received[-1]["texts"][0])["data"] == data["data"] + assert result["data"]["messages"] == messages + + +@pytest.mark.asyncio +async def test_anthropic_invoke_response_masks_are_written_back() -> None: + response = { + "id": "msg_bdrk_1", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Charged 4111 1111 1111 1111"}], + "stop_reason": "end_turn", + } + + result: Final = await BedrockPassthroughGuardrailHandler().process_output_response( + response=response, + guardrail_to_apply=_ScriptedGuardrail(lambda inputs: {"texts": _masked_texts(inputs)}), + request_data=_anthropic_invoke_data(), + ) + + assert result == {**response, "content": [{"type": "text", "text": "Charged [CARD] 1111 1111 1111"}]} + + def test_sse_events_keeps_a_data_line_whose_json_holds_unicode_line_separators() -> None: event: Final = { "type": "content_block_delta", diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 2ac31b6887d..cf2f25faaa9 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,12 +1,16 @@ import asyncio +import copy import json import sys import types +from collections.abc import Mapping +from datetime import datetime, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import MagicMock import pytest +from pydantic import TypeAdapter import litellm from litellm.constants import ( @@ -14,10 +18,15 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload, StandardLoggingPayloadErrorInformation from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -36,6 +45,66 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) +def _closing_input() -> SessionEndSignals: + signals: Final = SessionEndSignals() + signals.input_closing.set() + return signals + + +_SPEND_ROW_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ERROR_INFORMATION_ADAPTER: Final = TypeAdapter(StandardLoggingPayloadErrorInformation | None) +_BEDROCK_INPUT_CHUNK_ADAPTER: Final = TypeAdapter(Mapping[str, Mapping[str, object]]) +_STANDARD_LOGGING_PAYLOAD_ADAPTER: Final = TypeAdapter(StandardLoggingPayload) + + +def _spend_row( + logging_obj: "FakeLogging | LiteLLMLogging", +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: + return _spend_row_of(logging_obj.model_call_details) + + +def _spend_row_of( + model_call_details: dict[str, object], +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: + now: Final = datetime.now(timezone.utc) + row: Final = get_logging_payload(kwargs=model_call_details, response_obj=None, start_time=now, end_time=now) + metadata: Final = _SPEND_ROW_METADATA_ADAPTER.validate_json(row["metadata"]) + return row["status"], _ERROR_INFORMATION_ADAPTER.validate_python(metadata.get("error_information")) + + +class LoggedEventRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: Final[list[tuple[Literal["success", "failure"], dict[str, object]]]] = [] + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("success", dict(kwargs))) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("failure", dict(kwargs))) + + def payload(self, event: Literal["success", "failure"]) -> StandardLoggingPayload: + (logged,) = (kwargs for logged_event, kwargs in self.events if logged_event == event) + return _STANDARD_LOGGING_PAYLOAD_ADAPTER.validate_python(logged["standard_logging_object"]) + + +class HeldSuccessLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.entered.set() + await self.release.wait() + + class FakePayloadPart: def __init__(self, bytes_): self.bytes_ = bytes_ @@ -85,6 +154,19 @@ class HangingCloseInputStream(FakeInputStream): await asyncio.Future() +class BodyWriterInputStream(FakeInputStream): + """Models the SDK input stream, whose request-body writer task only finishes once the stream is closed""" + + def __init__(self): + super().__init__() + self._close_requested = asyncio.Event() + self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait()) + + async def close(self): + await super().close() + self._close_requested.set() + + class FakeBedrockStream: def __init__(self, input_stream=None): self.input_stream = input_stream if input_stream is not None else FakeInputStream() @@ -124,6 +206,11 @@ class FakeLogging: self.litellm_trace_id = trace_id self.model_call_details = {} + def attempt_scoped_copy(self): + attempt = copy.copy(self) + attempt.model_call_details = dict(self.model_call_details) + return attempt + class DisconnectingClientWS: def __init__(self, messages): @@ -159,12 +246,15 @@ class EndedBedrockStream: class RealtimeClientWS: - def __init__(self): + def __init__(self, messages=()): self.closed = False self.sent_to_client = [] self.scope = {} + self._messages = list(messages) async def receive_text(self): + if self._messages: + return self._messages.pop(0) raise RuntimeError("client disconnected") async def send_text(self, message): @@ -178,8 +268,7 @@ class ConnectedClientWS(RealtimeClientWS): """Client that sends its scripted messages and then stays connected until the server closes it""" def __init__(self, messages): - super().__init__() - self._messages = list(messages) + super().__init__(messages) self._closed_event = asyncio.Event() async def receive_text(self): @@ -251,6 +340,77 @@ class ScriptedBedrockStream: return (None, self._receiver) +def _sent_event_names(input_stream: FakeInputStream) -> list[str]: + return [ + next(iter(_BEDROCK_INPUT_CHUNK_ADAPTER.validate_json(chunk.value.bytes_)["event"])) + for chunk in input_stream.sent + ] + + +class SessionEndingInputStream(FakeInputStream): + def __init__(self): + super().__init__() + self.session_ended = asyncio.Event() + + async def send(self, event): + await super().send(event) + if "sessionEnd" in json.loads(event.value.bytes_.decode("utf-8")).get("event", {}): + self.session_ended.set() + + async def close(self): + await super().close() + self.session_ended.set() + + +class SlowClosingInputStream(SessionEndingInputStream): + async def close(self): + await asyncio.Event().wait() + + +class OutputEndsAfterSessionEndReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, session_ended): + super().__init__(payloads) + self._session_ended = session_ended + + async def receive(self): + if not self._payloads: + await self._session_ended.wait() + return await super().receive() + + +class SessionBoundBedrockStream: + def __init__(self, payloads=(), input_stream_type=SessionEndingInputStream): + self.input_stream = input_stream_type() + self._receiver = OutputEndsAfterSessionEndReceiver(payloads, self.input_stream.session_ended) + + async def await_output(self): + return (None, self._receiver) + + +class ConnectionResetOnAudioInputStream(FakeInputStream): + def __init__(self, connection_reset): + super().__init__() + self._connection_reset = connection_reset + + async def send(self, event): + await asyncio.sleep(0) + if self._connection_reset.is_set() or "audioInput" in json.loads(event.value.bytes_.decode("utf-8"))["event"]: + self._connection_reset.set() + raise RuntimeError("bedrock connection reset") + self.sent.append(event) + + +class OutputEndsAfterConnectionResetReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, connection_reset): + super().__init__(payloads) + self._connection_reset = connection_reset + + async def receive(self): + if not self._payloads: + await self._connection_reset.wait() + return await super().receive() + + class FakeAWSCredentialsIdentity: def __init__(self, access_key_id, secret_access_key, session_token=None): self.access_key_id = access_key_id @@ -352,7 +512,7 @@ def stub_aws_sdk_client(monkeypatch): raise stream captured["open_stream"] = stream return stream - stream = ScriptedBedrockStream(captured.get("scripted_payloads", [])) + stream = SessionBoundBedrockStream(captured.get("scripted_payloads", [])) captured["open_stream"] = stream return stream @@ -407,7 +567,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -422,7 +584,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {} + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.sent == [] @@ -437,9 +599,13 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - with pytest.raises(RuntimeError, match="bedrock send failed"): - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + with pytest.raises(BedrockError) as failure: + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock send failed" assert stream.input_stream.closed @pytest.mark.asyncio @@ -452,7 +618,10 @@ class TestBedrockRealtimeHandler: ) await asyncio.wait_for( - handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}), 2 + handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ), + 2, ) assert not stream.input_stream.closed @@ -466,7 +635,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -495,6 +666,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + signals=_closing_input(), ) ] @@ -526,6 +698,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + signals=_closing_input(), ) ] @@ -597,6 +770,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, + signals=_closing_input(), ): pass @@ -656,7 +830,9 @@ class TestBedrockRealtimeSessionLifecycle: [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() + ) acked = [json.loads(message) for message in client_ws.sent_to_client] updated = [event for event in acked if event["type"] == "session.updated"] @@ -670,7 +846,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) assert client_ws.sent_to_client == [] @@ -693,7 +871,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() + ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] assert [event["type"] for event in acked] == ["session.updated", "error", "session.updated"] @@ -720,10 +900,10 @@ class TestBedrockRealtimeSessionLifecycle: async def test_malformed_setting_is_rejected_or_dropped_and_the_session_still_answers( self, stub_aws_sdk_client, session, drop_params, expected_errors ): - stream: Final = ScriptedBedrockStream([]) + stream: Final = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [stream] user_text: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]} - websocket: Final = ConnectedClientWS( + websocket: Final = RealtimeClientWS( [ json.dumps({"type": "session.update", "session": session}), json.dumps({"type": "conversation.item.create", "item": user_text}), @@ -761,8 +941,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_escapes_and_fallback_replays_session_update(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.SESSION_UPDATE]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.SESSION_UPDATE]) + healthy_stream = SessionBoundBedrockStream() eager_failure = ServiceUnavailableException("fault injected before the stream was returned") stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), eager_failure, healthy_stream] @@ -848,9 +1028,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] - flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] - assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"] - assert stream.input_stream.closed with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -877,8 +1054,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_on_an_input_first_start_replays_the_input(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.USER_TEXT]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.USER_TEXT]) + healthy_stream = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), healthy_stream] with pytest.raises(BedrockError) as failure: @@ -907,7 +1084,7 @@ class TestBedrockRealtimeProviderFailurePropagation: [self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})] ) - with pytest.raises(RuntimeError, match="bedrock input stream reset"): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -915,6 +1092,8 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause" assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] @@ -923,10 +1102,10 @@ class TestBedrockRealtimeProviderFailurePropagation: async def test_success_dispatch_stamps_the_ownership_marker_only_when_spend_was_logged( self, stub_aws_sdk_client, spend_dispatch ): - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream(self.TEXT_TURN)] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=spend_dispatch["logging_obj"], **self.AWS_PARAMS, ) @@ -935,15 +1114,315 @@ class TestBedrockRealtimeProviderFailurePropagation: assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True idle_logging = FakeLogging() - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([])] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream()] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=idle_logging, **self.AWS_PARAMS, ) assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in idle_logging.model_call_details + @pytest.mark.asyncio + async def test_bedrock_stream_ending_mid_session_fails_the_session_and_keeps_partial_spend( + self, stub_aws_sdk_client, spend_dispatch + ): + handler: Final = BedrockRealtime() + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) + stream: Final = ScriptedBedrockStream(self.TEXT_TURN) + stub_aws_sdk_client["streams"] = [stream] + logging_obj: Final = spend_dispatch["logging_obj"] + request_metadata: Final = {"user_api_key": "hashed-key"} + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": request_metadata} + + with pytest.raises(BedrockError) as failure: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == 503 + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + status, error_information = _spend_row(logging_obj) + assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" + assert error_information is not None + assert error_information.get("error_code") == "503" + assert error_information.get("error_message") == failure.value.message + assert request_metadata == {"user_api_key": "hashed-key"}, "the request's own metadata stays as it was sent" + + with pytest.raises(BedrockError) as replay: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert replay.value.status_code == 400, "a router retry must not restart the dropped session" + assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop" + assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith( + failure.value.message + ), "the length-capped 1011 close reason must still carry the drop" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = HangingCloseInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert failure.value.status_code == status_code + assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_leaves_no_pending_input_writer( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = BodyWriterInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + await asyncio.wait_for(stream.input_stream.body_writer, timeout=1) + await spend_dispatch["coro"] + + @pytest.mark.asyncio + async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row( + self, stub_aws_sdk_client, spend_dispatch + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + recorder: Final = LoggedEventRecorder() + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-sonic-v1:0", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop", + function_id="nova-sonic-drop", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-sonic-v1:0", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + ((logged_event, usage_kwargs),) = recorder.events + assert logged_event == "success" + usage_record: Final = recorder.payload("success") + assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" + assert [event["type"] for event in usage_record["response"]["results"]] == ["response.done"] + assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + status, error_information = _spend_row_of(usage_kwargs) + assert status == "failure" + assert error_information is not None + assert error_information.get("error_code") == "503" + + @pytest.mark.asyncio + async def test_a_dropped_session_bills_its_usage_to_loggers_while_the_failure_event_runs( + self, stub_aws_sdk_client, spend_dispatch + ): + usage_event: Final = json.dumps( + {"event": {"usageEvent": {"totalInputTokens": 1000, "totalOutputTokens": 50, "totalTokens": 1050}}} + ) + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((*self.TEXT_TURN, usage_event))] + held: Final = HeldSuccessLogger() + recorder: Final = LoggedEventRecorder() + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-2-5-sonic", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop-order", + function_id="nova-sonic-drop-order", + dynamic_async_success_callbacks=[held, recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-2-5-sonic", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-2-5-sonic", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + success_dispatch: Final = asyncio.create_task(spend_dispatch["coro"]) + await asyncio.wait_for(held.entered.wait(), timeout=5) + logging_obj.failure_handler(failure.value, "", logging_obj.start_time, datetime.now(), for_async_request=True) + await logging_obj.async_failure_handler(failure.value, "", logging_obj.start_time, datetime.now()) + held.release.set() + await success_dispatch + + usage_record: Final = recorder.payload("success") + assert recorder.payload("failure")["status"] == "failure" + assert usage_record["status"] == "success" + assert usage_record["total_tokens"] == 1050 + assert usage_record["response_cost"] > 0 + ((_, usage_kwargs),) = (logged for logged in recorder.events if logged[0] == "success") + status, _ = _spend_row_of(usage_kwargs) + assert status == "failure" + + @pytest.mark.asyncio + async def test_a_dropped_session_without_request_metadata_still_fails_and_bills_its_turn( + self, stub_aws_sdk_client, spend_dispatch + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": None, "metadata": {}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_session_bedrock_fails_before_any_logged_event_dispatches_no_success( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((), receiver_type=receiver_type)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + assert "coro" not in spend_dispatch, "a failed session with no usage gets no success event, as on base" + assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in logging_obj.model_call_details + + @pytest.mark.asyncio + async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( + self, stub_aws_sdk_client, spend_dispatch + ): + stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + spend_dispatch["logging_obj"].model_call_details["litellm_params"] = { + "litellm_metadata": {"user_api_key": "hashed-key"} + } + + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert _sent_event_names(stream.input_stream)[-1] == "sessionEnd" + assert not stream.input_stream.closed, "the input close must still be pending when the output ends" + assert websocket.closed + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None) + + @pytest.mark.asyncio + async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( + self, stub_aws_sdk_client, spend_dispatch + ): + connection_reset: Final = asyncio.Event() + stream: Final = ScriptedBedrockStream( + self.TEXT_TURN, + receiver_type=lambda payloads: OutputEndsAfterConnectionResetReceiver(payloads, connection_reset), + ) + stream.input_stream = ConnectionResetOnAudioInputStream(connection_reset) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO]) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert (failure.value.status_code, failure.value.message) == ( + 503, + "Bedrock realtime stream ended before the session closed", + ), "the app sees one cause whichever side of the stream reports the reset first" + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + @pytest.mark.asyncio async def test_stream_failure_after_client_disconnect_is_not_a_provider_failure(self, stub_aws_sdk_client): stream = ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver) @@ -982,7 +1461,13 @@ class TestBedrockRealtimeProviderFailurePropagation: with pytest.raises(ServiceUnavailableException): await handler._forward_client_to_bedrock( - client_ws, stream, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", {}, FakeLogging() + client_ws, + stream, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + {}, + FakeLogging(), + signals=SessionEndSignals(), ) assert client_ws.sent_to_client == [] @@ -1128,7 +1613,6 @@ class TestBedrockRealtimeSdkLifecycle: ) assert stub_aws_sdk_client["client_closed"] - assert stub_aws_sdk_client["input_closed_before_client_close"] @pytest.mark.asyncio async def test_client_without_close_completes_session(self, monkeypatch): diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 38c69e0ce27..80715105b7f 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,5 +1,7 @@ import asyncio, importlib, litellm, pytest +import respx +from httpx import Response from typing import Final @@ -581,6 +583,188 @@ def test_deployment_model_info_breakpoint_flag_covers_an_unmapped_arn(local_mode assert bedrock_model_accepts_cache_points(unflagged_arn) is True + +_CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6}, +} +_CACHE_MARK: Final = {"type": "ephemeral"} +_MARKED_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, + "cache_control": _CACHE_MARK, +} +_COMPUTER_TOOL: Final = { + "type": "computer_20250124", + "name": "computer", + "display_width_px": 1024, + "display_height_px": 768, +} +_MARKED_TOOL_CONVERSATION: Final = ( + {"role": "system", "content": "stable"}, + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris and Rome?", "cache_control": _CACHE_MARK}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Checking both.", "cache_control": _CACHE_MARK}], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}{"city": "Rome"}'}, + "cache_control": _CACHE_MARK, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}]}, + {"role": "tool", "tool_call_id": "call_1_1", "content": "Cloudy"}, + {"role": "assistant", "content": "Paris is sunny, Rome is cloudy.", "cache_control": _CACHE_MARK}, + {"role": "user", "content": "Thanks"}, +) + + +@pytest.mark.parametrize( + ("model", "flagged", "declared", "tools"), + [ + pytest.param("moonshotai.kimi-k3", "opted-in", True, (_MARKED_WEATHER_TOOL,), id="kimi-opted-in"), + pytest.param( + "us.anthropic.claude-sonnet-5-5", + "opted-out", + False, + (_COMPUTER_TOOL, _MARKED_WEATHER_TOOL), + id="claude-opted-out-next-to-computer-use", + ), + ], +) +@pytest.mark.parametrize("entrypoint", ["completion", "acompletion"]) +@pytest.mark.parametrize("flagless_registers_first", [False, True]) +async def test_a_bedrock_deployment_sends_cache_points_by_its_own_flag_not_a_siblings_on_the_same_model( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, + model: str, + flagged: str, + declared: bool, + tools: tuple[dict[str, object], ...], + entrypoint: str, + flagless_registers_first: bool, +) -> None: + from litellm import Router + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + converse: Final = respx_mock.post(f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/converse").mock( + return_value=Response(200, json=_CONVERSE_REPLY) + ) + flags: Final = {flagged: {"supports_prompt_cache_breakpoint": declared}} + registration_order: Final = ("flagless", flagged) if flagless_registers_first else (flagged, "flagless") + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": f"bedrock/{model}", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{name}-id", **flags.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + messages: Final = [dict(message) for message in _MARKED_TOOL_CONVERSATION] + + async def sends_a_cache_point(name: str) -> bool: + match entrypoint: + case "completion": + router.completion(model=name, messages=messages, tools=list(tools)) + case "acompletion": + await router.acompletion(model=name, messages=messages, tools=list(tools)) + return b"cachePoint" in converse.calls.last.request.content + + assert {name: await sends_a_cache_point(name) for name in registration_order} == { + flagged: declared, + "flagless": not declared, + } + +async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_model_opted_its_model_in( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + from litellm import Router + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + converse: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/moonshotai.kimi-k3/converse" + ).mock(return_value=Response(200, json=_CONVERSE_REPLY)) + litellm.register_model({"bedrock/moonshotai.kimi-k3": {"supports_prompt_cache_breakpoint": True}}) + router: Final = Router( + model_list=[ + { + "model_name": "flagless", + "litellm_params": { + "model": "bedrock/moonshotai.kimi-k3", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "flagless-id"}, + } + ], + num_retries=0, + ) + + await router.acompletion(model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]) + + assert b"cachePoint" in converse.calls.last.request.content + + +@pytest.mark.parametrize( + ("catalog_caching", "deployment_caching", "expected"), + [(None, True, True), (True, False, False), (True, None, True), (None, None, False)], +) +def test_a_routed_deployments_own_prompt_caching_flag_decides_cache_points_for_a_catalogued_model( + catalog_caching: bool | None, deployment_caching: bool | None, expected: bool, local_model_cost_map: None +) -> None: + from litellm.llms.bedrock.common_utils import bedrock_model_accepts_cache_points + + model: Final = f"qa-catalogued-model-{catalog_caching}-{deployment_caching}" + catalog_flag: Final = {} if catalog_caching is None else {"supports_prompt_caching": catalog_caching} + deployment_flag: Final = {} if deployment_caching is None else {"supports_prompt_caching": deployment_caching} + litellm.register_model({model: {"litellm_provider": "bedrock", "mode": "chat", **catalog_flag}}) + + assert bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", **deployment_flag}) is expected + + +@pytest.mark.parametrize( + ("declared", "catalog_caching", "expected"), + [ + ("true", False, True), + (1, False, True), + (" TRUE ", False, True), + ("false", True, False), + (0, True, False), + ("maybe", True, True), + ("maybe", False, False), + ], +) +def test_a_deployments_prompt_caching_flag_written_as_a_string_or_number_decides_cache_points_as_the_bool_it_spells( + declared: object, catalog_caching: bool, expected: bool, local_model_cost_map: None +) -> None: + from litellm.llms.bedrock.common_utils import bedrock_model_accepts_cache_points + + model: Final = f"qa-catalogued-model-lax-{catalog_caching}" + litellm.register_model( + {model: {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_caching": catalog_caching}} + ) + + assert ( + bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared}) + is expected + ) + def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): from litellm.llms.bedrock.common_utils import merge_bedrock_aws_request_params diff --git a/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py b/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py new file mode 100644 index 00000000000..f5cb874c6ef --- /dev/null +++ b/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py @@ -0,0 +1,35 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +BEDROCK_COHERE_INVOKE_URL: Final = ( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-english-v3/invoke" +) +COHERE_EMBED_RESPONSE: Final = {"embeddings": [[0.1, 0.2]], "meta": {"billed_units": {"input_tokens": 1}}} + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_cohere_embedding_sends_the_caller_timeout( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(BEDROCK_COHERE_INVOKE_URL).mock( + return_value=httpx.Response(200, json=COHERE_EMBED_RESPONSE) + ) + request: Final = { + "model": "bedrock/cohere.embed-english-v3", + "input": ["hello"], + "timeout": 45, + "aws_region_name": "us-east-1", + "aws_access_key_id": "mockaws_access_key_id", + "aws_secret_access_key": "mockaws_secret_access_key", + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.calls.last.request.extensions["timeout"]["read"] == 45 + assert response.data[0]["embedding"] == [0.1, 0.2] diff --git a/tests/unit/llms/cohere/embed/test_v1_transformation.py b/tests/unit/llms/cohere/embed/test_v1_transformation.py index 31b4450ed88..63e61eaecd1 100644 --- a/tests/unit/llms/cohere/embed/test_v1_transformation.py +++ b/tests/unit/llms/cohere/embed/test_v1_transformation.py @@ -1,5 +1,7 @@ from unittest.mock import MagicMock +import httpx +import pytest from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig from litellm.types.utils import EmbeddingResponse @@ -183,6 +185,7 @@ class TestCohereEmbeddingV1Transform: "meta": {}, # No billed_units } mock_response.json = MagicMock(return_value=response_json) + mock_response.headers = httpx.Headers() input_data = ["test text"] data = {"texts": input_data, "input_type": "search_query"} @@ -208,3 +211,31 @@ class TestCohereEmbeddingV1Transform: # Verify encoding was called self.encoding.encode.assert_called() + + @pytest.mark.parametrize( + ("body_key", "item", "expected_tokens"), + [("images", "data:image/png;base64," + "iVBORw0KGgo" * 400, 2047), ("texts", "a short sentence", 5)], + ) + def test_transform_response_counts_bedrock_billed_tokens_only_for_images( + self, body_key: str, item: str, expected_tokens: int + ): + response = httpx.Response( + 200, + json={"id": "e1", body_key: [{}], "embeddings": [[0.1, 0.2]], "response_type": "embeddings_floats"}, + headers={"x-amzn-bedrock-input-token-count": "2047"}, + ) + + result = self.config.transform_response( + response=response, + api_key=None, + logging_obj=self.logging_obj, + data={body_key: [item], "input_type": "image" if body_key == "images" else "search_document"}, + model_response=EmbeddingResponse(), + model="cohere.embed-v4:0", + encoding=self.encoding, + input=[item], + ) + + assert result.usage is not None + assert result.usage.prompt_tokens == expected_tokens + assert result.usage.total_tokens == expected_tokens diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 01760572f33..dd0e5762845 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5,7 +5,7 @@ import json import logging import threading import time -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, Mock, patch @@ -68,6 +68,7 @@ from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ) +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( @@ -3614,20 +3615,40 @@ class TestServerFulfilledToolsInRequest: == frozenset() ) - def test_interception_callbacks_declare_their_retrieval_tools(self): + @pytest.mark.parametrize("hook", ["compression", "headroom", "compresr"]) + def test_a_retrieval_hook_holds_back_a_stream_carrying_its_retrieve_tool(self, hook): from litellm.integrations.compression_interception.handler import ( LITELLM_CONTENT_RETRIEVE_TOOL_NAME, CompressionInterceptionLogger, ) + from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( + COMPRESR_RETRIEVE_TOOL_NAME, + CompresrGuardrail, + ) from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( HEADROOM_RETRIEVE_TOOL_NAME, HeadroomGuardrail, ) - assert HeadroomGuardrail.server_fulfilled_tool_names == frozenset({HEADROOM_RETRIEVE_TOOL_NAME}) - assert CompressionInterceptionLogger.server_fulfilled_tool_names == frozenset( - {LITELLM_CONTENT_RETRIEVE_TOOL_NAME} - ) + callback, retrieve_tool_name = { + "compression": (CompressionInterceptionLogger(), LITELLM_CONTENT_RETRIEVE_TOOL_NAME), + "headroom": ( + HeadroomGuardrail(api_base="https://headroom.test", api_key="hr-test", guardrail_name="headroom"), + HEADROOM_RETRIEVE_TOOL_NAME, + ), + "compresr": ( + CompresrGuardrail(api_base="https://compresr.test", api_key="cr-test", guardrail_name="compresr"), + COMPRESR_RETRIEVE_TOOL_NAME, + ), + }[hook] + tools = [ + {"name": "Bash", "input_schema": {"type": "object"}}, + {"name": retrieve_tool_name, "input_schema": {"type": "object"}}, + ] + + assert BaseLLMHTTPHandler._server_fulfilled_tools_in_request( + logging_obj=self._logging_obj_with([callback]), tools=tools + ) == frozenset({retrieve_tool_name}) def _make_stub_direct_vector_store_config(response): @@ -5341,6 +5362,165 @@ async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_s assert followup_calls[0]["prompt_cache_key"] == "from-plan-params" +class _PlannedChatLoop(CustomLogger): + def __init__(self, plan: AgenticLoopPlan, claimed_call_name: str = "litellm_web_search") -> None: + super().__init__() + self._plan = plan + self._claimed_call_name = claimed_call_name + self.logging_objs: list[object] = [] + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + claimed: Final = [ + {"id": call.id, "name": call.function.name} + for call in _tool_calls_of(response) + if call.function.name == self._claimed_call_name + ] + return True, {"tool_calls": claimed or [{"id": "call_search", "name": self._claimed_call_name}]} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + self.logging_objs.append(logging_obj) + return self._plan + + +class _RunOnlyChatLoop(CustomLogger): + def __init__(self, result: ModelResponse) -> None: + super().__init__() + self._result = result + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} + + async def async_run_chat_completion_agentic_loop( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> object: + return self._result + + +def _tool_calls_of(response: object) -> list: + if not isinstance(response, ModelResponse) or not response.choices: + return [] + return response.choices[0].message.tool_calls or [] + + +async def _run_chat_completion_agentic_hooks( + response: ModelResponse, *callbacks: CustomLogger, kwargs: Mapping[str, object] = MappingProxyType({}) +) -> object: + logging_obj: Final = Mock(litellm_call_id="call-1", dynamic_success_callbacks=list(callbacks), model_call_details={}) + return await BaseLLMHTTPHandler()._call_agentic_chat_completion_hooks( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is new in litellm?"}], + optional_params={}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="openai", + kwargs=dict(kwargs), + ) + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_surface_a_failed_follow_up_instead_of_the_tool_call_turn(): + rate_limited_follow_up: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "here are the search results"}], + kwargs={"mock_response": "litellm.RateLimitError"}, + ), + ) + + with pytest.raises(litellm.RateLimitError): + await _run_chat_completion_agentic_hooks(ModelResponse(), _PlannedChatLoop(rate_limited_follow_up)) + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_return_the_tool_call_turn_when_the_plan_terminates(): + response: Final = ModelResponse() + + assert await _run_chat_completion_agentic_hooks(response, _PlannedChatLoop(AgenticLoopPlan(terminate=True))) is response + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_move_past_a_plan_that_declines_the_loop(): + override: Final = ModelResponse() + declined: Final = _PlannedChatLoop(AgenticLoopPlan(run_agentic_loop=False)) + overriding: Final = _PlannedChatLoop(AgenticLoopPlan(response_override=override)) + + assert await _run_chat_completion_agentic_hooks(ModelResponse(), declined, overriding) is override + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_return_a_run_only_loops_result(): + loop_result: Final = ModelResponse() + + assert await _run_chat_completion_agentic_hooks(ModelResponse(), _RunOnlyChatLoop(loop_result)) is loop_result + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_at_the_cap_drop_every_hooks_internal_call_and_keep_the_clients(): + response: Final = ModelResponse( + choices=[ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}} + for name in ("litellm_web_search", "litellm_code_execution", "lookup") + ], + }, + } + ] + ) + searching: Final = _PlannedChatLoop(AgenticLoopPlan(run_agentic_loop=True)) + running_code: Final = _PlannedChatLoop( + AgenticLoopPlan(run_agentic_loop=True), claimed_call_name="litellm_code_execution" + ) + + result: Final = await _run_chat_completion_agentic_hooks( + response, searching, running_code, kwargs={"_agentic_loop_depth": 1, "max_agentic_loops": 1} + ) + + assert [call.id for call in _tool_calls_of(result)] == ["call_lookup"] + + @pytest.mark.asyncio async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword""" @@ -5364,17 +5544,17 @@ async def test_chat_completion_agentic_followup_does_not_repeat_request_params_f ), ) - response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( - plan=plan, + response: Final = await BaseLLMHTTPHandler()._call_agentic_chat_completion_hooks( + response=ModelResponse(), model="gpt-5", messages=[{"role": "user", "content": "x"}], optional_params={"temperature": 0.2}, - kwargs=dict(request_kwargs), + logging_obj=Mock( + litellm_call_id="call-1", dynamic_success_callbacks=[_PlannedChatLoop(plan)], model_call_details={} + ), + stream=False, custom_llm_provider="openai", - depth=0, - max_loops=3, - fingerprints=[], - fingerprint="fp", + kwargs=dict(request_kwargs), ) assert response == "followup-response" @@ -5384,6 +5564,138 @@ async def test_chat_completion_agentic_followup_does_not_repeat_request_params_f assert followup_calls[0]["model"] == "openai/gpt-5" +def _a_search_tool_call_turn(call_number: int) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": f"chatcmpl-{call_number}", + "object": "chat.completion", + "created": 0, + "model": "served-model", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": f"call-{call_number}", + "type": "function", + "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }, + ) + + +class _SearchAgainLoop(CustomLogger): + def __init__(self, client: AsyncHTTPHandler) -> None: + super().__init__() + self._client = client + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + searches: Final = [ + {"id": call.id, "name": call.function.name} + for call in response.choices[0].message.tool_calls or [] + if call.function.name == "litellm_web_search" + ] + return bool(searches), {"tool_calls": searches} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[*messages, {"role": "user", "content": "here are the search results"}], + kwargs={"client": self._client}, + ), + ) + + +@pytest.mark.asyncio +async def test_hosted_vllm_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(monkeypatch): + loop: Final = _PlannedChatLoop( + AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "here are the search results"}], + kwargs={"mock_response": "litellm.RateLimitError"}, + ), + ) + ) + monkeypatch.setattr(litellm, "callbacks", [loop]) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="hosted_vllm/served-model", + api_base="https://fake-vllm.test/v1", + api_key="transport-only", + client=AsyncHTTPHandler(transport=httpx.MockTransport(lambda request: _a_search_tool_call_turn(1))), + messages=[{"role": "user", "content": "search the web for the weather"}], + input_cost_per_token=0.001, + output_cost_per_token=0.002, + num_retries=0, + max_retries=0, + ) + + (logging_obj,) = loop.logging_objs + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx(5 * 0.001 + 3 * 0.002) + + +@pytest.mark.asyncio +async def test_hosted_vllm_ends_the_turn_at_max_agentic_loops_without_the_internal_search_call(monkeypatch): + upstream_calls: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def always_search(request: httpx.Request) -> httpx.Response: + upstream_calls.put_nowait(request) + return _a_search_tool_call_turn(upstream_calls.qsize()) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(always_search)) + monkeypatch.setattr(litellm, "callbacks", [_SearchAgainLoop(client)]) + + response: Final = await asyncio.wait_for( + litellm.acompletion( + model="hosted_vllm/served-model", + api_base="https://fake-vllm.test/v1", + api_key="transport-only", + client=client, + messages=[{"role": "user", "content": "search the web for the weather"}], + max_agentic_loops=1, + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + assert upstream_calls.qsize() == 2 + assert not response.choices[0].message.tool_calls + + _UPSTREAM_HEADERS: Final = {"x-request-id": "req_upstream", "x-ratelimit-remaining-requests": "41"} @@ -6016,63 +6328,6 @@ async def _search_through(transport: httpx.MockTransport, is_async: bool) -> obj return await litellm.vector_stores.asearch(**search_kwargs, client=client) -class TestAgenticFollowUpKeepsTheProviderPrefix: - @staticmethod - async def _run_followup( - model: str, custom_llm_provider: str, patched_model: str | None = None - ) -> ModelResponse: - from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler - from litellm.types.integrations.custom_logger import ( - AgenticLoopPlan, - AgenticLoopRequestPatch, - ) - - response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( - plan=AgenticLoopPlan( - run_agentic_loop=True, - request_patch=AgenticLoopRequestPatch( - messages=[{"role": "user", "content": "hi"}], model=patched_model - ), - ), - model=model, - messages=[{"role": "user", "content": "hi"}], - optional_params={"mock_response": "ok from followup"}, - kwargs={}, - custom_llm_provider=custom_llm_provider, - depth=0, - max_loops=2, - fingerprints=[], - fingerprint="fp", - ) - assert isinstance(response, ModelResponse) - return response - - @pytest.mark.asyncio - async def test_a_sub_path_model_still_resolves_a_provider(self) -> None: - response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock") - - assert response.choices[0].message.content == "ok from followup" - - @pytest.mark.asyncio - async def test_an_ordinary_model_still_resolves_a_provider(self) -> None: - response: Final = await self._run_followup("gpt-4o-mini", "openai") - - assert response.choices[0].message.content == "ok from followup" - - @pytest.mark.asyncio - async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None: - response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5") - - assert response.model == "claude-sonnet-4-5" - assert response._hidden_params["custom_llm_provider"] == "anthropic" - - @pytest.mark.asyncio - async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None: - response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") - - assert response._hidden_params["custom_llm_provider"] == "openrouter" - - @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index d2b78235c29..e04dd35f851 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -19,6 +19,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.databricks.common_utils import DatabricksException @@ -2591,6 +2592,50 @@ def test_databricks_claude_json_schema_preserves_refs_in_tool_parameters() -> No ] +@pytest.mark.parametrize( + ("model", "thinking_params", "forced"), + ( + ("databricks-claude-opus-5", {}, False), + ("databricks-claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False), + ("databricks-claude-opus-5-5", {}, False), + ("databricks-claude-opus-4-8", {}, True), + ("databricks-claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ), + ids=( + "opus-5-default-thinking", + "opus-4-8-adaptive", + "opus-5-5-without-forced-tools", + "opus-4-8-default-off", + "opus-5-thinking-disabled", + ), +) +def test_databricks_claude_json_tool_is_forced_only_when_the_model_will_not_think( + local_model_cost_map: None, model: str, thinking_params: dict[str, object], forced: bool +) -> None: + optional_params: Final = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="databricks", + response_format=copy.deepcopy(JSON_SCHEMA_RESPONSE_FORMAT), + **thinking_params, + ) + request: Final[dict[str, object]] = TypeAdapter(dict[str, object]).validate_python( + DatabricksConfig().transform_request( # pyright: ignore[reportUnknownMemberType] # inherited hook is untyped + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=dict(optional_params), + litellm_params={}, + headers={}, + ) + ) + + assert ( + request.get("tool_choice") == {"type": "function", "function": {"name": "json_tool_call"}} + ) is forced + assert optional_params["tools"][0]["function"].get("description") == ( + None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION + ) + + def test_databricks_pydantic_json_schema_preserves_nested_refs() -> None: class Person(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/tests/unit/llms/empiriolabs/__init__.py b/tests/unit/llms/empiriolabs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/empiriolabs/decisions/__init__.py b/tests/unit/llms/empiriolabs/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/empiriolabs/decisions/test_empiriolabs_decisions_transformation.py b/tests/unit/llms/empiriolabs/decisions/test_empiriolabs_decisions_transformation.py new file mode 100644 index 00000000000..02e9eab00ae --- /dev/null +++ b/tests/unit/llms/empiriolabs/decisions/test_empiriolabs_decisions_transformation.py @@ -0,0 +1,40 @@ +from collections.abc import Mapping + +import pytest + +from litellm.llms.empiriolabs.decisions.transformation import EmpirioLabsDecisionsConfig + + +@pytest.mark.parametrize( + ("environment", "connection"), + ( + ({}, ("https://api.empiriolabs.ai", None)), + ( + {"EMPIRIOLABS_API_BASE": "https://gateway.example/v1", "EMPIRIOLABS_API_KEY": "environment-key"}, + ("https://gateway.example/v1", "environment-key"), + ), + ), + ids=("public_api_without_a_key", "environment"), +) +def test_connection_comes_from_the_environment_before_the_public_api( + environment: Mapping[str, str], + connection: tuple[str, str | None], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + monkeypatch.delenv("EMPIRIOLABS_API_KEY", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + assert EmpirioLabsDecisionsConfig().resolve_connection(None, None) == connection + + +@pytest.mark.parametrize( + "api_base", + ("https://api.empiriolabs.ai", "https://api.empiriolabs.ai/v1", "https://api.empiriolabs.ai/v1/"), +) +def test_requests_go_to_v1_decisions_whether_or_not_the_base_ends_in_v1(api_base: str) -> None: + assert ( + EmpirioLabsDecisionsConfig().get_complete_url(api_base=api_base, model="aplomb-1") + == "https://api.empiriolabs.ai/v1/decisions" + ) diff --git a/tests/unit/llms/gemini/test_gemini_client_setup.py b/tests/unit/llms/gemini/test_gemini_client_setup.py index 48b010aca48..bc3a9aec365 100644 --- a/tests/unit/llms/gemini/test_gemini_client_setup.py +++ b/tests/unit/llms/gemini/test_gemini_client_setup.py @@ -12,7 +12,7 @@ def mock_gemini_api_key(monkeypatch): def test_gemini_completion(): response = completion( - model="gemini/gemini-2.0-flash-exp-image-generation", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Test message"}], mock_response="Test Message", ) diff --git a/tests/unit/llms/github_copilot/test_github_host.py b/tests/unit/llms/github_copilot/test_github_host.py index df2efe4669b..daf5bfaab80 100644 --- a/tests/unit/llms/github_copilot/test_github_host.py +++ b/tests/unit/llms/github_copilot/test_github_host.py @@ -54,6 +54,7 @@ def test_ghe_endpoints_ignore_github_com_url_overrides(monkeypatch: pytest.Monke assert endpoints.device_code_url == "https://octo.ghe.com/login/device/code" assert endpoints.access_token_url == "https://octo.ghe.com/login/oauth/access_token" assert endpoints.api_key_url == "https://api.octo.ghe.com/copilot_internal/v2/token" + assert endpoints.copilot_user_url == "https://api.octo.ghe.com/copilot_internal/user" assert endpoints.user_url == "https://api.octo.ghe.com/user" assert endpoints.default_copilot_api_base == "https://copilot-api.octo.ghe.com" assert endpoints.client_id == "custom-client" diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index cf6fb9fa71f..5274f0c1cb8 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -307,6 +307,71 @@ async def test_aexchange_401_raises_and_evicts(): ) +ACCOUNT_URL = "https://api.github.com/copilot_internal/user" +_CUSTOM_CLIENT_ID: Final = "Ov23liCustomOrgApp" +_ACCOUNT_API_BASE: Final = "https://api.business.githubcopilot.com" + + +def _account_payload(access_type_sku="copilot_enterprise_seat", chat_enabled=True, api_base=_ACCOUNT_API_BASE): + return {"access_type_sku": access_type_sku, "chat_enabled": chat_enabled, "endpoints": {"api": api_base}} + + +def _account_responder(payload=None, status=200, model_payload=None): + def respond(request: httpx.Request) -> httpx.Response: + if str(request.url) == ACCOUNT_URL: + return httpx.Response(status, json=payload or _account_payload(), request=request) + if model_payload is not None and request.url.host.endswith("githubcopilot.com"): + return httpx.Response(200, json=model_payload, request=request) + pytest.fail(f"unexpected request to {request.url}") + + return respond + + +@pytest.fixture +def custom_client_id(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITHUB_COPILOT_CLIENT_ID", _CUSTOM_CLIENT_ID) + + +@pytest.mark.asyncio +async def test_custom_client_id_uses_the_github_token_without_the_copilot_token_swap(custom_client_id): + endpoints: Final = github_host_endpoints("github.com") + with github_http(_account_responder()) as requests: + first = await aexchange_github_token("user-a", "gho_org_app", "copilot-cred", endpoints) + second = await aexchange_github_token("user-a", "gho_org_app", "copilot-cred", endpoints) + + assert [str(r.url) for r in requests] == [ACCOUNT_URL] + assert requests[0].headers["Authorization"] == "token gho_org_app" + assert first.token == second.token == "gho_org_app" + assert first.api_base == _ACCOUNT_API_BASE + + +@pytest.mark.parametrize( + "payload", + [ + _account_payload(access_type_sku="no_access", chat_enabled=False), + _account_payload(chat_enabled=False), + _account_payload(access_type_sku="no_access"), + ], +) +def test_custom_client_id_without_a_copilot_seat_raises_reconnect(custom_client_id, payload): + with github_http(_account_responder(payload)): + with pytest.raises(CallerCredentialAuthenticationError): + exchange_github_token("user-a", "gho_org_app", "copilot-cred", github_host_endpoints("github.com")) + + +@pytest.mark.parametrize("status", [401, 403, 404]) +def test_custom_client_id_rejected_account_lookup_raises_reconnect(custom_client_id, status): + with github_http(_account_responder(status=status)): + with pytest.raises(CallerCredentialAuthenticationError): + exchange_github_token("user-a", "gho_org_app", "copilot-cred", github_host_endpoints("github.com")) + + +def test_custom_client_id_account_api_base_must_stay_on_copilot_hosts(custom_client_id): + with github_http(_account_responder(_account_payload(api_base="https://evil.example"))): + session = exchange_github_token("user-a", "gho_org_app", "copilot-cred", github_host_endpoints("github.com")) + assert session.api_base == "https://api.githubcopilot.com" + + def _kwargs_with_connection( token: str = "gho_user_token", github_host: str | None = "github.com", @@ -740,6 +805,25 @@ def test_e2e_embedding_per_user_caller_authorization_cannot_override_session(per _assert_session_token_on_the_wire(requests) +def test_e2e_custom_client_id_sends_the_github_token_to_the_accounts_copilot_host( + per_user_credential, custom_client_id +): + with ( + github_http(_account_responder(model_payload=_embedding_response_payload())) as requests, + _no_authenticator(), + ): + litellm.embedding( + model="github_copilot/text-embedding-3-small", + input=["hi"], + litellm_credential_name="copilot-cred", + secret_fields=_secret_fields("user-a", "gho_org_app"), + ) + assert EXCHANGE_URL not in [str(r.url) for r in requests] + upstream = [r for r in requests if r.url.host.endswith("githubcopilot.com")] + assert upstream[0].url.host == httpx.URL(_ACCOUNT_API_BASE).host + assert upstream[0].headers["authorization"] == "Bearer gho_org_app" + + def test_e2e_responses_per_user_caller_authorization_cannot_override_session(per_user_credential): with ( github_http(_copilot_surface_responder(_responses_payload())) as requests, diff --git a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py index ea1e04291ca..7040fbe5fcc 100644 --- a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -1,12 +1,12 @@ import json from typing import Final -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch - -import litellm +import httpx import pytest import respx +import litellm MOCK_EMBEDDING_RESPONSE = [[0.1, 0.2, 0.3, 0.4, 0.5]] @@ -52,12 +52,10 @@ def mock_embedding_async_http_handler(): class TestHuggingFaceEmbedding: @pytest.fixture(autouse=True) def setup(self, mock_embedding_http_handler, mock_embedding_async_http_handler, mock_hf_config_fetch): - self.mock_get_task_patcher = patch( - "litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model" - ) + self.mock_get_task_patcher = patch("litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model") self.mock_get_task = self.mock_get_task_patcher.start() - def mock_get_task_side_effect(model, task_type, api_base): + def mock_get_task_side_effect(model, task_type, api_base, timeout=None): if task_type is not None: return task_type return "sentence-similarity" @@ -180,7 +178,9 @@ async def test_hf_embedding_sentence_sim_sends_one_similarity_request( monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setenv("HUGGINGFACE_API_KEY", "hf-test-token") with respx.mock(assert_all_called=False) as hf: - hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond(json={"pipeline_tag": "sentence-similarity"}) + hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond( + json={"pipeline_tag": "sentence-similarity"} + ) hf.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512}) inference: Final = hf.post( "https://router.huggingface.co/hf-inference/pipeline/sentence-similarity/sentence-transformers/TaylorAI/bge-micro-v2" @@ -197,3 +197,33 @@ async def test_hf_embedding_sentence_sim_sends_one_similarity_request( } assert inference.calls.last.request.headers["Authorization"] == "Bearer hf-test-token" assert response.data == [{"object": "embedding", "index": 0, "embedding": 0.7708950042724609}] + + +HF_MODEL_LOOKUP_URL: Final = "https://huggingface.co/api/models/BAAI/bge-m3" +HF_EMBED_URL: Final = "https://router.huggingface.co/hf-inference/pipeline/feature-extraction/BAAI/bge-m3" + + +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_huggingface_embedding_sends_the_caller_timeout( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512}) + lookup: Final = respx_mock.get(HF_MODEL_LOOKUP_URL).mock( + return_value=httpx.Response(200, json={"pipeline_tag": "feature-extraction"}) + ) + route: Final = respx_mock.post(HF_EMBED_URL).mock(return_value=httpx.Response(200, json=MOCK_EMBEDDING_RESPONSE)) + request: Final = { + "model": "huggingface/BAAI/bge-m3", + "input": ["hello"], + "api_key": "hf_test", + "timeout": 45, + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert all(call.request.extensions["timeout"]["read"] == 45 for call in lookup.calls) + assert lookup.called + assert route.calls.last.request.extensions["timeout"]["read"] == 45 + assert response.data[0]["embedding"] == MOCK_EMBEDDING_RESPONSE[0] diff --git a/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py b/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py index dbb5cde6060..40ee4d61ac4 100644 --- a/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py +++ b/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py @@ -1,780 +1,641 @@ -""" -End-to-end-ish tests for NvidiaRivaAudioTranscription. - -We mock ``riva.client`` so the test does not need the real gRPC SDK or a -running Riva server. The mock also lets us assert how Auth metadata is -constructed (NVCF vs self-hosted) and how the streaming generator output -is aggregated. -""" - import asyncio import io -from types import SimpleNamespace -from unittest.mock import MagicMock +import ipaddress +import threading +import uuid +import wave +from array import array +from collections.abc import Iterable, Iterator +from concurrent import futures +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timezone +from queue import SimpleQueue +from typing import Final +import grpc import httpx -import numpy as np import pytest -import soundfile as sf +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID +from riva.client.proto import riva_asr_pb2, riva_asr_pb2_grpc, riva_audio_pb2 import litellm -from litellm.llms.nvidia_riva.audio_transcription import handler as handler_mod -from litellm.llms.nvidia_riva.audio_transcription.handler import ( - NvidiaRivaAudioTranscription, -) +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.nvidia_riva.audio_transcription.handler import NvidiaRivaAudioTranscription from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException from litellm.types.utils import TranscriptionResponse +_SAMPLE_RATE_HZ: Final = 16000 +_SERVER_WAIT_SECONDS: Final = 30 +_STALL_LIMIT_SECONDS: Final = 60 +_NO_DEADLINE_SECONDS: Final = 10**9 +_STALL_TIMEOUT_SECONDS: Final = 0.5 +_API_KEY: Final = "nvapi-test" -def _make_wav_bytes(seconds: float = 1.0, sample_rate: int = 16000) -> bytes: - n = int(sample_rate * seconds) - samples = (0.05 * np.sin(np.linspace(0, 2 * np.pi * 220 * seconds, n))).astype( - np.float32 + +@dataclass(frozen=True, slots=True) +class _Script: + streaming: tuple[riva_asr_pb2.StreamingRecognizeResponse, ...] = () + offline: riva_asr_pb2.RecognizeResponse = field(default_factory=riva_asr_pb2.RecognizeResponse) + abort: tuple[grpc.StatusCode, str] | None = None + stall: bool = False + + +@dataclass(frozen=True, slots=True) +class _Call: + rpc: str + metadata: tuple[tuple[str, str], ...] + config: riva_asr_pb2.RecognitionConfig + audio: bytes + deadline_seconds: float | None + cancelled: bool + + def header(self, key: str) -> tuple[str, ...]: + return tuple(value for name, value in self.metadata if name == key) + + +@dataclass(frozen=True, slots=True) +class _Riva: + address: str + calls: SimpleQueue[_Call] + + def only_call(self) -> _Call: + call: Final = self.calls.get(timeout=_SERVER_WAIT_SECONDS) + assert self.calls.empty(), "Riva received more than one call" + return call + + +class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer): + def __init__(self, script: _Script, calls: SimpleQueue[_Call]) -> None: + self._script: Final = script + self._calls: Final = calls + + def _settle( + self, context: grpc.ServicerContext, rpc: str, config: riva_asr_pb2.RecognitionConfig, audio: bytes + ) -> None: + remaining: Final = context.time_remaining() + released: Final = threading.Event() + if self._script.stall and context.add_callback(released.set): + released.wait(timeout=_STALL_LIMIT_SECONDS) + self._calls.put( + _Call( + rpc=rpc, + metadata=tuple((item.key, item.value) for item in context.invocation_metadata()), + config=config, + audio=audio, + deadline_seconds=None if remaining is None or remaining > _NO_DEADLINE_SECONDS else remaining, + cancelled=not context.is_active(), + ) + ) + if self._script.abort is not None: + context.abort(*self._script.abort) + + def Recognize( + self, request: riva_asr_pb2.RecognizeRequest, context: grpc.ServicerContext + ) -> riva_asr_pb2.RecognizeResponse: + self._settle(context, "Recognize", request.config, request.audio) + return self._script.offline + + def StreamingRecognize( + self, request_iterator: Iterable[riva_asr_pb2.StreamingRecognizeRequest], context: grpc.ServicerContext + ) -> Iterator[riva_asr_pb2.StreamingRecognizeResponse]: + first, *rest = tuple(request_iterator) + self._settle( + context, "StreamingRecognize", first.streaming_config.config, b"".join(r.audio_content for r in rest) + ) + yield from self._script.streaming + + +@dataclass(frozen=True, slots=True) +class _Tls: + key_pem: bytes + cert_pem: bytes + + +def _self_signed_loopback_tls() -> _Tls: + key: Final = ec.generate_private_key(ec.SECP256R1()) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime(2000, 1, 1, tzinfo=timezone.utc)) + .not_valid_after(datetime(2099, 1, 1, tzinfo=timezone.utc)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) ) - buf = io.BytesIO() - sf.write(buf, samples, sample_rate, format="WAV", subtype="PCM_16") - return buf.getvalue() - - -def _fake_word(word: str, start_ms: int, end_ms: int): - return SimpleNamespace(word=word, start_time=start_ms, end_time=end_ms) - - -def _fake_alternative(transcript: str, words=None): - return SimpleNamespace(transcript=transcript, words=words or []) - - -def _fake_result(is_final: bool, alternatives): - return SimpleNamespace(is_final=is_final, alternatives=alternatives) - - -def _fake_response(results): - return SimpleNamespace(results=results) - - -@pytest.fixture -def mock_riva(monkeypatch): - """ - Stand-ins for the bits of ``riva.client`` the handler touches: - - ``Auth`` (constructor) - - ``ASRService`` with ``streaming_response_generator`` - - ``RecognitionConfig``, ``StreamingRecognitionConfig``, ``EndpointingConfig`` - - ``AudioEncoding`` namespace with ``LINEAR_PCM`` - """ - auth_calls = {} - - class FakeAuth: - def __init__(self, *args, **kwargs): - # Support both keyword and positional Auth constructors. - if kwargs: - auth_calls["uri"] = kwargs.get("uri") - auth_calls["use_ssl"] = kwargs.get("use_ssl") - auth_calls["metadata_args"] = kwargs.get("metadata_args") - else: - # positional: (None, use_ssl, uri, metadata) - auth_calls["use_ssl"] = args[1] if len(args) > 1 else None - auth_calls["uri"] = args[2] if len(args) > 2 else None - auth_calls["metadata_args"] = args[3] if len(args) > 3 else None - - class FakeRecognitionConfig: - def __init__(self, **kwargs): - self._kwargs = kwargs - self.endpointing_config = SimpleNamespace(CopyFrom=lambda _: None) - - class FakeStreamingRecognitionConfig: - def __init__(self, config, interim_results): - self.config = config - self.interim_results = interim_results - - class FakeEndpointingConfig: - def __init__(self, **kwargs): - self._kwargs = kwargs - - class FakeAudioEncoding: - LINEAR_PCM = "LINEAR_PCM" - - streaming_responses_holder = {"value": []} - offline_response_holder = {"value": None} - offline_error_holder = {"value": None} - offline_future_holder = {"value": None} - streaming_calls = [] - offline_calls = {} - - class FakeRecognizeFuture: - def result(self, timeout: float | None = None) -> object: - offline_calls["timeout"] = timeout - if offline_error_holder["value"] is not None: - raise offline_error_holder["value"] - return offline_response_holder["value"] - - def done(self) -> bool: - return True - - def cancel(self) -> bool: - return False - - class FakeASRService: - def __init__(self, auth): - self.auth = auth - - def streaming_response_generator(self, audio_chunks, streaming_config): - # Drain audio_chunks generator so we exercise the chunking path. - streaming_calls.append(list(audio_chunks)) - yield from streaming_responses_holder["value"] - - def offline_recognize(self, audio_bytes: bytes, config: object, future: bool = False) -> object: - offline_calls["audio_bytes"] = audio_bytes - offline_calls["config"] = config - pending = offline_future_holder["value"] or FakeRecognizeFuture() - return pending if future else pending.result() - - fake_riva_client = SimpleNamespace( - Auth=FakeAuth, - ASRService=FakeASRService, - RecognitionConfig=FakeRecognitionConfig, - StreamingRecognitionConfig=FakeStreamingRecognitionConfig, - EndpointingConfig=FakeEndpointingConfig, - AudioEncoding=FakeAudioEncoding, - ) - - def fake_import_riva(): - return fake_riva_client, fake_riva_client - - monkeypatch.setattr(handler_mod, "_import_riva", fake_import_riva) - - return SimpleNamespace( - auth_calls=auth_calls, - responses=streaming_responses_holder, - offline_response=offline_response_holder, - offline_error=offline_error_holder, - offline_future=offline_future_holder, - streaming_calls=streaming_calls, - offline_calls=offline_calls, - client=fake_riva_client, + return _Tls( + key_pem=key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ), + cert_pem=cert.public_bytes(serialization.Encoding.PEM), ) -@pytest.fixture -def logging_obj(): - return MagicMock() +@contextmanager +def _riva_server(script: _Script, max_receive_bytes: int | None = None, tls: _Tls | None = None) -> Iterator[_Riva]: + calls: Final[SimpleQueue[_Call]] = SimpleQueue() + options: Final = () if max_receive_bytes is None else (("grpc.max_receive_message_length", max_receive_bytes),) + server: Final = grpc.server(futures.ThreadPoolExecutor(max_workers=4), options=options) + riva_asr_pb2_grpc.add_RivaSpeechRecognitionServicer_to_server(_Asr(script, calls), server) + port: Final = ( + server.add_insecure_port("127.0.0.1:0") + if tls is None + else server.add_secure_port("127.0.0.1:0", grpc.ssl_server_credentials(((tls.key_pem, tls.cert_pem),))) + ) + server.start() + try: + yield _Riva(f"127.0.0.1:{port}", calls) + finally: + server.stop(grace=None).wait() -def test_sync_path_aggregates_only_final_results(mock_riva, logging_obj): - mock_riva.responses["value"] = [ - # Empty heartbeat chunk: ignore. - _fake_response(results=[]), - # Interim chunk (not final): ignore. - _fake_response( - results=[ - _fake_result( - is_final=False, alternatives=[_fake_alternative("partial...")] - ) - ] - ), - # Two final chunks aggregated. - _fake_response( - results=[ - _fake_result( - is_final=True, - alternatives=[ - _fake_alternative( - "Hello,", - words=[_fake_word("Hello,", 0, 320)], - ) - ], - ) - ] - ), - _fake_response( - results=[ - _fake_result( - is_final=True, - alternatives=[ - _fake_alternative( - " world.", - words=[_fake_word("world.", 480, 870)], - ) - ], - ) - ] - ), - ] +def _samples(seconds: float) -> array[int]: + return array("h", (((index * 37) % 2001 - 1000) * 16 for index in range(int(seconds * _SAMPLE_RATE_HZ)))) - impl = NvidiaRivaAudioTranscription() - response: TranscriptionResponse = impl.audio_transcriptions( - model="nvidia/parakeet-ctc-1_1b-asr", - audio_file=_make_wav_bytes(), - optional_params={ - "language_code": "en-US", - "enable_word_time_offsets": True, - "response_format": "verbose_json", - "timestamp_granularities": ["word"], - }, - litellm_params={}, + +def _wav(seconds: float = 1.0) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as writer: + writer.setnchannels(1) + writer.setsampwidth(2) + writer.setframerate(_SAMPLE_RATE_HZ) + writer.writeframes(_samples(seconds).tobytes()) + return buffer.getvalue() + + +def _carries_clip(audio: bytes, seconds: float) -> bool: + sent: Final = _samples(seconds) + if len(audio) != len(sent) * 2: + return False + return all(abs(left - right) <= 1 for left, right in zip(sent, array("h", audio), strict=True)) + + +def _alternative( + transcript: str, words: tuple[tuple[str, int, int], ...] = () +) -> riva_asr_pb2.SpeechRecognitionAlternative: + return riva_asr_pb2.SpeechRecognitionAlternative( + transcript=transcript, + words=[riva_asr_pb2.WordInfo(word=word, start_time=start, end_time=end) for word, start, end in words], + ) + + +def _streamed(transcript: str, is_final: bool = True) -> riva_asr_pb2.StreamingRecognizeResponse: + return riva_asr_pb2.StreamingRecognizeResponse( + results=[riva_asr_pb2.StreamingRecognitionResult(alternatives=[_alternative(transcript)], is_final=is_final)] + ) + + +def _offline(*transcripts: str) -> riva_asr_pb2.RecognizeResponse: + return riva_asr_pb2.RecognizeResponse( + results=[riva_asr_pb2.SpeechRecognitionResult(alternatives=[_alternative(text)]) for text in transcripts] + ) + + +def _answers(transcript: str) -> _Script: + return _Script(streaming=(_streamed(transcript),), offline=_offline(transcript)) + + +def _logging() -> Logging: + return Logging( + model="m", + messages=[], + stream=False, + call_type="transcription", + start_time=datetime(2026, 1, 1), + litellm_call_id=str(uuid.uuid4()), + function_id="riva-test", + ) + + +def _transcribe( + riva: _Riva, + optional_params: dict[str, object] | None = None, + *, + api_key: str | None = None, + timeout: float | str | httpx.Timeout | None = 60, + litellm_params: dict[str, object] | None = None, +) -> TranscriptionResponse: + return NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_wav(), + optional_params={"language_code": "en-US", **(optional_params or {})}, + litellm_params=litellm_params or {}, model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key="nvapi-xxx", - api_base="grpc.nvcf.nvidia.com:443", - ) - - assert response.text == "Hello, world." - # duration is propagated from the resampler. - assert response._hidden_params["audio_transcription_duration"] == pytest.approx( - 1.0, abs=0.05 - ) - # word timestamps converted from ms to seconds. - words = response["words"] - assert words[0]["start"] == pytest.approx(0.0) - assert words[1]["end"] == pytest.approx(0.87) - assert ( - logging_obj.pre_call.call_args.kwargs["additional_args"]["atranscription"] - is False + timeout=timeout, + logging_obj=_logging(), + api_key=api_key, + api_base=riva.address, ) -def test_riva_offline_uses_unary_recognize_and_keeps_results_without_is_final( - mock_riva: SimpleNamespace, logging_obj: MagicMock -) -> None: - """ - Offline-only NIMs (Parakeet TDT, Whisper) reject StreamingRecognize. With - ``riva_offline`` the handler must call ``offline_recognize`` with the whole - resampled buffer and a bare ``RecognitionConfig``, and must keep every - result even though ``RecognizeResponse`` results carry no ``is_final``. - """ - mock_riva.offline_response["value"] = _fake_response( +def test_streaming_aggregates_only_final_results_and_their_word_timings() -> None: + heartbeat: Final = riva_asr_pb2.StreamingRecognizeResponse() + hello: Final = riva_asr_pb2.StreamingRecognizeResponse( results=[ - SimpleNamespace(alternatives=[_fake_alternative("Hello,", words=[_fake_word("Hello,", 0, 320)])]), - SimpleNamespace(alternatives=[_fake_alternative(" world.", words=[_fake_word("world.", 480, 870)])]), + riva_asr_pb2.StreamingRecognitionResult( + alternatives=[_alternative("Hello,", (("Hello,", 0, 320),))], is_final=True + ) ] ) - - impl = NvidiaRivaAudioTranscription() - response: TranscriptionResponse = impl.audio_transcriptions( - model="nvidia/parakeet-0.6b-tdt", - audio_file=_make_wav_bytes(), - optional_params={ - "language_code": "multi", - "riva_offline": True, - "enable_word_time_offsets": True, - "response_format": "verbose_json", - "timestamp_granularities": ["word"], - }, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", + world: Final = riva_asr_pb2.StreamingRecognizeResponse( + results=[ + riva_asr_pb2.StreamingRecognitionResult( + alternatives=[_alternative(" world.", (("world.", 480, 870),))], is_final=True + ) + ] ) + with _riva_server(_Script(streaming=(heartbeat, _streamed("partial...", is_final=False), hello, world))) as riva: + response: Final = _transcribe( + riva, + {"enable_word_time_offsets": True, "response_format": "verbose_json", "timestamp_granularities": ["word"]}, + ) + call: Final = riva.only_call() assert response.text == "Hello, world." - assert response["words"][1]["end"] == pytest.approx(0.87) - assert response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0, abs=0.05) - assert mock_riva.streaming_calls == [] - assert len(mock_riva.offline_calls["audio_bytes"]) == pytest.approx(32000, abs=1600) - assert mock_riva.offline_calls["config"]._kwargs["language_code"] == "multi" - assert logging_obj.pre_call.call_args.kwargs["additional_args"]["complete_input_dict"]["riva_offline"] is True - - -def test_default_mode_streams_and_never_calls_offline_recognize( - mock_riva: SimpleNamespace, logging_obj: MagicMock -) -> None: - mock_riva.responses["value"] = [ - _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) + assert response["words"] == [ + {"word": "Hello,", "start": 0.0, "end": 0.32}, + {"word": "world.", "start": 0.48, "end": 0.87}, ] + assert response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + assert call.rpc == "StreamingRecognize" + assert _carries_clip(call.audio, 1.0), "streamed chunks do not add up to the resampled clip" - impl = NvidiaRivaAudioTranscription() - response = impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", + +def test_riva_offline_uses_recognize_and_keeps_results_without_is_final() -> None: + offline: Final = riva_asr_pb2.RecognizeResponse( + results=[ + riva_asr_pb2.SpeechRecognitionResult(alternatives=[_alternative("Hello,", (("Hello,", 0, 320),))]), + riva_asr_pb2.SpeechRecognitionResult(alternatives=[_alternative(" world.", (("world.", 480, 870),))]), + ] ) - assert response.text == "streamed" - assert mock_riva.offline_calls == {} - assert len(mock_riva.streaming_calls) == 1 - assert logging_obj.pre_call.call_args.kwargs["additional_args"]["complete_input_dict"]["riva_offline"] is False - - - -@pytest.mark.parametrize(("riva_offline", "expect_offline"), [("false", False), ("False", False), ("true", True)]) -def test_riva_offline_string_from_multipart_form_is_parsed( - mock_riva: SimpleNamespace, logging_obj: MagicMock, riva_offline: str, expect_offline: bool -) -> None: - mock_riva.responses["value"] = [ - _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) - ] - mock_riva.offline_response["value"] = _fake_response( - results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] - ) - - response = NvidiaRivaAudioTranscription().audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US", "riva_offline": riva_offline}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - - assert response.text == ("offline" if expect_offline else "streamed") - assert (mock_riva.offline_calls != {}) is expect_offline - - -@pytest.mark.parametrize("riva_offline", ["yes", "", 1]) -def test_riva_offline_malformed_value_is_rejected_before_any_rpc( - mock_riva: SimpleNamespace, logging_obj: MagicMock, riva_offline: object -) -> None: - with pytest.raises(NvidiaRivaException) as excinfo: - NvidiaRivaAudioTranscription().audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US", "riva_offline": riva_offline}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", + with _riva_server(_Script(offline=offline)) as riva: + response: Final = _transcribe( + riva, + { + "language_code": "multi", + "riva_offline": True, + "enable_word_time_offsets": True, + "response_format": "verbose_json", + "timestamp_granularities": ["word"], + }, ) + call: Final = riva.only_call() - assert excinfo.value.status_code == 400 - assert "riva_offline" in excinfo.value.message - assert mock_riva.offline_calls == {} - assert mock_riva.streaming_calls == [] + assert response.text == "Hello, world." + assert response["words"][1] == {"word": "world.", "start": 0.48, "end": 0.87} + assert response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + assert call.rpc == "Recognize" + assert call.config.language_code == "multi" + assert _carries_clip(call.audio, 1.0), "offline request does not carry the whole resampled clip" -def test_riva_offline_malformed_value_is_dropped_under_drop_params( - mock_riva: SimpleNamespace, logging_obj: MagicMock -) -> None: - mock_riva.responses["value"] = [ - _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) - ] +@pytest.mark.parametrize("riva_offline", [False, True]) +def test_recognition_config_reaches_riva_as_nvidia_decodes_it(riva_offline: bool) -> None: + with _riva_server(_answers("ok")) as riva: + _transcribe( + riva, + { + "riva_offline": riva_offline, + "enable_word_time_offsets": True, + "riva_model_name": "parakeet", + "sample_rate_hertz": 8000, + "profanity_filter": True, + }, + ) + call: Final = riva.only_call() - response = NvidiaRivaAudioTranscription().audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US", "riva_offline": "yes"}, - litellm_params={"drop_params": True}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", + assert call.config == riva_asr_pb2.RecognitionConfig( + encoding=riva_audio_pb2.AudioEncoding.LINEAR_PCM, + sample_rate_hertz=_SAMPLE_RATE_HZ, + language_code="en-US", + max_alternatives=1, + profanity_filter=True, + audio_channel_count=1, + enable_word_time_offsets=True, + enable_automatic_punctuation=True, + model="parakeet", ) + +def test_default_mode_streams_and_never_calls_recognize() -> None: + with _riva_server(_answers("streamed")) as riva: + response: Final = _transcribe(riva) + call: Final = riva.only_call() + assert response.text == "streamed" - assert mock_riva.offline_calls == {} - - -def test_deployment_riva_offline_reaches_the_handler_through_transcription(mock_riva: SimpleNamespace) -> None: - mock_riva.offline_response["value"] = _fake_response( - results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] - ) - - response = litellm.transcription( - model="nvidia_riva/nvidia/parakeet-0.6b-tdt", - file=("clip.wav", _make_wav_bytes(), "audio/wav"), - api_base="localhost:50051", - riva_offline=True, - ) - - assert response.text == "offline" - assert mock_riva.streaming_calls == [] - assert mock_riva.offline_calls != {} + assert call.rpc == "StreamingRecognize" @pytest.mark.parametrize( - ("timeout", "waited_for"), - [("7.5", 7.5), (httpx.Timeout(30.0, read=7.5), 7.5), (httpx.Timeout(5.0, read=None), 600.0), (None, None)], + ("riva_offline", "rpc"), [("false", "StreamingRecognize"), ("False", "StreamingRecognize"), ("true", "Recognize")] ) -def test_offline_recognition_waits_the_timeout_in_seconds_whatever_its_form( - mock_riva: SimpleNamespace, timeout: str | httpx.Timeout | None, waited_for: float | None -) -> None: - mock_riva.offline_response["value"] = _fake_response( - results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] - ) +def test_riva_offline_string_from_multipart_form_is_parsed(riva_offline: str, rpc: str) -> None: + with _riva_server(_Script(streaming=(_streamed("streamed"),), offline=_offline("offline"))) as riva: + response: Final = _transcribe(riva, {"riva_offline": riva_offline}) + call: Final = riva.only_call() - response = litellm.transcription( - model="nvidia_riva/nvidia/parakeet-0.6b-tdt", - file=("clip.wav", _make_wav_bytes(), "audio/wav"), - api_base="localhost:50051", - riva_offline=True, - timeout=timeout, - ) + assert call.rpc == rpc + assert response.text == ("offline" if rpc == "Recognize" else "streamed") + + +@pytest.mark.parametrize("riva_offline", ["yes", "", 1]) +def test_riva_offline_malformed_value_is_rejected_before_any_rpc(riva_offline: object) -> None: + with _riva_server(_answers("ok")) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, {"riva_offline": riva_offline}) + + assert riva.calls.empty() + + assert excinfo.value.status_code == 400 + assert "riva_offline" in excinfo.value.message + + +def test_riva_offline_malformed_value_is_dropped_under_drop_params() -> None: + with _riva_server(_Script(streaming=(_streamed("streamed"),), offline=_offline("offline"))) as riva: + response: Final = _transcribe(riva, {"riva_offline": "yes"}, litellm_params={"drop_params": True}) + call: Final = riva.only_call() + + assert response.text == "streamed" + assert call.rpc == "StreamingRecognize" + + +def test_deployment_riva_offline_reaches_the_handler_through_transcription() -> None: + with _riva_server(_Script(offline=_offline("offline"))) as riva: + response: Final = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _wav(), "audio/wav"), + api_base=riva.address, + riva_offline=True, + ) + call: Final = riva.only_call() assert response.text == "offline" - assert mock_riva.offline_calls["timeout"] == waited_for + assert call.rpc == "Recognize" -@pytest.mark.parametrize("timeout", ["abc", "30s"]) -def test_malformed_timeout_on_offline_call_is_a_400_before_any_rpc( - mock_riva: SimpleNamespace, timeout: str +@pytest.mark.parametrize( + ("timeout", "deadline_seconds"), + [ + (12.5, 12.5), + ("12.5", 12.5), + (httpx.Timeout(60.0, read=12.5), 12.5), + (httpx.Timeout(5.0, read=None), 600.0), + (None, None), + ], +) +def test_streaming_deadline_is_the_timeout_in_seconds_whatever_its_form( + timeout: float | str | httpx.Timeout | None, deadline_seconds: float | None ) -> None: - with pytest.raises(NvidiaRivaException) as excinfo: - litellm.transcription( - model="nvidia_riva/nvidia/parakeet-0.6b-tdt", - file=("clip.wav", _make_wav_bytes(), "audio/wav"), - api_base="localhost:50051", - riva_offline=True, + with _riva_server(_answers("streamed")) as riva: + response: Final = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-ctc-1.1b", + file=("clip.wav", _wav(), "audio/wav"), + api_base=riva.address, timeout=timeout, ) + call: Final = riva.only_call() + + assert response.text == "streamed" + assert call.deadline_seconds == (None if deadline_seconds is None else pytest.approx(deadline_seconds, rel=0.25)) + + +@pytest.mark.parametrize("riva_offline", [False, True]) +@pytest.mark.parametrize("timeout", ["abc", "30s"]) +def test_malformed_timeout_is_a_400_before_any_rpc(riva_offline: bool, timeout: str) -> None: + with _riva_server(_answers("ok")) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _wav(), "audio/wav"), + api_base=riva.address, + riva_offline=riva_offline, + timeout=timeout, + ) + + assert riva.calls.empty() assert excinfo.value.status_code == 400 assert "timeout" in excinfo.value.message - assert "audio_bytes" not in mock_riva.offline_calls -def test_malformed_timeout_on_offline_call_waits_the_default_under_drop_params( - mock_riva: SimpleNamespace, -) -> None: - mock_riva.offline_response["value"] = _fake_response( - results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] - ) - - response = litellm.transcription( - model="nvidia_riva/nvidia/parakeet-0.6b-tdt", - file=("clip.wav", _make_wav_bytes(), "audio/wav"), - api_base="localhost:50051", - riva_offline=True, - timeout="abc", - drop_params=True, - ) - - assert response.text == "offline" - assert mock_riva.offline_calls["timeout"] == 600.0 - - -def test_malformed_timeout_still_streams_when_the_client_takes_no_timeout(mock_riva: SimpleNamespace) -> None: - mock_riva.responses["value"] = [ - _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) - ] - - response = litellm.transcription( - model="nvidia_riva/nvidia/parakeet-ctc-1.1b", - file=("clip.wav", _make_wav_bytes(), "audio/wav"), - api_base="localhost:50051", - timeout="abc", - ) - - assert response.text == "streamed" - assert len(mock_riva.streaming_calls) == 1 - - -def test_oversized_offline_audio_is_a_non_retryable_413(mock_riva: SimpleNamespace, logging_obj: MagicMock) -> None: - class FakeGrpcError(Exception): - def code(self) -> SimpleNamespace: - return SimpleNamespace(name="RESOURCE_EXHAUSTED") - - def details(self) -> str: - return "SERVER: Received message larger than max (4348825 vs. 4194304)" - - mock_riva.offline_error["value"] = FakeGrpcError("rpc fail") - - with pytest.raises(NvidiaRivaException) as excinfo: - NvidiaRivaAudioTranscription().audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US", "riva_offline": True}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", +def test_malformed_streaming_timeout_falls_back_to_the_default_deadline_under_drop_params() -> None: + with _riva_server(_answers("ok")) as riva: + response: Final = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-ctc-1.1b", + file=("clip.wav", _wav(), "audio/wav"), + api_base=riva.address, + timeout="abc", + drop_params=True, ) + call: Final = riva.only_call() - assert excinfo.value.status_code == 413 + assert response.text == "ok" + assert call.rpc == "StreamingRecognize" + assert call.deadline_seconds == pytest.approx(600.0, rel=0.25) + + +def test_malformed_offline_timeout_is_ignored_under_drop_params() -> None: + with _riva_server(_answers("ok")) as riva: + response: Final = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _wav(), "audio/wav"), + api_base=riva.address, + riva_offline=True, + timeout="abc", + drop_params=True, + ) + call: Final = riva.only_call() + + assert response.text == "ok" + assert call.rpc == "Recognize" + + +@pytest.mark.parametrize( + "timeout", [_STALL_TIMEOUT_SECONDS, str(_STALL_TIMEOUT_SECONDS), httpx.Timeout(30.0, read=_STALL_TIMEOUT_SECONDS)] +) +def test_stalled_offline_recognition_is_cancelled_and_times_out_as_408(timeout: float | str | httpx.Timeout) -> None: + with _riva_server(_Script(stall=True)) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, {"riva_offline": True}, timeout=timeout) + call: Final = riva.only_call() + + assert excinfo.value.status_code == 408 + assert f"within {_STALL_TIMEOUT_SECONDS} seconds" in excinfo.value.message + assert call.cancelled is True + + +def test_stalled_streaming_recognition_hits_its_deadline_as_504() -> None: + with _riva_server(_Script(stall=True)) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, timeout=_STALL_TIMEOUT_SECONDS) + call: Final = riva.only_call() + + assert excinfo.value.status_code == 504, excinfo.value.message + assert "DEADLINE_EXCEEDED" in excinfo.value.message + assert call.cancelled is True + + +def test_oversized_offline_audio_is_a_non_retryable_413() -> None: + with _riva_server(_answers("ok"), max_receive_bytes=4096) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, {"riva_offline": True}) + + assert riva.calls.empty() + + assert excinfo.value.status_code == 413, excinfo.value.message assert litellm.utils.should_retry(excinfo.value.status_code) is False -def test_stalled_offline_recognition_is_cancelled_and_times_out_as_408( - mock_riva: SimpleNamespace, logging_obj: MagicMock +@pytest.mark.parametrize( + ("riva_offline", "code", "status_code"), + [ + (False, grpc.StatusCode.UNAUTHENTICATED, 401), + (True, grpc.StatusCode.INVALID_ARGUMENT, 400), + (False, grpc.StatusCode.RESOURCE_EXHAUSTED, 429), + ], +) +def test_grpc_error_is_wrapped_as_nvidia_riva_exception( + riva_offline: bool, code: grpc.StatusCode, status_code: int ) -> None: - class FutureTimeoutError(Exception): - pass + with _riva_server(_Script(abort=(code, "Riva said no"))) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, {"riva_offline": riva_offline}, api_key=_API_KEY) - class StalledRecognizeFuture: - def __init__(self) -> None: - self.waited_for: float | None = None - self.cancelled = False + assert excinfo.value.status_code == status_code + assert code.name in excinfo.value.message + assert "Riva said no" in excinfo.value.message - def result(self, timeout: float | None = None) -> object: - self.waited_for = timeout - raise FutureTimeoutError() - def done(self) -> bool: - return self.cancelled +@pytest.mark.parametrize("riva_offline", [False, True]) +def test_self_hosted_api_key_is_sent_as_one_bearer_token(riva_offline: bool) -> None: + with _riva_server(_answers("ok")) as riva: + _transcribe(riva, {"riva_offline": riva_offline}, api_key=_API_KEY) + call: Final = riva.only_call() - def cancel(self) -> bool: - self.cancelled = True - return True + assert call.header("authorization") == (f"Bearer {_API_KEY}",) + assert call.header("function-id") == () - stalled = StalledRecognizeFuture() - mock_riva.offline_future["value"] = stalled +@pytest.mark.parametrize("riva_offline", [False, True]) +def test_nvcf_function_id_and_api_key_are_each_sent_once(riva_offline: bool) -> None: + with _riva_server(_answers("ok")) as riva: + _transcribe( + riva, {"riva_offline": riva_offline, "nvcf_function_id": "abc-123", "use_ssl": False}, api_key=_API_KEY + ) + call: Final = riva.only_call() + + assert call.header("function-id") == ("abc-123",) + assert call.header("authorization") == (f"Bearer {_API_KEY}",) + + +@pytest.mark.parametrize("riva_offline", [False, True]) +def test_no_api_key_sends_no_authorization(riva_offline: bool) -> None: + with _riva_server(_answers("ok")) as riva: + _transcribe(riva, {"riva_offline": riva_offline}) + call: Final = riva.only_call() + + assert call.header("authorization") == () + assert call.header("function-id") == () + + +@pytest.mark.parametrize("riva_offline", [False, True]) +@pytest.mark.parametrize( + ("optional_params", "function_ids"), + [({"use_ssl": True}, ()), ({"nvcf_function_id": "abc-123"}, ("abc-123",))], + ids=["use_ssl", "nvcf_default_tls"], +) +def test_tls_api_key_is_sent_as_one_bearer_token( + monkeypatch: pytest.MonkeyPatch, + optional_params: dict[str, object], + function_ids: tuple[str, ...], + riva_offline: bool, +) -> None: + tls: Final = _self_signed_loopback_tls() + system_roots: Final = grpc.ssl_channel_credentials + monkeypatch.setattr(grpc, "ssl_channel_credentials", lambda: system_roots(root_certificates=tls.cert_pem)) + with _riva_server(_answers("ok"), tls=tls) as riva: + response: Final = _transcribe(riva, {**optional_params, "riva_offline": riva_offline}, api_key=_API_KEY) + call: Final = riva.only_call() + + assert response.text == "ok" + assert call.header("authorization") == (f"Bearer {_API_KEY}",) + assert call.header("function-id") == function_ids + + +@pytest.mark.parametrize("optional_params", [{"use_ssl": True}, {"nvcf_function_id": "abc-123"}]) +def test_tls_never_falls_back_to_plaintext(optional_params: dict[str, object]) -> None: + with _riva_server(_answers("ok")) as riva: + with pytest.raises(NvidiaRivaException) as excinfo: + _transcribe(riva, optional_params, api_key=_API_KEY, timeout=_SERVER_WAIT_SECONDS) + + assert riva.calls.empty() + + assert excinfo.value.status_code == 503, excinfo.value.message + + +def test_explicit_endpointing_config_keeps_zero_thresholds() -> None: + with _riva_server(_answers("ok")) as riva: + _transcribe(riva, {"endpointing_config": {"stop_threshold": 0.0, "stop_history": 800}}) + call: Final = riva.only_call() + + assert call.config.HasField("endpointing_config") + assert call.config.endpointing_config.HasField("stop_threshold") + assert call.config.endpointing_config == riva_asr_pb2.EndpointingConfig(stop_threshold=0.0, stop_history=800) + + +def test_unknown_endpointing_field_falls_back_to_riva_defaults() -> None: + with _riva_server(_answers("ok")) as riva: + response: Final = _transcribe(riva, {"endpointing_config": {"stop_threshold": 0.5, "not_a_field": 1}}) + call: Final = riva.only_call() + + assert response.text == "ok" + assert not call.config.HasField("endpointing_config") + + +def test_missing_api_base_raises_clear_error() -> None: with pytest.raises(NvidiaRivaException) as excinfo: NvidiaRivaAudioTranscription().audio_transcriptions( model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US", "riva_offline": True}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=7.5, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - - assert excinfo.value.status_code == 408 - assert stalled.waited_for == 7.5 - assert stalled.cancelled is True - -def test_riva_offline_grpc_error_is_wrapped(mock_riva: SimpleNamespace, logging_obj: MagicMock) -> None: - class FakeGrpcError(Exception): - def code(self) -> SimpleNamespace: - return SimpleNamespace(name="INVALID_ARGUMENT") - - def details(self) -> str: - return "Unavailable model requested" - - mock_riva.offline_error["value"] = FakeGrpcError("rpc fail") - - impl = NvidiaRivaAudioTranscription() - with pytest.raises(NvidiaRivaException) as excinfo: - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "multi", "riva_offline": True}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - assert excinfo.value.status_code == 400 - assert "INVALID_ARGUMENT" in excinfo.value.message - - -def test_auth_nvcf_defaults_use_ssl_and_attaches_function_id(mock_riva, logging_obj): - mock_riva.responses["value"] = [ - _fake_response( - results=[ - _fake_result( - is_final=True, - alternatives=[_fake_alternative("ok")], - ) - ] - ) - ] - impl = NvidiaRivaAudioTranscription() - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={ - "nvcf_function_id": "abc-123", - "language_code": "en-US", - }, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key="nvapi-xxx", - api_base="grpc.nvcf.nvidia.com:443", - ) - - assert mock_riva.auth_calls["uri"] == "grpc.nvcf.nvidia.com:443" - assert mock_riva.auth_calls["use_ssl"] is True - metadata = dict(mock_riva.auth_calls["metadata_args"]) - assert metadata["function-id"] == "abc-123" - assert metadata["authorization"] == "Bearer nvapi-xxx" - - -def test_auth_self_hosted_defaults_no_ssl_and_no_function_id(mock_riva, logging_obj): - mock_riva.responses["value"] = [ - _fake_response( - results=[ - _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) - ] - ) - ] - impl = NvidiaRivaAudioTranscription() - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - - assert mock_riva.auth_calls["uri"] == "localhost:50051" - assert mock_riva.auth_calls["use_ssl"] is False - metadata = dict(mock_riva.auth_calls["metadata_args"]) - # No function-id, no authorization metadata. - assert "function-id" not in metadata - assert "authorization" not in metadata - - -def test_explicit_use_ssl_override_wins(mock_riva, logging_obj): - """ - Self-hosted Riva behind an ingress with TLS termination is a real - deployment topology. ``use_ssl=True`` must be honored even without an - NVCF function id. - """ - mock_riva.responses["value"] = [ - _fake_response( - results=[ - _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) - ] - ) - ] - impl = NvidiaRivaAudioTranscription() - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"use_ssl": True, "language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="riva.internal.company.com:443", - ) - - assert mock_riva.auth_calls["use_ssl"] is True - - -def test_missing_api_base_raises_clear_error(mock_riva, logging_obj): - impl = NvidiaRivaAudioTranscription() - with pytest.raises(NvidiaRivaException) as excinfo: - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), + audio_file=_wav(), optional_params={}, litellm_params={}, model_response=TranscriptionResponse(), timeout=60, - logging_obj=logging_obj, + logging_obj=_logging(), api_key=None, api_base=None, ) + + assert excinfo.value.status_code == 400 assert "api_base" in excinfo.value.message -def test_async_path_uses_to_thread(mock_riva, logging_obj): - mock_riva.responses["value"] = [ - _fake_response( - results=[ - _fake_result( - is_final=True, alternatives=[_fake_alternative("async ok")] - ) - ] - ) - ] - impl = NvidiaRivaAudioTranscription() - response = asyncio.run( - impl.async_audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - ) - assert response.text == "async ok" - assert ( - logging_obj.pre_call.call_args.kwargs["additional_args"]["atranscription"] - is True - ) - - -@pytest.mark.parametrize("timeout", [12.5, "12.5", httpx.Timeout(60.0, read=12.5)]) -def test_timeout_is_forwarded_to_streaming_generator_when_supported( - mock_riva: SimpleNamespace, logging_obj: MagicMock, timeout: float | str | httpx.Timeout -) -> None: - """ - Without a deadline the gRPC stream can block forever on a stalled Riva - server. The handler must forward the call-level ``timeout`` to - ``streaming_response_generator`` whenever the installed riva-client - accepts a ``timeout`` kwarg. - """ - captured_kwargs = {} - - def streaming_with_timeout(self, audio_chunks, streaming_config, timeout=None): - captured_kwargs["timeout"] = timeout - list(audio_chunks) - yield from [ - _fake_response( - results=[ - _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) - ] +def test_async_path_transcribes() -> None: + with _riva_server(_answers("async ok")) as riva: + response: Final = asyncio.run( + NvidiaRivaAudioTranscription().async_audio_transcriptions( + model="m", + audio_file=_wav(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=_logging(), + api_key=_API_KEY, + api_base=riva.address, ) - ] - - mock_riva.client.ASRService.streaming_response_generator = streaming_with_timeout - - impl = NvidiaRivaAudioTranscription() - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=timeout, - logging_obj=logging_obj, - api_key=None, - api_base="localhost:50051", - ) - assert captured_kwargs["timeout"] == pytest.approx(12.5) - - -def test_grpc_error_is_wrapped_as_nvidia_riva_exception(mock_riva, logging_obj): - class FakeGrpcError(Exception): - def code(self): - return SimpleNamespace(name="UNAUTHENTICATED") - - def details(self): - return "bad token" - - def raising_streaming_response_generator(self, audio_chunks, streaming_config): - list(audio_chunks) - raise FakeGrpcError("rpc fail") - - mock_riva.client.ASRService.streaming_response_generator = ( - raising_streaming_response_generator - ) - - impl = NvidiaRivaAudioTranscription() - with pytest.raises(NvidiaRivaException) as excinfo: - impl.audio_transcriptions( - model="m", - audio_file=_make_wav_bytes(), - optional_params={"language_code": "en-US"}, - litellm_params={}, - model_response=TranscriptionResponse(), - timeout=60, - logging_obj=logging_obj, - api_key="nvapi-xxx", - api_base="grpc.nvcf.nvidia.com:443", ) + call: Final = riva.only_call() - assert excinfo.value.status_code == 401 - assert "UNAUTHENTICATED" in excinfo.value.message + assert response.text == "async ok" + assert call.header("authorization") == (f"Bearer {_API_KEY}",) diff --git a/tests/unit/llms/nvidia_riva/audio_transcription/test_riva_asr_schema.py b/tests/unit/llms/nvidia_riva/audio_transcription/test_riva_asr_schema.py new file mode 100644 index 00000000000..124e3c28291 --- /dev/null +++ b/tests/unit/llms/nvidia_riva/audio_transcription/test_riva_asr_schema.py @@ -0,0 +1,41 @@ +from collections.abc import Iterator +from typing import Final + +from google.protobuf.descriptor import FieldDescriptor +from riva.client.proto import riva_asr_pb2, riva_audio_pb2 + +from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema + + +def _declared_fields() -> Iterator[tuple[str, FieldDescriptor]]: + for message_name, _ in riva_asr_schema._MESSAGES: + message: Final = riva_asr_schema._POOL.FindMessageTypeByName(f"{riva_asr_schema._PACKAGE}.{message_name}") + yield from ((message_name, field) for field in message.fields) + + +def _wire_shape(field: FieldDescriptor | None) -> tuple[object, ...] | None: + if field is None: + return None + return ( + field.number, + field.type, + field.label, + field.has_presence, + field.message_type.full_name if field.message_type else None, + field.enum_type.name if field.enum_type else None, + field.containing_oneof.name if field.containing_oneof else None, + ) + + +def _nvidia_field(message_name: str, field_name: str) -> FieldDescriptor | None: + return riva_asr_pb2.DESCRIPTOR.message_types_by_name[message_name].fields_by_name.get(field_name) + + +def test_every_declared_field_matches_nvidias_riva_schema() -> None: + declared: Final = tuple(_declared_fields()) + + assert len(declared) == sum(len(fields) for _, fields in riva_asr_schema._MESSAGES) + assert {f"{message}.{field.name}": _wire_shape(field) for message, field in declared} == { + f"{message}.{field.name}": _wire_shape(_nvidia_field(message, field.name)) for message, field in declared + } + assert riva_asr_schema.LINEAR_PCM == riva_audio_pb2.AudioEncoding.LINEAR_PCM diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 4aa93848c79..263c2f7618d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2173,3 +2173,45 @@ async def test_openai_request_rewrites_media_parts_without_touching_the_callers_ {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}}, ] assert messages == messages_before + + +_OPENAI_DEPLOYMENT_FLAGS: Final = {"opted-in": {"supports_prompt_cache_breakpoint": True}} + + +@pytest.mark.parametrize("registration_order", [("opted-in", "flagless"), ("flagless", "opted-in")]) +async def test_chat_sends_a_breakpoint_by_the_deployments_own_flag_not_a_siblings_on_the_same_model( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, str] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + catalog_row_without_a_flag: Final = {"litellm_provider": "openai", "mode": "chat"} + monkeypatch.setitem(litellm.model_cost, "qa-openai-shared", catalog_row_without_a_flag) + chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": "openai/qa-openai-shared", + "api_base": "https://api.openai.com/v1", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{name}-id", **_OPENAI_DEPLOYMENT_FLAGS.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + + async def sends_a_breakpoint(name: str) -> bool: + await router.acompletion( + model=name, messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + ) + return b"prompt_cache_breakpoint" in chat.calls.last.request.content + + assert {name: await sends_a_breakpoint(name) for name in registration_order} == { + "opted-in": True, + "flagless": False, + } diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index eddfe578256..186ee545323 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,10 +1,16 @@ from collections.abc import Mapping +from dataclasses import replace from typing import Final +import httpx import pytest from pydantic import TypeAdapter -from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsResponseValidationError, + ir_to_systemone_response, + systemone_request_to_ir, +) from litellm.llms.openai.decisions.transformation import ( OpenAIDecisionsConfig, ir_to_openai_request, @@ -106,6 +112,35 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None: assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE +@pytest.mark.parametrize("answer_type", ["choice", "score"]) +def test_a_choice_or_score_answer_without_confidence_stays_a_provider_error(answer_type: str) -> None: + answers: Final = [ + {key: value for key, value in answer.items() if answer["type"] != answer_type or key != "confidence"} + for answer in _OPENAI_RESPONSE["answers"] + ] + with pytest.raises(DecisionsResponseValidationError) as raised: + OpenAIDecisionsConfig().transform_decisions_response( + "gpt-6-luna", + "openai", + httpx.Response(200, json={**_OPENAI_RESPONSE, "answers": answers}), + _openai_ir(_OPENAI_REQUEST), + ) + assert raised.value.status_code == 500 + + +def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + parsed: Final = replace( + OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []} + ) + + assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE + assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == { + **_OPENAI_RESPONSE, + "citations": [], + } + + def test_answers_openai_did_not_return_are_refusals() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} diff --git a/tests/unit/llms/openai/live/test_handler.py b/tests/unit/llms/openai/live/test_handler.py index ec1130376d0..bb7cda99e79 100644 --- a/tests/unit/llms/openai/live/test_handler.py +++ b/tests/unit/llms/openai/live/test_handler.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Callable, Coroutine from datetime import datetime from typing import Final, NoReturn, cast @@ -13,11 +13,13 @@ from websockets.frames import Close from websockets.http11 import Response import litellm +from litellm import Router from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.cost_calculator import handle_realtime_stream_cost_calculation from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.openai.live import handler as live_handler from litellm.llms.openai.live.handler import LiveSessionStart, OpenAILiveSessions, parse_live_session_start +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.utils import LiteLLMRealtimeStreamLoggingObject, Usage _LIVE_EVENT_FRAME_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -1206,3 +1208,91 @@ async def test_async_live_session_reports_redacted_server_error_and_bounds_close finally: if worker.coroutine is not None: await worker.coroutine + + +_LIVE_SESSION_STARTED: Final = '{"type":"session.started","session":{"model":"gpt-live-1"}}' +_LIVE_SESSION_CLOSED: Final = '{"type":"session.closed","usage":{"seconds":4}}' + + +def _upstream_closing_the_session() -> _FakeUpstream: + return _FakeUpstream(client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED, _LIVE_SESSION_CLOSED)) + + +def _upstream_failing_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_error=ConnectionClosedError(Close(code=1011, reason="upstream failed"), None), + ) + + +def _upstream_dropping_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED,), close_error=ConnectionClosedError(None, None) + ) + + +def _upstream_failing_a_client_send() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_response=_LIVE_SESSION_CLOSED, + fail_on_send_type="conversation.item.create", + send_error=OSError("upstream socket write failed"), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_factory", "counted_successes"), + [ + (_upstream_closing_the_session, 1), + (_upstream_failing_after_a_frame, 0), + (_upstream_dropping_after_a_frame, 0), + (_upstream_failing_a_client_send, 0), + ], + ids=["session-closed", "upstream-1011-after-a-frame", "upstream-1006-after-a-frame", "upstream-send-failed"], +) +async def test_live_session_counts_a_router_success_only_when_it_ended_cleanly( + upstream_factory: Callable[[], _FakeUpstream], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + {"model_name": "live-voice", "litellm_params": {"model": "openai/gpt-live-1"}, "model_info": {"id": "live-a"}} + ], + num_retries=0, + ) + worker: Final = _FakeLoggingWorker() + + async def _arealtime(litellm_logging_obj: Logging, **kwargs: object) -> None: + await OpenAILiveSessions( + logging_worker=worker, websocket_connector=_FakeConnector(upstream_factory()) + ).async_live_session( + model="gpt-live-1", + websocket=_FakeClientWebSocket(messages=('{"type":"conversation.item.create"}',)), + logging_obj=litellm_logging_obj, + session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + api_base="https://api.openai.com/", + api_key="live-test-key", + ) + + await asyncio.wait_for( + router._ageneric_api_call_with_fallbacks( + model="live-voice", + original_function=_arealtime, + litellm_logging_obj=Logging( + model="gpt-live-1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="live-session", + function_id="live-session", + ), + ), + timeout=5, + ) + + if worker.coroutine is not None: + worker.coroutine.close() + assert get_deployment_successes_for_current_minute(router, "live-a") == counted_successes diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index b31db62589d..8ed65ef925c 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,7 +1,11 @@ +import asyncio import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import TypeAdapter from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from litellm.types.realtime import RealtimeQueryParams @@ -526,3 +530,83 @@ async def test_realtime_query_params_preserve_missing_model(monkeypatch): called_kwargs = mock_async_realtime.call_args.kwargs assert called_kwargs["query_params"] == {"intent": "transcription"} + + +class _BackendConnection: + def __init__(self, connection): + self.connection = connection + + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + +def test_translation_url_uses_dedicated_path(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + url = handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> None: + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + backend: Final = AsyncMock() + pending: Final = asyncio.Event() + + async def receive_backend() -> str: + await pending.wait() + return "" + + backend.recv.side_effect = receive_backend + websocket: Final = MagicMock() + websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} + websocket.receive_text = AsyncMock( + side_effect=( + json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"output": {"language": "fr"}}}, + } + ), + RuntimeError("connection closed"), + ) + ) + websocket.close = AsyncMock() + logging_obj: Final = MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock()) + handler: Final = OpenAIRealtime() + expected_url: Final = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert ( + handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + == expected_url + ) + + with patch("websockets.connect", return_value=_BackendConnection(backend)) as connect: + await handler.async_realtime( + model="gpt-realtime-translate", + websocket=websocket, + logging_obj=logging_obj, + api_base="https://api.openai.com/v1", + api_key="sk-test", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + + connect.assert_called_once() + assert connect.call_args.args[0] == expected_url + assert "OpenAI-Beta" not in connect.call_args.kwargs["additional_headers"] + sent: Final = TypeAdapter(Mapping[str, object]).validate_json(backend.send.call_args_list[0].args[0]) + assert sent["session"] == {"audio": {"output": {"language": "fr"}}} diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 3bb9a4ab1ab..f637271d667 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest - +import litellm from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -21,9 +21,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest def test_openai_transcription_session_url(): cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -32,24 +30,36 @@ def test_openai_transcription_session_url_strips_trailing_v1(): """A /v1 suffix must not be duplicated in the path.""" cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com/v1", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) -def test_azure_transcription_session_url_uses_deployment_and_api_version(): +@pytest.mark.parametrize( + "api_base", + ["https://my.openai.azure.com", "https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/v1/"], +) +def test_azure_transcription_session_url_uses_deployment_and_api_version(api_base: str): cfg = AzureRealtimeHTTPConfig() url = cfg.get_transcription_session_url( - api_base="https://my.openai.azure.com", + api_base=api_base, model="whisper-deploy", api_version="2025-04-01-preview", ) - assert ( - url - == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" - ) + assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + cfg = AzureRealtimeHTTPConfig() + base = "https://my.openai.azure.com" + + assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls") + litellm.get_model_info.cache_clear() def test_request_resolves_model_returns_none_when_both_absent(): @@ -141,6 +151,27 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets" +@pytest.mark.asyncio +async def test_azure_client_secret_routes_on_a_provider_qualified_session_model(): + mock_response = MagicMock(spec=httpx.Response) + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=mock_response) + + result = await litellm.acreate_realtime_client_secret( + session={"type": "realtime", "model": "azure/gpt-realtime-2.1"}, + api_base="https://my.openai.azure.com", + api_key="azure-test-key", + api_version="v1", + client=mock_client, + ) + + assert result is mock_response + request = mock_client.post.call_args.kwargs + assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request["headers"]["api-key"] == "azure-test-key" + assert request["json"]["session"]["model"] == "gpt-realtime-2.1" + + @pytest.mark.asyncio async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): """ @@ -169,18 +200,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): assert kwargs["url"].endswith("/v1/realtime/transcription_sessions") # The litellm-only routing hint must not be forwarded upstream. assert "model" not in kwargs["json"] - assert kwargs["json"]["input_audio_transcription"] == { - "model": "gpt-realtime-whisper" - } + assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"} def test_append_query_params_skips_existing_keys(): from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler url = "wss://example.com/v1/realtime?model=gpt-4o" - result = BaseLLMHTTPHandler._append_query_params( - url, {"model": "ignored", "intent": "transcription"} - ) + result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"}) assert "model=ignored" not in result assert "intent=transcription" in result diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py new file mode 100644 index 00000000000..525782704c1 --- /dev/null +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -0,0 +1,185 @@ +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig +from litellm.types.realtime import RealtimeSessionConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_azure_realtime_http_headers_use_ad_token_without_api_key(): + config = AzureRealtimeHTTPConfig() + params = GenericLiteLLMParams(azure_ad_token="test-ad-token") + + assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == { + "X-Trace": "trace", + "Authorization": "Bearer test-ad-token", + } + assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"} + + +def test_realtime_session_config_supports_translation_and_live_transcription_fields(): + session = RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio={ + "input": { + "transcription": { + "model": "gpt-live-transcribe", + "delay": "minimal", + "languages": ["en", "fr"], + "keywords": ["LiteLLM"], + } + }, + "output": {"language": "es"}, + }, + ) + + assert session.audio is not None + assert session.audio.input is not None + assert session.audio.input.transcription is not None + assert session.audio.input.transcription.delay == "minimal" + assert session.audio.input.transcription.languages == ["en", "fr"] + assert session.audio.input.transcription.keywords == ["LiteLLM"] + assert session.audio.output is not None + assert session.audio.output.language == "es" + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ( + "https://api.openai.com", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ( + "https://api.openai.com/v1", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ], +) +def test_openai_translation_client_secret_url(api_base: str, expected: str): + config = OpenAIRealtimeHTTPConfig() + assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected + + +def test_openai_translation_calls_url(): + config = OpenAIRealtimeHTTPConfig() + assert ( + config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate") + == "https://api.openai.com/v1/realtime/translations/calls" + ) + + +@pytest.mark.parametrize("suffix", ("", "/", "/openai", "/openai/v1/")) +def test_azure_translation_urls_use_ga_paths(suffix: str): + config: Final = AzureRealtimeHTTPConfig() + api_base: Final = f"https://example.openai.azure.com{suffix}" + assert ( + config.get_translation_client_secret_url(api_base, "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" + ) + assert ( + config.get_translation_calls_url(api_base, "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_custom_translation_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}} + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://api.openai.com", + api_key="sk-test", + request_data=request_data, + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + ) + + assert response.status_code == 200 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets" + assert call["json"] == request_data + + +@pytest.mark.asyncio +async def test_azure_translation_client_secret_supports_entra_bearer_auth(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request( + "POST", + "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets", + ), + ) + ) + handler = BaseLLMHTTPHandler() + + await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.openai.azure.com", + api_key="", + request_data={"session": {"type": "translation", "model": "translate-deployment"}}, + logging_obj=MagicMock(), + timeout=10, + provider_config=AzureRealtimeHTTPConfig(), + model="translate-deployment", + extra_headers={"Authorization": "Bearer entra-token"}, + client=client, + ) + + headers = client.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +@pytest.mark.asyncio +async def test_translation_calls_use_translation_session_and_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 201, + content=b"v=0\r\n", + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://api.openai.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + translation=True, + ) + + assert response.status_code == 201 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" + assert call["headers"]["Content-Type"] == "application/sdp" + assert call["content"] == "v=0\r\n" diff --git a/tests/unit/llms/openai/responses/test_openai_responses_data_residency.py b/tests/unit/llms/openai/responses/test_openai_responses_data_residency.py index cf9f9706cfe..8fce7a73422 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_data_residency.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_data_residency.py @@ -112,6 +112,28 @@ def test_responses_us_api_base_sets_data_residency(): assert logging_obj.litellm_params.get("data_residency") == "us" +def test_responses_fedramp_api_base_sets_data_residency(): + mock_client = _make_mock_http_client(_make_responses_api_response_body()) + captured, init_spy = _capture_logging_obj() + + with ( + patch( + "litellm.llms.custom_httpx.llm_http_handler.get_httpx_client", + return_value=mock_client, + ), + patch.object(litellm.Logging, "__init__", init_spy), + ): + litellm.responses( + model="gpt-4.1", + input="hi", + api_base="https://gov.api.openai.com/v1", + api_key="test-key", + ) + + logging_obj = captured["logging_obj"] + assert logging_obj.litellm_params.get("data_residency") == "fedramp" + + def test_responses_global_api_base_leaves_data_residency_none(): mock_client = _make_mock_http_client(_make_responses_api_response_body()) captured, init_spy = _capture_logging_obj() diff --git a/tests/unit/llms/openai/test_data_residency.py b/tests/unit/llms/openai/test_data_residency.py index ecb5739133c..443d14d3102 100644 --- a/tests/unit/llms/openai/test_data_residency.py +++ b/tests/unit/llms/openai/test_data_residency.py @@ -2,7 +2,9 @@ import pytest +import litellm from litellm.llms.openai.data_residency import infer_openai_data_residency +from litellm.types.utils import DataResidency @pytest.mark.parametrize( @@ -13,6 +15,8 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency ("https://us.api.openai.com/v1", "us"), ("https://us.api.openai.com", "us"), ("https://EU.api.openai.com/v1", "eu"), + ("https://gov.api.openai.com/v1", DataResidency.FEDRAMP.value), + ("https://GOV.api.openai.com/v1", DataResidency.FEDRAMP.value), ("https://api.openai.com/v1", None), ("https://api.openai.com", None), ("https://example.com/v1", None), @@ -28,7 +32,26 @@ def test_infer_openai_data_residency(api_base, expected): @pytest.mark.parametrize("custom_llm_provider", [None, "anthropic", "azure", "bedrock"]) def test_infer_openai_data_residency_non_openai_provider(custom_llm_provider): - assert ( - infer_openai_data_residency(custom_llm_provider, "https://eu.api.openai.com/v1") - is None - ) + assert infer_openai_data_residency(custom_llm_provider, "https://eu.api.openai.com/v1") is None + + +@pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) +def test_infer_openai_data_residency_from_env_when_api_base_unset(monkeypatch, env_var): + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv(env_var, "https://gov.api.openai.com/v1") + assert infer_openai_data_residency("openai", None) == DataResidency.FEDRAMP.value + + +def test_infer_openai_data_residency_explicit_api_base_wins_over_env(monkeypatch): + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("OPENAI_BASE_URL", "https://gov.api.openai.com/v1") + assert infer_openai_data_residency("openai", "https://eu.api.openai.com/v1") == "eu" + + +def test_infer_openai_data_residency_from_litellm_api_base(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", "https://us.api.openai.com/v1") + assert infer_openai_data_residency("openai", None) == "us" diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 77a7c3b83fd..f61cfba9a10 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -3,7 +3,7 @@ import gzip import importlib import json import os -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -19,6 +19,8 @@ from pydantic import BaseModel, ConfigDict, JsonValue import litellm from litellm import create_thread, get_thread from litellm.caching.caching import Cache, LiteLLMCacheType +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.llms.openai.openai import ( AssistantEventHandler, AsyncAssistantEventHandler, @@ -225,6 +227,333 @@ async def test_acompletion_returns_prompt_cache_details_over_injected_transport( assert prompt_tokens_details.cached_tokens == 7 + +class _SearchingChatGate(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.runs = 0 + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + return True, {"tool_calls": [{"id": "call-1"}]} + + +class _FailingFollowUpChatLoop(_SearchingChatGate): + def __init__(self, error: Exception) -> None: + super().__init__() + self.error = error + self.logging_objs: list[object] = [] + + async def async_run_chat_completion_agentic_loop( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> object: + self.runs += 1 + self.logging_objs.append(logging_obj) + raise self.error + + +class _RaisingAgenticGate(CustomLogger): + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + raise ValueError("gate bug") + + +class _AnsweringChatLoop(_SearchingChatGate): + async def async_run_chat_completion_agentic_loop( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> object: + self.runs += 1 + return litellm.ModelResponse( + choices=[ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "after the search"}} + ] + ) + + +def _respond_with_a_search_tool_call(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-search", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }, + ) + + +async def _acompletion_through(http_client: httpx.AsyncClient) -> object: + return await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=AsyncOpenAI(api_key="transport-only", http_client=http_client), + messages=[{"role": "user", "content": "search the web for the weather"}], + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + +@pytest.mark.asyncio +async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn( + monkeypatch: pytest.MonkeyPatch, +): + follow_up_error: Final = litellm.RateLimitError( + message="follow-up rate limited", llm_provider="openai", model="gpt-5.6" + ) + loop: Final = _FailingFollowUpChatLoop(follow_up_error) + monkeypatch.setattr(litellm, "callbacks", [loop]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + with pytest.raises(litellm.RateLimitError) as raised: + await _acompletion_through(http_client) + + assert raised.value.message == follow_up_error.message + assert loop.runs == 1 + + +@pytest.mark.asyncio +async def test_a_failed_agentic_follow_up_bills_the_first_call_on_the_failure_log(monkeypatch: pytest.MonkeyPatch): + loop: Final = _FailingFollowUpChatLoop( + litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model="gpt-5.6") + ) + monkeypatch.setattr(litellm, "callbacks", [loop]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + with pytest.raises(litellm.RateLimitError): + await _acompletion_through(http_client) + + (logging_obj,) = loop.logging_objs + price: Final = litellm.model_cost["gpt-5.6"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + 5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"] + ) + + +@pytest.mark.asyncio +async def test_acompletion_does_not_resend_the_first_call_when_the_agentic_follow_up_answers_422( + monkeypatch: pytest.MonkeyPatch, +): + follow_up_error: Final = litellm.UnprocessableEntityError( + message="follow-up rejected", + model="gpt-5.6", + llm_provider="openai", + response=httpx.Response(422, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")), + ) + loop: Final = _FailingFollowUpChatLoop(follow_up_error) + monkeypatch.setattr(litellm, "callbacks", [loop]) + monkeypatch.setattr(litellm, "drop_params", True) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + with pytest.raises(litellm.UnprocessableEntityError) as raised: + await _acompletion_through(http_client) + + assert raised.value.message == follow_up_error.message + assert loop.runs == 1 + + +def _rejecting_the_user_param(outbound: asyncio.Queue[dict[str, object]]) -> httpx.MockTransport: + def respond(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + outbound.put_nowait(body) + if "user" in body: + rejection: Final = json.dumps({"detail": [{"loc": ["body", "user"], "msg": "extra fields not permitted"}]}) + return httpx.Response(422, json={"error": {"message": rejection}}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-resent", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}}], + }, + ) + + return httpx.MockTransport(respond) + + +async def _acompletion_with_a_user_param(http_client: httpx.AsyncClient) -> object: + return await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=AsyncOpenAI(api_key="transport-only", http_client=http_client), + messages=[{"role": "user", "content": "hi"}], + user="end-user-1", + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + +@pytest.mark.asyncio +async def test_acompletion_resends_without_the_param_a_422_rejects_when_drop_params_is_set( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "drop_params", True) + outbound: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async with httpx.AsyncClient(transport=_rejecting_the_user_param(outbound)) as http_client: + response: Final = await _acompletion_with_a_user_param(http_client) + + assert response.choices[0].message.content == "done" + assert outbound.get_nowait()["user"] == "end-user-1" + assert "user" not in outbound.get_nowait() + assert outbound.empty() + + +@pytest.mark.asyncio +async def test_acompletion_raises_a_422_without_resending_when_drop_params_is_off(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "drop_params", False) + outbound: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async with httpx.AsyncClient(transport=_rejecting_the_user_param(outbound)) as http_client: + with pytest.raises(litellm.BadRequestError, match="extra fields not permitted"): + await _acompletion_with_a_user_param(http_client) + + assert outbound.get_nowait()["user"] == "end-user-1" + assert outbound.empty() + + +@pytest.mark.asyncio +async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(monkeypatch: pytest.MonkeyPatch): + loop: Final = _AnsweringChatLoop() + monkeypatch.setattr(litellm, "callbacks", [_RaisingAgenticGate(), loop]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + response = await _acompletion_through(http_client) + + assert loop.runs == 1 + assert response.choices[0].message.content == "after the search" + + +class _SearchAgainLoop(CustomLogger): + def __init__(self, client: AsyncOpenAI) -> None: + super().__init__() + self._client = client + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + searches: Final = [ + {"id": call.id, "name": call.function.name} + for call in response.choices[0].message.tool_calls or [] + if call.function.name == "litellm_web_search" + ] + return bool(searches), {"tool_calls": searches} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + optional_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[*messages, {"role": "user", "content": "here are the search results"}], + kwargs={"client": self._client}, + ), + ) + + +@pytest.mark.asyncio +async def test_acompletion_ends_the_turn_at_max_agentic_loops_without_the_internal_search_call( + monkeypatch: pytest.MonkeyPatch, +): + upstream_calls: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def always_search(request: httpx.Request) -> httpx.Response: + upstream_calls.put_nowait(request) + return _respond_with_a_search_tool_call(request) + + async with httpx.AsyncClient(transport=httpx.MockTransport(always_search)) as http_client: + client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) + monkeypatch.setattr(litellm, "callbacks", [_SearchAgainLoop(client)]) + response: Final = await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=client, + messages=[{"role": "user", "content": "search the web for the weather"}], + max_agentic_loops=1, + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + assert upstream_calls.qsize() == 2 + assert not response.choices[0].message.tool_calls + + _REASONING_TOOL_CALL_REPLY: Final = { "id": "chatcmpl-fake-stream", "object": "chat.completion", diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py new file mode 100644 index 00000000000..9bb5fb6d300 --- /dev/null +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -0,0 +1,951 @@ +import asyncio +import io +import json +import queue +import wave +from collections.abc import Callable, Iterator +from datetime import datetime +from typing import Final, NamedTuple +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, +) +from litellm.litellm_core_utils.redact_messages import REDACTED_BY_LITELLM +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, +) +from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import TranscriptionResponse +from litellm.utils import get_optional_params_transcription + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def test_gpt_transcribe_config_uses_native_parameters_and_json(): + config = OpenAIGPTTranscribeAudioTranscriptionConfig() + supported = config.get_supported_openai_params("gpt-transcribe") + assert supported == ["prompt", "response_format", "keywords", "languages", "stream"] + + audio_file = io.BytesIO(b"audio") + request = config.transform_audio_transcription_request( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True, "response_format": "json"}, + litellm_params={}, + ) + assert request.data["response_format"] == "json" + assert request.data["keywords"] == ["LiteLLM"] + assert request.data["languages"] == ["en", "fr"] + assert request.data["stream"] is True + + +def test_gpt_transcribe_optional_params_are_preserved(): + params = get_optional_params_transcription( + model="gpt-transcribe", + custom_llm_provider="openai", + keywords=["LiteLLM", "Realtime API"], + languages=["en", "fr"], + stream=True, + ) + assert params == { + "keywords": ["LiteLLM", "Realtime API"], + "languages": ["en", "fr"], + "stream": True, + } + + +def test_transcription_response_preserves_empty_languages(): + response = TranscriptionResponse(text="hello", languages=[]) + assert json.loads(response.model_dump_json())["languages"] == [] + + +def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + 'data: {"type":"transcript.text.done","text":"hello world",' + '"usage":{"type":"duration","seconds":2.5},"languages":[{"code":"en"}]}\n\n' + ), + ) + + logging_obj: Final = MagicMock() + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + stream: Final = client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1), litellm_exception_mapper({})) + received: Final = tuple(wrapped) + wrapped.close() + + assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done") + logging_obj.success_handler.assert_called_once() + logged_response: Final = logging_obj.success_handler.call_args.args[0] + assert logged_response.text == "hello world" + assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + assert json.loads(logged_response.model_dump_json())["languages"] == [{"code": "en"}] + logging_obj.failure_handler.assert_not_called() + + +@pytest.mark.asyncio +async def test_openai_handler_returns_native_typed_stream(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + assert b'name="stream"' in body + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "languages": [], + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + handler = OpenAIAudioTranscription() + result = handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + logging_obj=logging_obj, + api_key="sk-test", + api_base="https://example.com/v1", + client=openai_client, + atranscription=True, + provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(), + ) + stream = await result + assert isinstance(stream, AsyncStream) + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now(), litellm_exception_mapper({})) + received = [event async for event in wrapped_stream] + await wrapped_stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + assert received[-1].languages == [] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response.text == "hello world" + assert logged_response.languages == () + + +@pytest.mark.asyncio +async def test_atranscription_stream_preserves_duration_for_callback_cost(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-transcribe", messages=[], stream=True, call_type="atranscription", + start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test", + ) + audio_file = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + + stream = await litellm.atranscription( + model="openai/gpt-transcribe", + file=audio_file, + stream=True, + client=openai_client, + litellm_logging_obj=logging_obj, + ) + received = [event async for event in stream] + await stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + logged_response = logging_obj.model_call_details["async_complete_streaming_response"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075) + assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075) + assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + + +@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) +def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): + litellm.transcription( + model=model, + file=audio_file, + response_format="verbose_json", + api_key="sk-test", + ) + + +@pytest.mark.parametrize("drop_globally", (True, False)) +def test_gpt_transcribe_drops_whisper_response_formats_under_drop_params( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_globally: bool +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + monkeypatch.setattr(litellm, "drop_params", drop_globally) + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + response: Final = litellm.transcription( + model="gpt-transcribe", + file=audio_file, + response_format="verbose_json", + client=client, + **({} if drop_globally else {"drop_params": True}), + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b"verbose_json" not in sent_bodies[0] + + +def _openai_transcription_client(transport: httpx.MockTransport) -> OpenAI: + return OpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=httpx.Client(transport=transport)) + + +def _azure_transcription_client(transport: httpx.MockTransport) -> AzureOpenAI: + return AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=transport), + ) + + +@pytest.mark.parametrize( + ("model", "make_client"), + (("gpt-transcribe", _openai_transcription_client), ("azure/gpt-transcribe", _azure_transcription_client)), +) +def test_gpt_transcribe_sends_text_response_format_to_the_provider( + local_model_cost_map: None, model: str, make_client: Callable[[httpx.MockTransport], OpenAI | AzureOpenAI] +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, text="hello", headers={"content-type": "text/plain"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with make_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model=model, + file=audio_file, + response_format="text", + api_key="sk-test", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="response_format"\r\n\r\ntext\r\n' in sent_bodies[0] + + +def test_file_transcription_reaches_models_whose_cost_map_lists_other_endpoints( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + sent_requests: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock upstreams received + + def send_gemini_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "interaction-1", + "status": "completed", + "steps": [ + {"type": "model_generation", "content": [{"type": "text", "text": "hello", "annotations": []}]} + ], + }, + ) + + def send_whisper_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response(200, json={"text": "hello"}) + + gemini_response: Final = litellm.transcription( + model="gemini/gemini-2.5-flash", + file=_one_second_wav(), + api_key="gemini-test-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send_gemini_response))), + ) + monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=httpx.MockTransport(send_whisper_response))) + whisper_response: Final = litellm.transcription( + model="openai/gpt-realtime-whisper", + file=_one_second_wav(), + api_key="self-hosted-whisper-key", + api_base="https://whisper.self-hosted.example/v1", + ) + + assert gemini_response.text == "hello" + assert whisper_response.text == "hello" + assert tuple(str(request.url.host) for request in sent_requests) == ( + "generativelanguage.googleapis.com", + "whisper.self-hosted.example", + ) + + +def test_azure_gpt_transcribe_uses_deployment_scoped_route(): + def send_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ( + "https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions" + f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}" + ) + return httpx.Response( + 200, + json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(send_response)) + client = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert response.languages is not None + assert [language.code for language in response.languages] == ["en"] + client.close() + + +@pytest.mark.asyncio +async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + return httpx.Response(200, json={"text": "hello"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncAzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version="2025-04-01-preview", + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = await AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"]}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version="2025-04-01-preview", + client=client, + atranscription=True, + ) + + assert response.text == "hello" + await client.close() + + +def _one_second_wav() -> io.BytesIO: + audio_file: Final = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + audio_file.seek(0) + return audio_file + + +class _LoggedSpend(NamedTuple): + cost: float | None + usage: dict[str, object] | None + + +def _logged_spend(kwargs: dict[str, object]) -> _LoggedSpend: + payload: Final = kwargs["standard_logging_object"] + return _LoggedSpend(payload["response_cost"], payload["metadata"]["usage_object"]) + + +class _SuccessSpendRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.async_spends: Final[asyncio.Queue[_LoggedSpend]] = asyncio.Queue() + self.sync_spends: Final[queue.Queue[_LoggedSpend]] = queue.Queue() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + await self.async_spends.put(_logged_spend(kwargs)) + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.sync_spends.put(_logged_spend(kwargs)) + + +@pytest.mark.asyncio +async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=False) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/whisper-1", file=_one_second_wav(), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + ignored_stream: Final = await litellm.atranscription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + ignored_stream_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend + + +def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=False) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + azure_params: Final = { + "client": client, + "api_key": "azure-test-key", + "api_base": "https://example.openai.azure.com", + "api_version": litellm.AZURE_DEFAULT_API_VERSION, + } + litellm.transcription(model="azure/whisper-1", file=_one_second_wav(), **azure_params) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) + ignored_stream: Final = litellm.transcription( + model="azure/whisper-1", file=_one_second_wav(), stream=True, **azure_params + ) + ignored_stream_spend: Final = recorder.sync_spends.get(timeout=5) + client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend + + +def _duration_priced_transcript(streamed: bool) -> httpx.Response: + usage: Final = {"type": "duration", "seconds": 2} + if not streamed: + return httpx.Response(200, json={"text": "hello world", "usage": usage}) + events: Final = ( + {"type": "transcript.text.delta", "delta": "hello world"}, + {"type": "transcript.text.done", "text": "hello world", "usage": usage}, + ) + return httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in events).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +def test_whisper_stream_request_answered_with_json_keeps_the_unstreamed_response_shape( + local_model_cost_map: None, +) -> None: + verbose_transcript: Final = { + "task": "transcribe", + "language": "english", + "duration": 1.0, + "text": "hello world", + "usage": {"type": "duration", "seconds": 1}, + } + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=verbose_transcript)) + ) as client: + unstreamed: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + ignored_stream: Final = litellm.transcription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + + assert isinstance(ignored_stream, TranscriptionResponse) + assert json.loads(ignored_stream.model_dump_json()) == json.loads(unstreamed.model_dump_json()) + + +@pytest.mark.parametrize("requested_stream", ("true", "True", "1", "yes", "on")) +def test_transcription_stream_requested_as_a_form_string_streams_like_a_bool( + local_model_cost_map: None, requested_stream: str +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=_asks_for_stream(sent_bodies[-1])) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=requested_stream, client=client + ) + events: Final = [event.type for event in stream] + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert len(sent_bodies) == 1 + + +def test_transcription_stream_that_is_not_a_boolean_answers_400_without_calling_the_upstream( + local_model_cost_map: None, +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + with pytest.raises(litellm.BadRequestError) as raised: + litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", client=client) + + assert raised.value.status_code == 400 + assert sent_bodies == [] + + +def test_transcription_with_an_empty_stream_form_value_runs_unstreamed_without_a_stream_field( + local_model_cost_map: None, +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream="", client=client + ) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "hello world" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def test_transcription_stream_that_is_not_a_boolean_is_dropped_under_drop_params(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", drop_params=True, client=client + ) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "hello world" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def _asks_for_stream(request_body: bytes) -> bool: + return b'name="stream"\r\n\r\ntrue' in request_body + + +@pytest.mark.asyncio +async def test_atranscription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend + + +class _TranscriptRedactor(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +@pytest.mark.asyncio +async def test_atranscription_stream_under_a_post_call_guardrail_streams_the_guarded_transcript( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", + file=_one_second_wav(), + stream=True, + client=client, + guardrails=["transcript-redactor"], + ) + events: Final = [(event.type, event.model_dump(exclude={"type"}, exclude_unset=True)) async for event in stream] + await client.close() + + assert events == [ + ("transcript.text.delta", {"delta": "[REDACTED]"}), + ("transcript.text.done", {"text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}), + ] + + +class _StreamedTranscriptRecorder(CustomLogger): + def __init__(self, message_logging: bool) -> None: + super().__init__(message_logging=message_logging) + self.logged: Final[asyncio.Queue[frozenset[object]]] = asyncio.Queue() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + logged_responses: Final = ( + response_obj, + *(kwargs.get(key) for key in ("complete_streaming_response", "async_complete_streaming_response")), + ) + await self.logged.put(frozenset(response.text for response in logged_responses if response is not None)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("opt_out", ("global", "callback")) +async def test_atranscription_stream_keeps_the_transcript_out_of_callbacks_that_turned_off_message_logging( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, opt_out: str +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + recorder: Final = _StreamedTranscriptRecorder(message_logging=opt_out != "callback") + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "turn_off_message_logging", opt_out == "global") + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + streamed_text: Final = [event.delta async for event in stream if event.type == "transcript.text.delta"] + logged_texts: Final = await asyncio.wait_for(recorder.logged.get(), timeout=5) + await client.close() + + assert streamed_text == ["hello world"] + assert logged_texts == {REDACTED_BY_LITELLM} + + +def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(request.read())) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type for event in stream] + streamed_spend: Final = recorder.sync_spends.get(timeout=5) + client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend + + +class _LoggedResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.responses: Final[queue.Queue[object]] = queue.Queue() + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.responses.put(kwargs["standard_logging_object"]["response"]) + + +@pytest.mark.parametrize( + "provider_body", + ({"text": "hello"}, {"text": "hello", "languages": ["en"]}, {"text": "hello", "languages": None}), +) +def test_transcription_without_parsed_languages_has_no_languages_key( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object] +) -> None: + recorder: Final = _LoggedResponseRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + logged_response: Final = recorder.responses.get(timeout=5) + + assert response.text == "hello" + assert "languages" not in json.loads(response.model_dump_json()) + assert isinstance(logged_response, dict) + assert "languages" not in logged_response + + +def test_unstreamed_transcription_sends_no_stream_field(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello"}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def test_transcription_forwards_both_language_and_languages(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), language="en", languages=["fr"], client=client + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="language"\r\n\r\nen\r\n' in sent_bodies[0] + assert b'name="languages[]"\r\n\r\nfr\r\n' in sent_bodies[0] + + +def _gpt_transcribe_cost_per_second() -> float: + rate: Final = litellm.model_cost["gpt-transcribe"]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +async def _spend_for_one_second_wav( + monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object], router: litellm.Router | None +) -> _LoggedSpend: + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ), + ) + if router is None: + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + else: + await router.atranscription(model="my-transcribe", file=_one_second_wav(), client=client) + spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + return spend + + +@pytest.mark.asyncio +async def test_custom_named_gpt_transcribe_deployment_bills_the_measured_upload_over_reported_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-transcribe", + "litellm_params": {"model": "openai/my-transcribe", "api_key": "sk-test"}, + "model_info": {"base_model": "gpt-transcribe"}, + } + ] + ) + + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 2}}, router + ) + + assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_gpt_transcribe_bills_the_file_length_when_the_provider_reports_zero_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 0}}, None + ) + + assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_m4a_transcription_stream_that_ends_early_logs_one_spend_row_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + if not _asks_for_stream(await request.aread()): + return httpx.Response(200, json={"text": "hello"}) + return httpx.Response( + 200, + content=f"data: {json.dumps({'type': 'transcript.text.delta', 'delta': 'hello'})}\n\n".encode(), + headers={"content-type": "text/event-stream"}, + ) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await asyncio.sleep(0.2) + await client.close() + + assert events == ["transcript.text.delta"] + assert streamed_spend.cost == unstreamed_spend.cost + assert recorder.async_spends.empty() diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index 703fa13cbc9..61798b49be0 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type. from unittest.mock import patch +import pytest + +import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing: assert result.usage.seconds == 295.8 def test_usage_duration_object_accepts_float_seconds(self): - assert ( - TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds - == 295.8 - ) + assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8 class TestTranscriptionDurationNotInResponseBody: @@ -126,6 +126,61 @@ class TestTranscriptionDurationNotInResponseBody: class TestCostCalculatorReadsDurationFromHiddenParams: """The cost calculator should read duration from _hidden_params via completion_cost().""" + @patch( # test-quality-ok: isolates duration selection from the model pricing table + "litellm.cost_calculator.openai_cost_per_second" + ) + def test_completion_cost_prefers_measured_duration_over_provider_usage_duration(self, mock_cost_fn): + mock_cost_fn.return_value = (0.001, 0.0) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = { + "audio_transcription_duration": 17.5, + "custom_llm_provider": "openai", + } + + completion_cost( + completion_response=response, + model="gpt-transcribe", + call_type="atranscription", + ) + + mock_cost_fn.assert_called_once() + _, kwargs = mock_cost_fn.call_args + assert kwargs["duration"] == 17.5 + + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("gpt-transcribe", "openai"), + ("azure/gpt-transcribe", "azure"), + ], + ) + def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode( + self, + monkeypatch, + model: str, + provider: str, + ): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setitem(litellm.model_cost, model, {**litellm.model_cost[model], "input_cost_per_second": 0.0125}) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = {"custom_llm_provider": provider} + + actual = completion_cost( + completion_response=response, + model=model, + call_type="atranscription", + custom_llm_provider=provider, + ) + + assert actual == pytest.approx(60.0 * 0.0125) + @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): """ diff --git a/tests/unit/llms/openai_like/test_dynamic_config.py b/tests/unit/llms/openai_like/test_dynamic_config.py index de70f98c3f1..f26f0821c4b 100644 --- a/tests/unit/llms/openai_like/test_dynamic_config.py +++ b/tests/unit/llms/openai_like/test_dynamic_config.py @@ -139,3 +139,24 @@ class TestForceStoreFalse: headers={}, ) assert params["store"] is True + + +class TestWebSearchRequestsFromToolUsage: + def test_counts_only_paid_search_tools(self): + payload = {"usage": {"tool_usage": {"web_search": 2, "image_search": 1, "code_interpreter": 3, "web_extractor": 1}}} + assert dynamic_config.web_search_requests_from_tool_usage(payload) == 3 + + @pytest.mark.parametrize( + "payload", + [ + {"usage": {"prompt_tokens": 1}}, + {"usage": {"tool_usage": {"code_interpreter": 2}}}, + {"usage": {"tool_usage": {"web_search": "2"}}}, + {"usage": {"tool_usage": {"web_search": True}}}, + {"usage": {"tool_usage": []}}, + {"usage": None}, + "not a mapping", + ], + ) + def test_anything_but_a_positive_search_count_is_none(self, payload): + assert dynamic_config.web_search_requests_from_tool_usage(payload) is None diff --git a/tests/unit/llms/openai_like/test_empiriolabs_provider.py b/tests/unit/llms/openai_like/test_empiriolabs_provider.py index a8acce5cc9d..58d8a2371d8 100644 --- a/tests/unit/llms/openai_like/test_empiriolabs_provider.py +++ b/tests/unit/llms/openai_like/test_empiriolabs_provider.py @@ -2,13 +2,19 @@ Unit tests for the EmpirioLabs OpenAI-like provider. """ +import json import os import sys +from typing import Final + +import pytest +import respx sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) ) +import litellm from litellm.llms.openai_like.dynamic_config import create_config_class from litellm.llms.openai_like.json_loader import JSONProviderRegistry @@ -61,3 +67,114 @@ def test_empiriolabs_complete_url_appends_endpoint(): stream=False, ) assert url == f"{EMPIRIOLABS_BASE_URL}/chat/completions" + + +def test_empiriolabs_chat_models_still_route_through_the_json_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EMPIRIOLABS_API_KEY", "environment-key") + monkeypatch.delenv("EMPIRIOLABS_API_BASE", raising=False) + + with respx.mock() as upstream: + route: Final = upstream.post(f"{EMPIRIOLABS_BASE_URL}/chat/completions").respond( + 200, + json={ + "id": "chatcmpl_empiriolabs", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "qwen3-7-plus", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from EmpirioLabs AI"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 5, "total_tokens": 9}, + }, + ) + response: Final = litellm.completion( + model="empiriolabs/qwen3-7-plus", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=16, + ) + + request: Final = route.calls.last.request + assert litellm.get_llm_provider("empiriolabs/qwen3-7-plus") == ( + "qwen3-7-plus", + "empiriolabs", + "environment-key", + EMPIRIOLABS_BASE_URL, + ) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer environment-key" + assert json.loads(request.content) == { + "model": "qwen3-7-plus", + "messages": [{"role": "user", "content": "Say hello"}], + "max_tokens": 16, + } + assert response.choices[0].message.content == "Hello from EmpirioLabs AI" + + +EMBEDDING_RESPONSE: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-v4", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, +} + + +def test_empiriolabs_embeddings_post_to_v1_embeddings_with_the_bearer_key(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{EMPIRIOLABS_BASE_URL}/embeddings").respond(200, json=EMBEDDING_RESPONSE) + response: Final = litellm.embedding( + model="empiriolabs/text-embedding-v4", + input=["Say hello"], + api_key="empiriolabs-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer empiriolabs-test-key" + assert body["model"] == "text-embedding-v4" + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +SEARCHED_CHAT_RESPONSE: Final = { + "id": "chatcmpl_empiriolabs_search", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "qwen3-7-plus", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Two searched answers"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 4, + "completion_tokens": 5, + "total_tokens": 9, + "tool_usage": {"web_search": 2, "code_interpreter": 1}, + }, +} + + +def test_empiriolabs_web_search_calls_in_tool_usage_are_billed_per_call(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + upstream.post(f"{EMPIRIOLABS_BASE_URL}/chat/completions").respond(200, json=SEARCHED_CHAT_RESPONSE) + response: Final = litellm.completion( + model="empiriolabs/qwen3-7-plus", + messages=[{"role": "user", "content": "Search twice"}], + api_key="empiriolabs-test-key", + ) + + row: Final = litellm.model_cost["empiriolabs/qwen3-7-plus"] + token_cost: Final = 4 * row["input_cost_per_token"] + 5 * row["output_cost_per_token"] + per_search: Final = row["search_context_cost_per_query"]["search_context_size_medium"] + assert response.usage.server_tool_use.web_search_requests == 2 + assert litellm.completion_cost( + completion_response=response, model="empiriolabs/qwen3-7-plus", custom_llm_provider="empiriolabs" + ) == pytest.approx(token_cost + 2 * per_search) diff --git a/tests/unit/llms/openai_like/test_wally_provider.py b/tests/unit/llms/openai_like/test_wally_provider.py new file mode 100644 index 00000000000..ac730b5760f --- /dev/null +++ b/tests/unit/llms/openai_like/test_wally_provider.py @@ -0,0 +1,269 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +_DEFAULT_API_BASE: Final = "https://inference.runanywhere.ai/v1" +_CHAT_COMPLETIONS_URL: Final = f"{_DEFAULT_API_BASE}/chat/completions" +_CHAT_COMPLETION: Final = { + "id": "chatcmpl_wally", + "object": "chat.completion", + "created": 1_790_000_000, + "model": "glm-5.3-flash", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Wally", "reasoning_content": "Say hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} +_EXPECTED_ENDPOINTS: Final = { + "chat_completions": True, + "messages": True, + "responses": True, + "embeddings": False, + "image_generations": False, + "audio_transcriptions": False, + "audio_speech": False, + "moderations": False, + "batches": False, + "rerank": False, +} +WALLY_MODELS: Final = tuple(sorted(name for name in litellm.model_cost if name.startswith("wally/"))) + + +def test_wally_is_a_registered_provider(): + assert litellm.LlmProviders.WALLY.value == "wally" + assert "wally" in litellm.provider_list + assert "wally" in litellm.constants.openai_compatible_providers + assert _DEFAULT_API_BASE in litellm.constants.openai_compatible_endpoints + + +def test_wally_provider_resolution(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("WALLY_API_KEY", "wally-test-key") + monkeypatch.delenv("WALLY_API_BASE", raising=False) + + model, provider, api_key, api_base = get_llm_provider( + model="wally/glm-5.3-flash", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "glm-5.3-flash" + assert provider == "wally" + assert api_key == "wally-test-key" + assert api_base == _DEFAULT_API_BASE + + +def test_wally_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("WALLY_API_KEY", "wally-env-key") + monkeypatch.setenv("WALLY_API_BASE", "https://wally.env.example/v1") + + _, provider, api_key, api_base = get_llm_provider( + model="wally/glm-5.3-flash", + custom_llm_provider=None, + api_base="https://wally.internal.example/v1", + api_key="wally-explicit-key", + ) + + assert provider == "wally" + assert api_key == "wally-explicit-key" + assert api_base == "https://wally.internal.example/v1" + + +def test_wally_api_base_env_overrides_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("WALLY_API_KEY", "wally-env-key") + monkeypatch.setenv("WALLY_API_BASE", "https://wally.env.example/v1") + + _, provider, _, api_base = get_llm_provider( + model="wally/glm-5.3-flash", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert provider == "wally" + assert api_base == "https://wally.env.example/v1" + + +def test_wally_api_base_autodetects_provider(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("WALLY_API_KEY", "wally-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model="glm-5.3-flash", + custom_llm_provider=None, + api_base=_DEFAULT_API_BASE, + api_key=None, + ) + + assert model == "glm-5.3-flash" + assert provider == "wally" + assert api_key == "wally-env-key" + assert api_base == _DEFAULT_API_BASE + + +@pytest.mark.parametrize("model", WALLY_MODELS) +def test_wally_model_cost_and_capabilities(model: str): + from litellm.cost_calculator import cost_per_token + from litellm.utils import ( + supports_function_calling, + supports_prompt_caching, + supports_reasoning, + supports_response_schema, + supports_tool_choice, + supports_vision, + ) + + entry: Final = litellm.model_cost[model] + prompt_cost, completion_cost = cost_per_token( + model=model, + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + custom_llm_provider="wally", + ) + cached_prompt_cost, _ = cost_per_token( + model=model, + prompt_tokens=1_000_000, + completion_tokens=0, + cache_read_input_tokens=1_000_000, + custom_llm_provider="wally", + ) + model_info: Final = litellm.get_model_info(model) + + assert prompt_cost == pytest.approx(entry["input_cost_per_token"] * 1_000_000) + assert completion_cost == pytest.approx(entry["output_cost_per_token"] * 1_000_000) + assert cached_prompt_cost == pytest.approx(entry["cache_read_input_token_cost"] * 1_000_000) + assert model_info["max_tokens"] == entry["max_output_tokens"] + assert model_info["litellm_provider"] == "wally" + assert supports_function_calling(model) is entry["supports_function_calling"] + assert supports_tool_choice(model) is entry["supports_tool_choice"] + assert supports_reasoning(model) is entry["supports_reasoning"] + assert supports_prompt_caching(model) is entry["supports_prompt_caching"] + assert supports_response_schema(model) is entry["supports_response_schema"] + assert supports_vision(model) is ("image" in entry["supported_modalities"]) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_wally_wildcard_expands_to_every_wally_row(): + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + assert set(get_known_models_from_wildcard("wally/*")) == set(WALLY_MODELS) + + +def test_wally_model_missing_from_cost_map_uses_provider_defaults(): + from litellm.utils import supports_function_calling, supports_reasoning + + assert supports_function_calling("wally/a-model-not-in-the-cost-map") is True + assert supports_reasoning("wally/a-model-not-in-the-cost-map") is True + + +def test_wally_backup_registry_mirrors_cost_map(): + package_root: Final = Path(litellm.__file__).parent + cost_map: Final = json.loads((package_root.parent / "model_prices_and_context_window.json").read_text()) + backup: Final = json.loads((package_root / "model_prices_and_context_window_backup.json").read_text()) + wally_entries: Final = {name: entry for name, entry in cost_map.items() if name.startswith("wally/")} + + assert wally_entries + assert tuple(sorted(wally_entries)) == WALLY_MODELS + assert wally_entries == {name: backup[name] for name in wally_entries} + + +def test_wally_is_available_in_add_model_form(): + fields_path: Final = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + providers: Final = json.loads(fields_path.read_text()) + wally: Final = next(provider for provider in providers if provider["litellm_provider"] == "wally") + + assert wally["provider"] == "WALLY" + assert wally["provider_display_name"] == "Wally" + assert wally["default_model_placeholder"] in WALLY_MODELS + assert {field["key"]: field["required"] for field in wally["credential_fields"]} == { + "api_base": False, + "api_key": True, + } + + +def test_wally_supported_endpoints(): + backup_path: Final = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" + root_path: Final = Path(litellm.__file__).parent.parent / "provider_endpoints_support.json" + + backup: Final = json.loads(backup_path.read_text())["providers"]["wally"]["endpoints"] + root: Final = json.loads(root_path.read_text())["providers"]["wally"]["endpoints"] + + assert backup == root + assert {endpoint: root[endpoint] for endpoint in _EXPECTED_ENDPOINTS} == _EXPECTED_ENDPOINTS + + +@pytest.mark.parametrize("reasoning_effort", ["none", "max"]) +def test_wally_chat_completion_request(reasoning_effort: str): + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_COMPLETIONS_URL).respond(200, json=_CHAT_COMPLETION) + response: Final = litellm.completion( + model="wally/glm-5.3-flash", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=64, + reasoning_effort=reasoning_effort, + api_key="wally-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == _CHAT_COMPLETIONS_URL + assert request.headers["authorization"] == "Bearer wally-test-key" + assert body["model"] == "glm-5.3-flash" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert body["max_completion_tokens"] == 64 + assert body["reasoning_effort"] == reasoning_effort + assert response.choices[0].message.content == "Hello from Wally" + assert response.choices[0].message.reasoning_content == "Say hi" + + +def test_wally_responses_request_is_bridged_to_chat_completions(): + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_COMPLETIONS_URL).respond(200, json=_CHAT_COMPLETION) + response: Final = litellm.responses( + model="wally/glm-5.3-flash", + input="Say hello", + api_key="wally-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer wally-test-key" + assert body["model"] == "glm-5.3-flash" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + message_items: Final = [item for item in response.output if item.type == "message"] + assert message_items[0].content[0].text == "Hello from Wally" + + +@pytest.mark.asyncio +async def test_wally_anthropic_messages_request_is_bridged_to_chat_completions(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_COMPLETIONS_URL).respond(200, json=_CHAT_COMPLETION) + response: Final = await litellm.anthropic.messages.acreate( + model="wally/glm-5.3-flash", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="wally-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer wally-test-key" + assert body["model"] == "glm-5.3-flash" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert body["max_tokens"] == 32 + assert any(block.get("text") == "Hello from Wally" for block in response["content"]) diff --git a/tests/unit/llms/opencode/chat/test_transformation.py b/tests/unit/llms/opencode/chat/test_transformation.py index 91361a3d81d..08c80db294c 100644 --- a/tests/unit/llms/opencode/chat/test_transformation.py +++ b/tests/unit/llms/opencode/chat/test_transformation.py @@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.opencode.chat.transformation import OpenCodeGoMessagesChatConfig from litellm.llms.opencode.common_utils import go_headers @@ -591,3 +592,22 @@ def test_stream_chunk_builder_reads_list_cost_from_dict_chunks(): assert assembled is not None assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"] + + +@pytest.mark.parametrize(("model", "cache_marks"), [("qwen3.8-flash", 0), ("minimax-m3", 0), ("claude-haiku-5-5", 4)]) +def test_go_messages_route_carries_prompt_cache_breakpoint_to_cache_control_only_for_claude(model: str, cache_marks: int): + marked_notes: Final = [ + {"type": "text", "text": f"Note {n}", "prompt_cache_breakpoint": {"mode": "explicit"}} for n in range(4) + ] + + request: Final = OpenCodeGoMessagesChatConfig().transform_request( + model=model, + messages=[{"role": "user", "content": [*marked_notes, {"type": "text", "text": "Answer in one line."}]}], + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + wire: Final = json.dumps(request) + assert wire.count('"cache_control"') == cache_marks + assert "prompt_cache_breakpoint" not in wire diff --git a/tests/unit/llms/opencode/test_transformation.py b/tests/unit/llms/opencode/test_transformation.py index 96203c23122..6f0fcc2f6af 100644 --- a/tests/unit/llms/opencode/test_transformation.py +++ b/tests/unit/llms/opencode/test_transformation.py @@ -4,7 +4,7 @@ import pytest from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header -from litellm.llms.opencode.transformation import OpenCodeProviderAdapter +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @pytest.mark.parametrize( @@ -22,7 +22,7 @@ def test_opencode_defaults_affinity_header_to_the_callers_session(api_base: str) headers: Final = {"Authorization": "Bearer test-key"} params: Final = {"metadata": {"session_id": "conversation-123"}} assert add_provider_affinity_header( - headers, params, OpenCodeProviderAdapter().get_provider_affinity_header(api_base) + headers, params, get_provider_affinity_header(api_base) ) == { **headers, "x-opencode-session": "conversation-123", @@ -50,7 +50,7 @@ def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: add_provider_affinity_header( headers, {"litellm_session_id": "conversation-123"}, - OpenCodeProviderAdapter().get_provider_affinity_header(api_base), + get_provider_affinity_header(api_base), ) == headers ) @@ -71,7 +71,7 @@ def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: def test_opencode_does_not_invent_a_conversation_id(params: dict[str, object]) -> None: assert ( add_provider_affinity_header( - {}, params, OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1") + {}, params, get_provider_affinity_header("https://opencode.ai/zen/go/v1") ) == {} ) @@ -81,10 +81,10 @@ def test_opencode_preserves_explicit_header_and_affinity_configuration() -> None assert add_provider_affinity_header( {"X-OpenCode-Session": "explicit"}, {"litellm_session_id": "conversation-123"}, - OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + get_provider_affinity_header("https://opencode.ai/zen/go/v1"), ) == {"X-OpenCode-Session": "explicit"} assert add_provider_affinity_header( {}, {"litellm_session_id": "conversation-123", "provider_affinity_header": "x-custom-session"}, - OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + get_provider_affinity_header("https://opencode.ai/zen/go/v1"), ) == {"x-custom-session": "conversation-123"} diff --git a/tests/unit/llms/parallel_ai/decisions/__init__.py b/tests/unit/llms/parallel_ai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py new file mode 100644 index 00000000000..9cd404f61f4 --- /dev/null +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -0,0 +1,423 @@ +import json +from collections.abc import Mapping +from dataclasses import replace +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsResponseValidationError, + ir_to_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.llms.parallel_ai.decisions.transformation import ParallelAIDecisionsConfig +from litellm.types.decisions import ( + DecisionsIRRefusal, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, Rules, function_setup + +QUESTIONS: Final = { + "available": {"type": "noul", "instructions": "Is this product available?"}, + "tier": {"type": "choice", "instructions": "Which tier is offered?", "criteria": {"free": "Free", "paid": "Paid"}}, + "fit": {"type": "score", "instructions": "How well does it fit?", "criteria": ["Poor", "Good", "Excellent"]}, + "unknown": {"type": "noul", "instructions": "Is there enough evidence?"}, +} +REQUEST: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate({"state": {"product": "Example"}, "questions": QUESTIONS}) +) +PAYLOAD: Final = { + "mode": "fast", + "answers": { + "available": {"type": "noul", "noul": 0.9}, + "tier": {"type": "choice", "choice": "paid", "probabilities": {"free": 0.2, "paid": 0.8}}, + "fit": { + "type": "score", + "score": 1.7, + "legend": {"0": "Poor", "1": "Good", "2": "Excellent"}, + "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}, + }, + "unknown": {"type": "refusal"}, + }, + "usage": {"input_tokens": 4212, "output_tokens": 0}, + "citations": [{"url": "https://example.com/product", "title": "Product", "excerpts": ["Available now"]}], + "decision_id": "decisions_example", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +async def test_sdk_sends_parallel_wire_format_and_preserves_evidence( + use_async: bool, surface: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/decision", "api_key": "test-key", "timeout": 5, "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + response: Final = await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + sent: Final = route.calls.last.request + body: Final = json.loads(sent.content) + assert body["mode"] == "fast" + assert "model" not in body + assert body["state"] == "Example" + assert sent.headers["x-api-key"] == "test-key" + assert "authorization" not in sent.headers + assert response.model == "decision" + assert response.model_dump()["citations"] == PAYLOAD["citations"] + assert response.model_dump()["decision_id"] == PAYLOAD["decision_id"] + await GLOBAL_LOGGING_WORKER.clear_queue() + + +def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders.PARALLEL_AI) + assert config is not None + assert config.transform_decisions_request("decision", REQUEST, "parallel_ai") == { + "mode": "fast", + "state": {"product": "Example"}, + "questions": QUESTIONS, + } + assert config.get_complete_url("https://gateway.example/v1/", "decision") == "https://gateway.example/v1/decisions" + + +@pytest.mark.parametrize("suffix", ["", "/", "/v1", "/v1/", "/v1/decisions", "/v1/decisions/", "/v1/search", "/v1/search/"]) +def test_base_carrying_a_parallel_route_reaches_the_decisions_endpoint(suffix: str) -> None: + complete_url: Final = ParallelAIDecisionsConfig().get_complete_url(f"https://api.parallel.ai{suffix}", "decision") + assert complete_url == "https://api.parallel.ai/v1/decisions" + + +def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None: + headers: Final = ParallelAIDecisionsConfig().validate_environment( + {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"}, + "decision", + "test-key", + {}, + ) + assert headers == {"x-api-key": "test-key", "Content-Type": "application/json", "X-Trace": "test"} + + +def test_openai_predicate_is_translated_to_a_named_noul_question() -> None: + request: Final = openai_request_to_ir( + OpenAIDecisionRequestBody.model_validate( + { + "input": "Example product", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + ) + assert ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai") == { + "mode": "fast", + "state": "Example product", + "questions": {"available": {"type": "noul", "instructions": "Available?"}}, + } + + +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_all_answers_usage_and_evidence_survive_response_translation(surface: str) -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + assert isinstance(response.answers[3], DecisionsIRRefusal) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + ) + result: Final = formatted.model_dump(mode="json") + assert result["citations"] == PAYLOAD["citations"] + assert result["decision_id"] == PAYLOAD["decision_id"] + assert result["mode"] == "fast" + assert result["usage"]["input_tokens"] == 4212 + assert result["usage"]["output_tokens"] == 0 + if surface == "systemone": + assert result["answers"]["available"]["noul"] == 0.9 + assert result["answers"]["tier"]["choice"] == "paid" + assert result["answers"]["fit"]["score"] == 1.7 + assert "unknown" not in result["answers"] + else: + assert result["answers"][3]["type"] == "refusal" + + +def test_rejects_more_than_one_hundred_questions() -> None: + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate( + { + "state": "Example", + "questions": {str(index): QUESTIONS["available"] for index in range(101)}, + } + ) + ) + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("model", ["fast", "unknown"]) +def test_rejects_unregistered_models_instead_of_forwarding_them_as_modes(model: str) -> None: + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request(model, REQUEST, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("confidence", [None, 0.8]) +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: float | None, surface: str) -> None: + payload: Final = { + **PAYLOAD, + "answers": {name: {**answer, "confidence": confidence} for name, answer in PAYLOAD["answers"].items()}, + } + response: Final = ParallelAIDecisionsConfig().parse_response(payload, REQUEST) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + ) + answers: Final = formatted.model_dump(mode="json")["answers"] + values: Final = answers.values() if isinstance(answers, dict) else answers + assert [answer["confidence"] for answer in values if answer["type"] in {"choice", "score"}] == [confidence] * 2 + + +def test_provider_extras_cannot_replace_normalized_response_fields() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}}) + formatted: Final = ir_to_openai_response( + with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + assert formatted.model == "decision" + assert len(formatted.answers) == 4 + assert formatted.usage.input_tokens == 4212 + assert formatted.model_dump()["citations"] == PAYLOAD["citations"] + + +def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST) + formatted: Final = ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + assert "internal_trace_id" not in formatted.model_dump() + + +@pytest.mark.parametrize("tokens", [0, 100, 32000]) +@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"]) +def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None: + rate: Final = litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] + cost: Final = litellm.cost_per_token( + model="parallel_ai/decision", + prompt_tokens=tokens, + completion_tokens=0, + call_type=call_type, + ) + assert cost == (rate, 0.0) + + +@pytest.mark.asyncio +async def test_deployment_query_price_replaces_the_catalog_price( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + price: Final = 25 * litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "webjev", + "litellm_params": { + "model": "parallel_ai/decision", + "api_key": "test-key", + "input_cost_per_query": price, + }, + } + ] + ) + fields: Final = { + "model": "webjev", + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + "litellm_call_id": "query-price-call", + } + logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields) + response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj) + assert response._hidden_params["response_cost"] == pytest.approx(price) + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.asyncio +async def test_query_price_on_another_provider_leaves_its_token_billing_alone( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + input_rate: Final = 1e-06 + tokens: Final = 1000 + model: Final = "fireworks_ai/accounts/example/models/token-billed" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_token": input_rate, "output_cost_per_token": 0.0} + | {"litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + respx_mock.post("https://api.fireworks.ai/inference/v1/systemone").respond( + json={ + "model": "token-billed", + "answers": {"available": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": tokens, "output_tokens": 0}, + } + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "token-billed", + "litellm_params": {"model": model, "api_key": "test-key", "input_cost_per_query": 0.025}, + } + ] + ) + fields: Final = { + "model": "token-billed", + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + "litellm_call_id": "token-billed-call", + } + logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields) + response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj) + assert response._hidden_params["response_cost"] == pytest.approx(tokens * input_rate) + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.parametrize( + "payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}] +) +def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[str, object]) -> None: + with pytest.raises(DecisionsResponseValidationError): + ParallelAIDecisionsConfig().transform_decisions_response( + "decision", "parallel_ai", httpx.Response(200, json=payload), REQUEST + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +@pytest.mark.parametrize("key_env", ["PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"]) +async def test_untrusted_base_cannot_receive_environment_key( + use_async: bool, surface: str, key_env: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + monkeypatch.setenv(key_env, "server-secret") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://attacker.example/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/decision", "api_base": "https://attacker.example", "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + + async def call() -> object: + return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + + with pytest.raises(litellm.BadRequestError, match="explicit api_key"): + await call() + assert not route.called + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.parametrize( + "api_base", + [ + "https://api.parallel.ai.attacker.example", + "https://api.parallel.ai@attacker.example", + "http://api.parallel.ai", + "https://api.parallel.ai:8443", + "https://attacker.example/api.parallel.ai", + "https://api.parallel.ai/other-tenant", + "https://api.parallel.ai/other-tenant/v1/search", + "https://attacker.example/v1/search", + "https://attacker.example/v1/decisions", + ], +) +def test_environment_key_rejects_untrusted_destination_variants(api_base: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + with pytest.raises(ValueError, match="explicit api_key"): + ParallelAIDecisionsConfig().resolve_connection(api_base, None) + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.parallel.ai", + "https://API.PARALLEL.AI:443/v1/", + "https://api.parallel.ai/v1/decisions", + "https://api.parallel.ai/v1/search", + ], +) +def test_default_destination_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://api.parallel.ai") + assert key == "server-secret" + + +@pytest.mark.parametrize( + "api_base", [None, "https://gateway.example/parallel/v1/", "https://gateway.example/parallel/v1/search"] +) +def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://gateway.example/parallel") + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://gateway.example/parallel") + assert key == "server-secret" + + +@pytest.mark.parametrize("api_key", [None, "caller-key"]) +@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"]) +def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None: + with pytest.raises(litellm.BadRequestError): + litellm.decisions( + model="parallel_ai/decision", + api_base=api_base, + api_key=api_key, + input="Example", + questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}], + num_retries=0, + ) + + +def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == ( + "https://gateway.example", + "caller-key", + ) diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 5e40c2ebb60..4c253a10116 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -549,7 +549,7 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) @pytest.mark.asyncio diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 2eecb83e2df..03bc308a89d 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1785,6 +1785,289 @@ def test_vertex_ai_map_tools(): assert tools == new_tools +def test_vertex_ai_map_tools_declares_a_namespace_by_its_members() -> None: + namespace = { + "type": "namespace", + "name": "github", + "tools": [ + {"type": "function", "name": "create_issue", "parameters": {"type": "object", "properties": {}}}, + {"type": "function", "function": {"name": "close_issue"}}, + ], + } + + tools = VertexGeminiConfig().map_function(value=[namespace], optional_params={}) + + assert [declaration["name"] for declaration in tools[0]["function_declarations"]] == [ + "create_issue", + "close_issue", + ] + + +def test_vertex_ai_map_tools_declares_a_namespace_member_like_the_same_chat_tool() -> None: + parameters = { + "type": "object", + "properties": {"repo": {"$ref": "#/$defs/Repo"}, "label": {"type": ["string", "null"]}}, + "$defs": {"Repo": {"type": "string"}}, + } + function = {"name": "create_issue", "description": "Open an issue", "parameters": parameters} + namespace = {"type": "namespace", "name": "github", "tools": [{"type": "function", **function}]} + + from_namespace = VertexGeminiConfig().map_function(value=[namespace], optional_params={}) + from_chat_tool = VertexGeminiConfig().map_function( + value=[{"type": "function", "function": function}], optional_params={} + ) + + assert from_namespace == from_chat_tool + + +_GET_WEATHER = {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}} + + +_WEATHER_NAMESPACE = {"type": "namespace", "name": "weather", "tools": [_GET_WEATHER]} +_TOP_LEVEL_GET_WEATHER = {"type": "function", "function": {k: v for k, v in _GET_WEATHER.items() if k != "type"}} + + +@pytest.mark.parametrize( + ("tools", "declared"), + [ + ( + [{"type": "namespace", "name": "forecast", "tools": [_GET_WEATHER]}, _WEATHER_NAMESPACE], + ["forecast__get_weather", "weather__get_weather"], + ), + ([_TOP_LEVEL_GET_WEATHER, _WEATHER_NAMESPACE], ["get_weather", "weather__get_weather"]), + ([_WEATHER_NAMESPACE, _TOP_LEVEL_GET_WEATHER], ["weather__get_weather", "get_weather"]), + ], + ids=["namespaces_share_a_member", "top_level_first", "namespace_first"], +) +def test_vertex_ai_map_tools_declares_qualified_member_names_when_another_tool_declares_a_member_name( + tools: list[dict], declared: list[str] +) -> None: + mapped = VertexGeminiConfig().map_function(value=tools, optional_params={}) + + assert [declaration["name"] for declaration in mapped[0]["function_declarations"]] == declared + + +def test_vertex_ai_map_tools_declares_a_member_a_namespace_lists_twice_once_from_its_first_entry() -> None: + first = {**_GET_WEATHER, "description": "Current weather"} + repeat = {**_GET_WEATHER, "description": "Forecast"} + namespace = {"type": "namespace", "name": "weather", "tools": [first, repeat]} + + mapped = VertexGeminiConfig().map_function(value=[namespace], optional_params={}) + + declarations = mapped[0]["function_declarations"] + assert [(declaration["name"], declaration["description"]) for declaration in declarations] == [ + ("get_weather", "Current weather") + ] + + +@pytest.mark.parametrize( + ("tools", "declared"), + [ + ( + [ + _WEATHER_NAMESPACE, + _TOP_LEVEL_GET_WEATHER, + {"type": "function", "function": {"name": "weather__get_weather", "description": "Top level"}}, + ], + [("get_weather", None), ("weather__get_weather", "Top level")], + ), + ( + [ + {"type": "namespace", "name": "mcp__ops", "tools": [{"type": "function", "name": "exec"}]}, + {"type": "namespace", "name": "mcp", "tools": [{"type": "function", "name": "ops__exec"}]}, + {"type": "function", "function": {"name": "exec"}}, + {"type": "function", "function": {"name": "ops__exec"}}, + ], + [("mcp__ops__exec", None), ("exec", None), ("ops__exec", None)], + ), + ( + [_TOP_LEVEL_GET_WEATHER, _TOP_LEVEL_GET_WEATHER, _WEATHER_NAMESPACE], + [("get_weather", None), ("get_weather", None), ("weather__get_weather", None)], + ), + ], + ids=["top_level_function_has_the_qualified_name", "namespaces_build_one_qualified_name", "client_repeats_a_function"], +) +def test_vertex_ai_map_tools_never_repeats_a_generated_name( + tools: list[dict], declared: list[tuple[str, str | None]] +) -> None: + mapped = VertexGeminiConfig().map_function(value=tools, optional_params={}) + + declarations = mapped[0]["function_declarations"] + assert [(declaration["name"], declaration.get("description")) for declaration in declarations] == declared + + +def test_vertex_ai_map_openai_params_qualifies_members_by_the_namespace_top_level_name() -> None: + namespace = { + "type": "namespace", + "name": "safe", + "function": {"name": "lookup"}, + "tools": [{"type": "function", "name": "lookup"}], + } + + mapped = VertexGeminiConfig().map_openai_params( + non_default_params={ + "tools": [namespace], + "tool_choice": {"type": "function", "function": {"name": "lookup"}}, + }, + optional_params={}, + model="gemini-3.8-flash", + drop_params=False, + ) + + declared = [declaration["name"] for tool in mapped["tools"] for declaration in tool["function_declarations"]] + assert declared == ["safe__lookup"] + assert mapped["tool_choice"]["functionCallingConfig"]["allowed_function_names"] == ["safe__lookup"] + + +def test_vertex_ai_map_openai_params_declares_qualified_member_names_when_functions_repeat_a_member() -> None: + mapped = VertexGeminiConfig().map_openai_params( + non_default_params={ + "tools": [{"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]}], + "functions": [{"name": "lookup", "parameters": {"type": "object", "properties": {}}}], + }, + optional_params={}, + model="gemini-3.8-flash", + drop_params=False, + ) + + declared = [declaration["name"] for tool in mapped["tools"] for declaration in tool["function_declarations"]] + assert declared == ["mcp__weather__lookup", "lookup"] + + +def test_vertex_ai_map_tools_keeps_repeated_chat_tools_when_no_namespace_is_sent() -> None: + function = {k: v for k, v in _GET_WEATHER.items() if k != "type"} + + mapped = VertexGeminiConfig().map_function( + value=[{"type": "function", "function": function}, {"type": "function", "function": function}], + optional_params={}, + ) + + assert [declaration["name"] for declaration in mapped[0]["function_declarations"]] == ["get_weather", "get_weather"] + + +@pytest.mark.parametrize( + ("forced_name", "other_tools", "allowed_function_names"), + [ + ("github", [], ["create_issue", "close_issue"]), + ("close_issue", [], ["close_issue"]), + ("github", [{"type": "function", "function": {"name": "github"}}], ["github"]), + ( + "github", + [{"type": "function", "function": {"name": "close_issue"}}], + ["github__create_issue", "github__close_issue"], + ), + ( + "close_issue", + [{"type": "namespace", "name": "gitlab", "tools": [{"type": "function", "name": "close_issue"}]}], + ["github__close_issue", "gitlab__close_issue"], + ), + ( + "github", + [ + {"type": "function", "function": {"name": "close_issue"}}, + {"type": "function", "function": {"name": "github__close_issue"}}, + ], + ["github__create_issue"], + ), + ( + "close_issue", + [ + {"type": "namespace", "name": "gitlab", "tools": [{"type": "function", "name": "close_issue"}]}, + {"type": "function", "function": {"name": "github__close_issue"}}, + ], + ["gitlab__close_issue"], + ), + ( + "mcp", + [ + {"type": "namespace", "name": "mcp__ops", "tools": [{"type": "function", "name": "exec"}]}, + { + "type": "namespace", + "name": "mcp", + "tools": [{"type": "function", "name": "ops__exec"}, {"type": "function", "name": "status"}], + }, + {"type": "function", "function": {"name": "exec"}}, + {"type": "function", "function": {"name": "ops__exec"}}, + ], + ["mcp__status"], + ), + ], + ids=[ + "names_the_namespace", + "names_a_member", + "names_a_function_sharing_the_namespace_name", + "names_a_namespace_whose_member_a_function_repeats", + "names_a_member_two_namespaces_share", + "names_a_namespace_whose_generated_name_a_function_holds", + "names_a_member_whose_generated_name_a_function_holds", + "names_a_namespace_whose_generated_name_another_namespace_holds", + ], +) +def test_vertex_ai_tool_choice_forces_only_functions_gemini_is_sent( + forced_name: str, other_tools: list[dict[str, object]], allowed_function_names: list[str] +) -> None: + namespace = { + "type": "namespace", + "name": "github", + "tools": [{"type": "function", "name": "create_issue"}, {"type": "function", "name": "close_issue"}], + } + + mapped = VertexGeminiConfig().map_openai_params( + non_default_params={ + "tools": [namespace, *other_tools], + "tool_choice": {"type": "function", "function": {"name": forced_name}}, + }, + optional_params={}, + model="gemini-3.8-flash", + drop_params=False, + ) + + declared = [declaration["name"] for declaration in mapped["tools"][0]["function_declarations"]] + assert mapped["tool_choice"]["functionCallingConfig"]["allowed_function_names"] == allowed_function_names + assert set(allowed_function_names) <= set(declared) + + +@pytest.mark.parametrize( + ("forced_name", "tools"), + [ + ( + "weather", + [ + {"type": "namespace", "name": "weather", "tools": [{"type": "function", "name": "lookup"}]}, + {"type": "function", "function": {"name": "lookup"}}, + {"type": "function", "function": {"name": "weather__lookup"}}, + ], + ), + ( + "ops__exec", + [ + {"type": "namespace", "name": "ops", "tools": [{"type": "function", "name": "exec"}]}, + {"type": "namespace", "name": "ops__exec", "tools": [{"type": "function", "name": "status"}]}, + {"type": "function", "function": {"name": "exec"}}, + {"type": "function", "function": {"name": "status"}}, + {"type": "function", "function": {"name": "ops__exec__status"}}, + ], + ), + ], + ids=["its_members_lose_to_functions", "another_namespace_generates_its_name"], +) +def test_vertex_ai_tool_choice_naming_a_namespace_gemini_gets_no_member_of( + forced_name: str, tools: list[dict[str, object]] +) -> None: + params = {"tools": tools, "tool_choice": {"type": "function", "function": {"name": forced_name}}} + + with pytest.raises(litellm.UnsupportedParamsError, match=f"tool_choice '{forced_name}'"): + VertexGeminiConfig().map_openai_params( + non_default_params=params, optional_params={}, model="gemini-3.8-flash", drop_params=False + ) + dropped = VertexGeminiConfig().map_openai_params( + non_default_params=params, optional_params={}, model="gemini-3.8-flash", drop_params=True + ) + + assert "tool_choice" not in dropped + assert dropped["tools"][0]["function_declarations"] + + def test_vertex_ai_map_tool_with_anyof(): """ Related issue: https://github.com/BerriAI/litellm/issues/11164 diff --git a/tests/unit/llms/vllm/test_vllm_common_utils.py b/tests/unit/llms/vllm/test_vllm_common_utils.py new file mode 100644 index 00000000000..aa8a4852273 --- /dev/null +++ b/tests/unit/llms/vllm/test_vllm_common_utils.py @@ -0,0 +1,27 @@ +from typing import Final + +import httpx +import pytest +import respx + +from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + +@respx.mock +def test_hosted_vllm_get_models_lists_the_server_ids_sending_the_key_as_a_bearer_token() -> None: + respx.get("http://vllm-sdk.test:8000/v1/models", headers={"Authorization": "Bearer vllm-sdk-key"}).mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}, {"id": "meta-llama/Llama-3.1-8B-Instruct"}]}) + ) + + listed: Final = HostedVLLMModelInfo().get_models(api_key="vllm-sdk-key", api_base="http://vllm-sdk.test:8000/v1") + + assert listed == ["qwen2.5", "meta-llama/Llama-3.1-8B-Instruct"] + + +def test_hosted_vllm_get_models_without_a_server_names_the_setting_that_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + + with pytest.raises(ValueError, match="HOSTED_VLLM_API_BASE"): + HostedVLLMModelInfo().get_models() diff --git a/tests/unit/mcp_client/test_mcp_client.py b/tests/unit/mcp_client/test_mcp_client.py index 43021d248a9..866a29b70e9 100644 --- a/tests/unit/mcp_client/test_mcp_client.py +++ b/tests/unit/mcp_client/test_mcp_client.py @@ -3,6 +3,7 @@ import base64 import importlib import json import os +import queue import sys from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager @@ -74,6 +75,7 @@ from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeade from litellm.types.mcp import MCPAuth, MCPStdioConfig, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer from tests._support.clock_loops import AutojumpClockLoop, ManualClockLoop +from tests._support.mcp_count_tool_upstream import count_tool_upstream, methods_received _JSONRPC_MESSAGE_ADAPTER: Final = TypeAdapter(JSONRPCMessage) @@ -5250,3 +5252,55 @@ async def test_notification_callback_keeps_stream_errors_separate_from_protocol_ await _receive_session_message(failure, stream_error=stream_error, on_notification=receiver) assert stream_error.result() is first receiver.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_skipping_client_never_asks_the_upstream_for_its_tools() -> None: + methods_seen: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + client: Final = _MockTransportClient( + count_tool_upstream({"n": 1}, methods_seen), + server_url="https://upstream.example/mcp", + transport_type=MCPTransport.http, + skip_tool_output_validation=True, + ) + + result: Final = await client.call_tool(CallToolRequestParams(name="count", arguments={})) + + assert isinstance(result, CallToolResult) + assert result.is_error is False + assert "tools/list" not in methods_received(methods_seen) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("skip_validation", "rejected"), [(False, True), (True, False)], ids=["validating", "skipping"] +) +async def test_invalid_structured_output_is_rejected_unless_validation_is_skipped( + skip_validation: bool, rejected: bool +) -> None: + client: Final = _MockTransportClient( + count_tool_upstream({"n": "not-an-integer"}, queue.SimpleQueue()), + server_url="https://upstream.example/mcp", + transport_type=MCPTransport.http, + skip_tool_output_validation=skip_validation, + ) + + result: Final = await client.call_tool(CallToolRequestParams(name="count", arguments={})) + + assert isinstance(result, CallToolResult) + assert result.is_error is rejected + + +@pytest.mark.asyncio +async def test_a_skipped_validation_passes_the_upstreams_result_through_unchanged() -> None: + client: Final = _MockTransportClient( + count_tool_upstream({"n": "not-an-integer"}, queue.SimpleQueue()), + server_url="https://upstream.example/mcp", + transport_type=MCPTransport.http, + skip_tool_output_validation=True, + ) + + result: Final = await client.call_tool(CallToolRequestParams(name="count", arguments={})) + + assert isinstance(result, CallToolResult) + assert result.structured_content == {"n": "not-an-integer"} diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index 2b682fda982..37dfdbf489c 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -364,6 +364,10 @@ async def test_unknown_invocation_target_leaves_billing_unset(monkeypatch: pytes ("/v1beta/models/gemini-model:generateContent", "POST", True), ("/v1/realtime", "GET", True), ("/v1/realtime", "POST", False), + ("/v1/realtime/translations", "GET", True), + ("/v1/realtime/translations", "POST", False), + ("/v1/realtime/translations/client_secrets", "POST", False), + ("/v1/realtime/translations/calls", "POST", False), ("/v1/realtime/client_secrets", "POST", False), ("/mcp/tools/call", "POST", True), ("/a2a/target/message/send", "POST", True), @@ -437,14 +441,21 @@ def test_managed_inference_ignores_unsupported_query_model(): ) -@pytest.mark.parametrize("route", ["/realtime", "/v1/realtime", "/openai/v1/realtime"]) +@pytest.mark.parametrize( + "route", + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + ), +) def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request with pytest.raises(HTTPException, match="explicit or configured model"): managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli") assert ( - managed_inference_request(route, {"model": "requested"}, {"completion_model": "allowed-default"}, "cli")[ + managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli", query_model="requested")[ "model" ] == "requested" @@ -465,6 +476,20 @@ def test_managed_live_upgrade_defers_the_model_to_the_session_start_frame(route: ) == {"model": "frame-model"} +@pytest.mark.parametrize("route", ("/realtime/translations", "/v1/realtime/translations")) +@pytest.mark.parametrize("query_model", (None, "requested")) +def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: str, query_model: str | None) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + admitted: Final = managed_inference_request( + route, {}, {"completion_model": "unrelated-default"}, "cli", query_model=query_model + ) + + assert admitted["model"] == resolve_realtime_route_model(query_model, None, is_translation=True) + assert isinstance(admitted["model"], str) and admitted["model"] + + @pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")]) def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None: context: Final = ManagedAgentContext.model_validate( @@ -573,21 +598,41 @@ async def test_ordinary_agent_admission_preserves_legacy_authentication( @pytest.mark.parametrize( "route", - tuple(dict.fromkeys( - LiteLLMRoutes.openai_routes.value - + LiteLLMRoutes.anthropic_routes.value - + LiteLLMRoutes.google_routes.value - )), + tuple( + dict.fromkeys( + LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.anthropic_routes.value + LiteLLMRoutes.google_routes.value + ) + ), ) def test_registered_inference_routes_have_an_explicit_managed_access_decision(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed normalized: Final = route.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") - unsupported: Final = normalized.startswith(( - "/videos", "/batches", "/files", "/fine_tuning", "/assistants", "/threads", "/utils/", - "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", - "/interactions", "/agents", "/responses/{", "/responses/input_tokens", - "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", - )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") + unsupported: Final = normalized.startswith( + ( + "/videos", + "/batches", + "/files", + "/fine_tuning", + "/assistants", + "/threads", + "/utils/", + "/vector_stores", + "/vector_store/", + "/search", + "/containers", + "/skills", + "/claude-code/", + "/interactions", + "/agents", + "/responses/{", + "/responses/input_tokens", + "/realtime/client_secrets", + "/realtime/calls", + "/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/realtime/translations/calls", + ) + ) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route diff --git a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py index 88da07c612b..0aacc1f8bda 100644 --- a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py @@ -1268,6 +1268,153 @@ async def test_task_method_failure_hook_uses_enriched_request_data(): assert failure_data.get("agent_id") == "test-agent" +_ENTRA_APP_CLIENT_ID: Final = "0d08e61f-6184-4980-9cef-a2e9128f4cb2" +_ENTRA_TRACE_ID: Final = "f0c149d3-ef20-4194-8b81-c81a5e002500" + + +class _JsonRpcError(BaseModel): + model_config = ConfigDict(frozen=True) + + code: int + message: str + + +class _JsonRpcErrorBody(BaseModel): + model_config = ConfigDict(frozen=True) + + error: _JsonRpcError + + +async def _send_message_to_entra_agent( + litellm_params: Mapping[str, str], *patches: AbstractContextManager[object] +) -> tuple[int, _JsonRpcError]: + agent = _make_agent_mock() + agent.litellm_params = dict(litellm_params) + mock_request = _make_request_mock( + "message/send", + {"message": {"role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1"}}, + ) + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, skip_guardrails=False: data + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in (*_base_patches(agent), *patches): + stack.enter_context(p) + stack.enter_context( + patch( # test-quality-ok: same proxy_logging_obj injection the sibling failure-hook tests use + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1"), + ) + + return response.status_code, _JsonRpcErrorBody.model_validate_json(response.body).error + + +@pytest.mark.asyncio +async def test_entra_rejecting_the_agent_credential_returns_401_with_only_the_aadsts_reason(): + from azure.core.exceptions import ClientAuthenticationError + + def rejected_token() -> str: + raise ClientAuthenticationError( + message="Authentication failed: AADSTS7000215: Invalid client secret provided. Ensure the secret being " + f"sent in the request is the client secret value, for a secret added to app '{_ENTRA_APP_CLIENT_ID}'." + ) + + status_code, error = await _send_message_to_entra_agent( + {"tenant_id": "tenant", "client_id": _ENTRA_APP_CLIENT_ID, "client_secret": "wrong-secret"}, + patch( # test-quality-ok: stands in for the Entra token endpoint answering a wrong secret, the shape azure-identity raises + "litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id", return_value=rejected_token + ), + ) + + assert status_code == 401 + assert "AADSTS7000215" in error.message + assert _ENTRA_APP_CLIENT_ID not in error.message + + +@pytest.mark.asyncio +async def test_entra_rejecting_the_agent_oidc_token_returns_401_with_only_the_aadsts_reason(monkeypatch, respx_mock): + monkeypatch.setenv("LITELLM_QA_FEDERATED_TOKEN", "federated-token") + monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) + respx_mock.post("https://login.microsoftonline.com/qa-tenant/oauth2/v2.0/token").mock( + return_value=httpx.Response( + 400, + json={ + "error": "invalid_client", + "error_description": "AADSTS700213: No matching federated identity record found for app " + f"'{_ENTRA_APP_CLIENT_ID}'. Trace ID: {_ENTRA_TRACE_ID}", + }, + ) + ) + + status_code, error = await _send_message_to_entra_agent( + { + "tenant_id": "qa-tenant", + "client_id": _ENTRA_APP_CLIENT_ID, + "azure_ad_token": "oidc/env/LITELLM_QA_FEDERATED_TOKEN", + } + ) + + assert status_code == 401 + assert error.code == -32000 + assert "AADSTS700213" in error.message + assert _ENTRA_APP_CLIENT_ID not in error.message + assert _ENTRA_TRACE_ID not in error.message + + +@pytest.mark.asyncio +async def test_upstream_authentication_error_without_an_entra_rejection_stays_an_internal_error(): + agent = _make_agent_mock() + mock_handler = MagicMock() + mock_handler.post = AsyncMock( + side_effect=litellm.AuthenticationError( + message="Incorrect API key provided.", llm_provider="openai", model="gpt-4.1" + ) + ) + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, skip_guardrails=False: data + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( # test-quality-ok: stands in for the upstream agent rejecting the provider key it was sent + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=mock_handler + ) + ) + stack.enter_context( + patch( # test-quality-ok: same proxy_logging_obj injection the sibling failure-hook tests use + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=_make_request_mock("tasks/get", {"id": "task-1"}), + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1"), + ) + + assert response.status_code == 500 + assert json.loads(response.body.decode())["error"]["code"] == -32603 + + @pytest.mark.asyncio async def test_agentcore_invalid_context_id_returns_jsonrpc_invalid_params_400(): from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py index 1c706978fed..2d28292a487 100644 --- a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py +++ b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -26,6 +26,7 @@ from litellm.proxy.analytics_endpoints.cache_activity import ( ADMIN: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-user") INTERNAL_USER: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user-1") +from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_error, statement_timeout_reads async def _no_teams(auth: UserAPIKeyAuth) -> tuple[str, ...]: @@ -70,10 +71,15 @@ MODEL_ROWS = [{"model": "gpt-5.1"}] def build_prisma(query_raw: AsyncMock) -> MagicMock: prisma = MagicMock() - prisma.db.query_raw = query_raw + statement_timeout_reads(prisma, query_raw) return prisma +def _spend_log_reads(prisma: MagicMock) -> AsyncMock: + """The raw reads the cache dashboard ran under the statement timeout.""" + return prisma.read_db.tx.return_value.query_raw + + def dispatching_query_raw() -> AsyncMock: async def dispatch(sql: str, *params: object) -> list[dict[str, object]]: if "error_code" in sql: @@ -120,7 +126,9 @@ async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock): await _activity(ADMIN, key_aliases=["my-key"], models=["gpt-5.1", "claude-opus-4-8"]) filtered_calls = [ - call for call in mock_prisma.db.query_raw.call_args_list if call.args[0] in (GROUPS_SQL, ERROR_BREAKDOWN_SQL) + call + for call in _spend_log_reads(mock_prisma).call_args_list + if call.args[0] in (GROUPS_SQL, ERROR_BREAKDOWN_SQL) ] assert len(filtered_calls) == 2 for call in filtered_calls: @@ -135,7 +143,9 @@ async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock) the SQL does with it is covered against Postgres in tests/integration/spend/test_cache_activity.py.""" await _activity(ADMIN) - exclusions_by_query = {call.args[0]: json.loads(call.args[-4]) for call in mock_prisma.db.query_raw.call_args_list} + exclusions_by_query = { + call.args[0]: json.loads(call.args[-4]) for call in _spend_log_reads(mock_prisma).call_args_list + } assert set(exclusions_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} for excluded_call_types in exclusions_by_query.values(): assert excluded_call_types == LiteLLMRoutes.info_routes.value @@ -147,7 +157,7 @@ async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock) async def test_admin_queries_grant_all_rows(mock_prisma: MagicMock): await _activity(ADMIN) - scope_by_query = {call.args[0]: call.args[-3:] for call in mock_prisma.db.query_raw.call_args_list} + scope_by_query = {call.args[0]: call.args[-3:] for call in _spend_log_reads(mock_prisma).call_args_list} assert set(scope_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} for scope_args in scope_by_query.values(): assert scope_args == (True, None, "[]") @@ -157,7 +167,7 @@ async def test_admin_queries_grant_all_rows(mock_prisma: MagicMock): async def test_internal_user_queries_are_scoped_to_own_and_team_rows(mock_prisma: MagicMock): await _activity(INTERNAL_USER, team_lookup=_team_nine) - scope_by_query = {call.args[0]: call.args[-3:] for call in mock_prisma.db.query_raw.call_args_list} + scope_by_query = {call.args[0]: call.args[-3:] for call in _spend_log_reads(mock_prisma).call_args_list} assert set(scope_by_query) == {GROUPS_SQL, ERROR_BREAKDOWN_SQL, KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL} for scope_args in scope_by_query.values(): assert scope_args == (False, "user-1", json.dumps(["team-9"])) @@ -168,7 +178,7 @@ async def test_key_without_user_id_grants_no_rows(mock_prisma: MagicMock): no_user = UserAPIKeyAuth(user_role=LitellmUserRoles.TEAM, user_id=None) await _activity(no_user) - for call in mock_prisma.db.query_raw.call_args_list: + for call in _spend_log_reads(mock_prisma).call_args_list: assert call.args[-3:] == (False, None, "[]") @@ -178,7 +188,27 @@ async def test_rejects_malformed_dates_with_400(mock_prisma: MagicMock): await _activity(ADMIN, start_date="07/01/2026") assert exc_info.value.status_code == 400 - mock_prisma.db.query_raw.assert_not_called() + _spend_log_reads(mock_prisma).assert_not_called() + + +@pytest.mark.asyncio +async def test_every_cache_activity_read_is_cancelled_by_postgres_after_30s(mock_prisma: MagicMock): + await _activity(ADMIN) + + transaction = mock_prisma.read_db.tx.return_value + assert transaction.statements == ("SET LOCAL statement_timeout = 30000",) * 4 + + +@pytest.mark.asyncio +async def test_cancelled_cache_activity_read_is_reported_as_503_with_a_date_range_hint(monkeypatch: pytest.MonkeyPatch): + prisma = build_prisma(AsyncMock(side_effect=statement_timeout_error())) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + + with pytest.raises(HTTPException) as exc_info: + await _activity(ADMIN) + + assert exc_info.value.status_code == 503 + assert exc_info.value.detail == {"error": "Cache activity took longer than 30s. Pick a shorter date range"} def test_totals_ratio_is_zero_without_requests(): diff --git a/tests/unit/proxy/anthropic_endpoints/test_answered_status.py b/tests/unit/proxy/anthropic_endpoints/test_answered_status.py new file mode 100644 index 00000000000..ff6c98e2282 --- /dev/null +++ b/tests/unit/proxy/anthropic_endpoints/test_answered_status.py @@ -0,0 +1,116 @@ +from typing import Final + +import openai +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.proxy._types import ProxyException +from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status + + +def _anthropic_internal_server_error(provider_status_code: int | None) -> litellm.InternalServerError: + return litellm.InternalServerError( + message="AnthropicError - Overloaded", + llm_provider="anthropic", + model="claude-opus-5-5", + provider_status_code=provider_status_code, + ) + + +@pytest.mark.parametrize("code", [400, 500, 503]) +def test_a_proxy_exception_passes_through_unchanged(code: int) -> None: + exc: Final = ProxyException(message="budget service unavailable", type="None", param=None, code=code) + + answered_exc: Final = with_answered_status(exc) + + assert answered_exc is exc + assert exc.code == str(code) + + +@pytest.mark.parametrize( + ("exc", "answered", "copy"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="BedrockException", llm_provider="bedrock", model="m"), + 503, + False, + id="service-unavailable-503", + ), + pytest.param(_anthropic_internal_server_error(529), 529, True, id="anthropic-overload"), + pytest.param(_anthropic_internal_server_error(500), 500, True, id="anthropic-500"), + pytest.param(_anthropic_internal_server_error(None), 500, False, id="internal-server-error"), + pytest.param( + MidStreamFallbackError( + message="fallbacks failed", + model="m", + llm_provider="anthropic", + original_exception=_anthropic_internal_server_error(529), + ), + 529, + True, + id="fallback-over-an-anthropic-overload", + ), + pytest.param( + MidStreamFallbackError( + message="fallbacks failed", + model="m", + llm_provider="anthropic", + original_exception=MidStreamFallbackError( + message="retry failed before content", + model="m", + llm_provider="anthropic", + original_exception=_anthropic_internal_server_error(529), + ), + ), + 529, + True, + id="nested-fallback-over-an-anthropic-overload", + ), + pytest.param( + litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="m"), + 429, + False, + id="rate-limit", + ), + ], +) +def test_a_litellm_exception_answers_the_status_anthropic_gives_its_failure( + exc: openai.APIStatusError, answered: int, copy: bool +) -> None: + raised_status: Final = exc.status_code + + answered_exc: Final = with_answered_status(exc) + + assert (answered_exc is not exc) is copy + assert (type(answered_exc), answered_exc.status_code, str(answered_exc)) == (type(exc), answered, str(exc)) + assert exc.status_code == raised_status + + +def _raise_a_bedrock_overload_caused_by_a_reset() -> None: + try: + raise ConnectionError("upstream reset") + except ConnectionError as upstream: + raise litellm.InternalServerError( + message="AnthropicError - Overloaded", + llm_provider="anthropic", + model="m", + provider_status_code=529, + ) from upstream + + +def test_the_answered_copy_keeps_the_raised_exceptions_cause_and_traceback() -> None: + with pytest.raises(litellm.InternalServerError) as raised: + _raise_a_bedrock_overload_caused_by_a_reset() + + answered_exc: Final = with_answered_status(raised.value) + + assert isinstance(raised.value.__cause__, ConnectionError) + assert (answered_exc.__cause__, answered_exc.__traceback__) == (raised.value.__cause__, raised.value.__traceback__) + + +def test_an_exception_without_a_status_passes_through_unchanged() -> None: + exc: Final = ValueError("no status") + + assert with_answered_status(exc) is exc + assert not hasattr(exc, "status_code") diff --git a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py index 9c693ff1c3e..ea2fe42a01f 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -4,15 +4,20 @@ Unit tests for claude_code_marketplace.py source validation. Covers the git-subdir and archive source types added alongside the existing github and url types. """ +import hashlib +import io import json -from collections.abc import Mapping +import zipfile +from collections.abc import AsyncIterator, Mapping from types import MappingProxyType -from typing import Final -from unittest.mock import AsyncMock, MagicMock +from typing import Final, TypeAlias +from unittest.mock import AsyncMock, MagicMock, call import httpx import pytest from fastapi import HTTPException +from prisma import Base64, models as prisma_models +from prisma.errors import UniqueViolationError from pydantic import ValidationError import litellm @@ -29,7 +34,9 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp register_plugin, update_plugin, ) +from litellm.proxy.deps.repositories import get_plugin_upload_repository from litellm.proxy.proxy_server import LitellmUserRoles, hash_token +from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.types.proxy.claude_code_endpoints import ( MarketplaceResponse, RegisterPluginRequest, @@ -83,6 +90,7 @@ def _make_mock_prisma(): mock_table.create = AsyncMock(side_effect=_create) mock_table.update = AsyncMock(side_effect=_update) mock_client.db.litellm_claudecodeplugintable = mock_table + mock_client.writer_db = mock_client.db return mock_client @@ -196,6 +204,25 @@ async def test_update_plugin_replaces_existing_source(): assert stored["version"] == "2.0.0" +@pytest.mark.asyncio +async def test_metadata_edits_preserve_uploaded_source_and_reject_a_changed_http_destination() -> None: + table: Final = litellm.proxy.proxy_server.prisma_client.writer_db.litellm_claudecodeplugintable + await table.create(data={"name": _UPLOAD_NAME, "manifest_json": json.dumps({"source": _UPLOAD_SOURCE})}) + request: Final = UpdatePluginRequest(source=_UPLOAD_SOURCE, description="Updated description") + response: Final = await update_plugin(plugin_name=_UPLOAD_NAME, request=request, user_api_key_dict=_USER) + expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description"} + assert response.plugin.description == "Updated description" + assert await _read_stored_manifest(_UPLOAD_NAME) == expected + with pytest.raises(HTTPException) as error: + await update_plugin( + plugin_name=_UPLOAD_NAME, user_api_key_dict=_USER, + request=UpdatePluginRequest(source={**_UPLOAD_SOURCE, "url": "http://other.example/skill.zip"}), + ) + assert error.value.status_code == 400 + table.update.assert_awaited_once() + assert await _read_stored_manifest(_UPLOAD_NAME) == expected + + @pytest.mark.asyncio async def test_update_plugin_not_found(): """PUT on a name that does not exist raises HTTP 404.""" @@ -1171,3 +1198,164 @@ async def test_marketplace_without_a_database_answers_db_not_connected_for_a_que assert response.status_code == 500 assert CommonProxyErrors.db_not_connected_error.value in response.text + + +_UploadClient: TypeAlias = tuple[httpx.AsyncClient, MagicMock, MagicMock] +_UPLOAD_NAME: Final = "local-skill" +_UPLOAD_DIGEST: Final = hashlib.sha256(b"stored archive").hexdigest() +_UPLOAD_PATH: Final = f"/claude-code/plugins/{_UPLOAD_NAME}/archive/{_UPLOAD_DIGEST}.zip" +_UPLOAD_SOURCE: Final = {"source": "archive", "url": f"http://test{_UPLOAD_PATH}", "sha256": _UPLOAD_DIGEST} + + +@pytest.fixture +async def upload_storage(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[_UploadClient]: + prisma: Final = litellm.proxy.proxy_server.prisma_client + prisma.writer_db = MagicMock() + plugins: Final = prisma.writer_db.litellm_claudecodeplugintable + archives: Final = prisma.writer_db.litellm_claudecodepluginupload + plugins.create = AsyncMock(return_value=prisma_models.LiteLLM_ClaudeCodePluginTable( + id="uploaded-id", name=_UPLOAD_NAME, enabled=True, + )) + plugins.find_unique = AsyncMock(return_value=prisma_models.LiteLLM_ClaudeCodePluginTable( + id="uploaded-id", name=_UPLOAD_NAME, enabled=True, manifest_json=json.dumps({"source": _UPLOAD_SOURCE}), + )) + archives.find_first = AsyncMock(return_value=prisma_models.LiteLLM_ClaudeCodePluginUpload( + plugin_name=_UPLOAD_NAME, content=Base64.encode(b"stored archive"), + )) + monkeypatch.setattr(litellm.proxy.proxy_server.app, "dependency_overrides", { + get_plugin_upload_repository: lambda: PluginUploadRepository(prisma), + }) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {}) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", DualCache()) + monkeypatch.setenv("PROXY_BASE_URL", "http://test") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + for key, skills in ((_LIMITED_KEY, [_UPLOAD_NAME]), ("sk-plain", [])): + litellm.proxy.proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), value=_granted_user(skills).model_copy(update={"token": hash_token(key)}), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=litellm.proxy.proxy_server.app), base_url="http://test", + ) as client: + yield client, plugins, archives + + +async def _post_skill_upload( + client: httpx.AsyncClient, content: bytes, metadata: str, key: str = MASTER_KEY, +) -> httpx.Response: + return await client.post( + "/claude-code/plugins/upload", headers={"Authorization": f"Bearer {key}"}, + data={"metadata": metadata}, files={"file": ("local.zip", content, "application/zip")}, + ) + + +@pytest.mark.asyncio +async def test_skill_upload_creates_metadata_and_bytes_atomically_and_rejects_duplicates( + upload_storage: _UploadClient, +) -> None: + client, plugins, _ = upload_storage + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("SKILL.md", "Read the personal instructions") + metadata: Final = {"name": _UPLOAD_NAME, "description": "My notes", "namespace": "personal", "version": "2.0.0"} + response: Final = await _post_skill_upload(client, buffer.getvalue(), json.dumps(metadata)) + assert response.status_code == 200, response.text + plugins.create.assert_awaited_once() + stored: Final = plugins.create.await_args.kwargs["data"] + content: Final = stored["uploaded_archive"]["create"]["content"].decode() + digest: Final = hashlib.sha256(content).hexdigest() + source: Final = { + "source": "archive", + "url": f"http://test/claude-code/plugins/{_UPLOAD_NAME}/archive/{digest}.zip", + "sha256": digest, + } + assert json.loads(stored["manifest_json"]) == {**metadata, "source": source} + assert stored == { + "name": _UPLOAD_NAME, "description": "My notes", "version": "2.0.0", + "manifest_json": stored["manifest_json"], "created_by": litellm.proxy.proxy_server.litellm_proxy_admin_name, + "uploaded_archive": {"create": {"content": Base64.encode(content)}}, + } + assert response.json() == {"status": "success", "action": "created", "plugin": { + "id": "uploaded-id", "name": _UPLOAD_NAME, "version": "2.0.0", "description": "My notes", + "source": source, "installation_preference": None, "enabled": True, + }} + plugins.create.side_effect = UniqueViolationError({"message": "duplicate", "error_code": "P2002"}) + duplicate: Final = await _post_skill_upload(client, buffer.getvalue(), json.dumps(metadata)) + assert duplicate.status_code == 409, duplicate.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("content, metadata, key, status", [ + (b"invalid", '{"name":"local-skill"}', MASTER_KEY, 400), + (b"x" * (10 * 1024 * 1024 + 1), '{"name":"local-skill"}', MASTER_KEY, 413), + (b"ignored", "not-json", MASTER_KEY, 400), + (b"ignored", '{"name":"../invalid"}', MASTER_KEY, 422), + (b"ignored", '{"name":"local-skill"}', _LIMITED_KEY, 403), +], ids=["invalid-zip", "file-over-cap", "invalid-metadata", "invalid-name", "non-admin"]) +async def test_skill_upload_rejects_invalid_or_unauthorized_input_without_writing( + upload_storage: _UploadClient, content: bytes, metadata: str, key: str, status: int, +) -> None: + client, plugins, _ = upload_storage + response: Final = await _post_skill_upload(client, content, metadata, key) + assert response.status_code == status, response.text + plugins.create.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled, key, source, blob, status, reads", [ + (True, None, _UPLOAD_SOURCE, True, 200, 1), + (False, None, _UPLOAD_SOURCE, True, 404, 0), + (False, "sk-plain", _UPLOAD_SOURCE, True, 404, 0), + (False, _LIMITED_KEY, _UPLOAD_SOURCE, True, 200, 1), + (True, None, None, True, 404, 0), + (True, None, {**_UPLOAD_SOURCE, "sha256": "0" * 64}, True, 404, 0), + (True, None, {**_UPLOAD_SOURCE, "url": "https://external.example/skill.zip"}, True, 404, 0), + (True, None, _UPLOAD_SOURCE, False, 404, 1), +]) +async def test_archive_reads_only_authorized_current_bytes( + upload_storage: _UploadClient, enabled: bool, key: str | None, + source: Mapping[str, str] | None, blob: bool, status: int, reads: int, +) -> None: + client, plugins, archives = upload_storage + manifest: Final = json.dumps({"source": source}) + plugins.find_unique.return_value = None if source is None else prisma_models.LiteLLM_ClaudeCodePluginTable( + id="uploaded-id", name=_UPLOAD_NAME, enabled=enabled, manifest_json=manifest, + ) + if not blob: + archives.find_first.return_value = None + response: Final = await client.get(_UPLOAD_PATH, headers={"Authorization": f"Bearer {key}"} if key else {}) + assert response.status_code == status, response.text + assert (response.content == b"stored archive") is (status == 200) + plugins.find_unique.assert_awaited_once_with(where={"name": _UPLOAD_NAME}) + assert archives.find_first.await_args_list == ([call(where={ + "plugin_name": _UPLOAD_NAME, "plugin": {"is": {"manifest_json": manifest, "enabled": enabled}}, + })] if reads else []) + litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable.find_unique.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("required, header, credential, query, status", [ + (True, "Authorization", _LIMITED_KEY, None, 200), + (False, "x-skill-key", _LIMITED_KEY, MASTER_KEY, 200), + (True, "x-skill-key", _LIMITED_KEY, MASTER_KEY, 200), + (False, "x-skill-key", "", MASTER_KEY, 401), + (False, None, None, MASTER_KEY, 404), + (False, None, None, "invalid-query-key", 404), + (True, None, None, MASTER_KEY, 401), + (True, None, None, None, 401), +]) +async def test_archive_requires_header_credentials_for_private_content( + upload_storage: _UploadClient, monkeypatch: pytest.MonkeyPatch, required: bool, + header: str | None, credential: str | None, query: str | None, status: int, +) -> None: + client, plugins, archives = upload_storage + plugins.find_unique.return_value.enabled = False + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", { + "litellm_key_header_name": "X-Skill-Key", "claude_code_marketplace_auth_required": required, + }) + response: Final = await client.get( + _UPLOAD_PATH, headers={header: f"Bearer {credential}" if credential else ""} if header else {}, + params={"key": query} if query else {}, + ) + assert response.status_code == status, response.text + assert (response.content == b"stored archive") is (status == 200) + assert archives.find_first.await_count == int(status == 200) diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 553c3d4a9d7..80e7e793f6e 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -2,6 +2,7 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses """ +import datetime import json import logging import unittest @@ -17,6 +18,7 @@ from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient from starlette.types import Message +import litellm from litellm import Router from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache @@ -248,6 +250,86 @@ class TestBlockedResponseUsage: assert response["content"] == [{"type": "text", "text": _BLOCK_TEXT}] +class TestAnthropicResponseReportsWhatAFailedAgenticFollowUpBills: + async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float: + import litellm + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + from litellm.types.utils import ModelResponse, Usage + + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + + async def failing_follow_up() -> ModelResponse: + raise follow_up_error + + async def fail_after_the_first_call(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None: + _, logging_obj = await self.common_processing_pre_call_logic( + request=kwargs["request"], + general_settings={}, + user_api_key_dict=kwargs["user_api_key_dict"], + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="anthropic_messages", + ) + logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now()) + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse( + model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ), + stream=False, + ) + raise ( + follow_up_error + if raise_the_follow_up_error + else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + ) + + request: Final = MagicMock(spec=Request) + request.headers = {} + with ( + patch.object( + ep, + "read_request_body", + new=AsyncMock( + return_value={ + "model": "gpt-4o", + "max_tokens": 64, + "messages": [{"role": "user", "content": "what is 6*7?"}], + } + ), + ), + patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fail_after_the_first_call), + patch.object(proxy_server, "proxy_logging_obj", proxy_logging), + ): + response: Final = await ep.anthropic_response( + fastapi_response=Response(), + request=request, + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/messages"), + ) + + assert response.status_code == 429 + return float(response.headers["x-litellm-response-cost"]) + + @pytest.mark.asyncio + async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None: + import litellm + + price: Final = litellm.model_cost["gpt-4o"] + + assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx( + 559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"] + ) + assert await self._error_response_cost(raise_the_follow_up_error=False) == 0 + + class TestProxyExceptionAnthropicEnvelope: @pytest.mark.asyncio async def test_anthropic_response_maps_proxy_exception_to_anthropic_envelope(self): @@ -302,7 +384,7 @@ class TestProxyExceptionAnthropicEnvelope: import litellm.proxy.anthropic_endpoints.endpoints as ep from litellm.proxy._types import ProxyException - request = MagicMock() + request: Final = MagicMock() request.headers = {} response = ep._anthropic_error_json_response( @@ -319,10 +401,135 @@ class TestProxyExceptionAnthropicEnvelope: assert fallback.status_code == 500 assert json.loads(fallback.body)["error"]["type"] == "api_error" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "unavailable, expected_status, expected_error_type, hook_receives_copy", + [ + pytest.param( + litellm.ServiceUnavailableError( + message='BedrockException - {"message": "Bedrock is unable to process your request."}', + llm_provider="bedrock", + model="global.anthropic.claude-opus-5-5", + ), + 503, + "api_error", + False, + id="bedrock-503", + ), + pytest.param( + litellm.InternalServerError( + message='AnthropicError - {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + llm_provider="anthropic", + model="claude-opus-5-5", + provider_status_code=529, + ), + 529, + "overloaded_error", + True, + id="anthropic-529", + ), + ], + ) + async def test_anthropic_response_records_the_answered_status( + self, + unavailable: litellm.ServiceUnavailableError | litellm.InternalServerError, + expected_status: int, + expected_error_type: str, + hook_receives_copy: bool, + ) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + raised_status: Final = unavailable.status_code + request: Final = MagicMock() + request.headers = {} + + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value={})), + patch.object( + ep.ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=AsyncMock(side_effect=unavailable), + ), + patch.object(proxy_server, "proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + response = await ep.anthropic_response( + fastapi_response=MagicMock(), + request=request, + user_api_key_dict=UserAPIKeyAuth(), + ) + + recorded: Final = mock_logging.post_call_failure_hook.await_args.kwargs["original_exception"] + assert recorded is not None + assert (recorded is not unavailable) is hook_receives_copy + assert recorded.status_code == expected_status + assert unavailable.status_code == raised_status + assert response.status_code == expected_status + assert json.loads(response.body)["error"]["type"] == expected_error_type + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "unavailable, expected_status, expected_error_type", + [ + pytest.param( + litellm.ServiceUnavailableError( + message="Bedrock is unable to process your request.", + llm_provider="bedrock", + model="global.anthropic.claude-opus-5-5", + ), + 503, + "api_error", + id="bedrock-503", + ), + pytest.param( + litellm.InternalServerError( + message='AnthropicError - {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + llm_provider="anthropic", + model="claude-opus-5-5", + provider_status_code=529, + ), + 529, + "overloaded_error", + id="anthropic-529", + ), + ], + ) + async def test_count_tokens_answers_provider_errors( + self, + unavailable: litellm.ServiceUnavailableError | litellm.InternalServerError, + expected_status: int, + expected_error_type: str, + ) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + request = MagicMock() + request.headers = {} + + with ( + patch.object( + ep, + "read_request_body", + new=AsyncMock(return_value={"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]}), + ), + patch.object( + proxy_server, "count_request_tokens", new=AsyncMock(side_effect=unavailable) + ), + ): + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + body: Final = json.loads(response.body) + assert response.status_code == expected_status + assert body["type"] == "error" + assert body["error"]["type"] == expected_error_type + @pytest.mark.parametrize("status_code", [402, 409, 422, 504]) def test_a_failure_inside_the_handler_keeps_the_api_error_type_for_a_status_outside_the_table( self, status_code: int - ): + ) -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep from litellm.proxy._types import ProxyException @@ -334,7 +541,6 @@ class TestProxyExceptionAnthropicEnvelope: assert response.status_code == status_code assert json.loads(response.body)["error"] == {"type": "api_error", "message": "scripted"} - @staticmethod def _call_id_error_response(general_settings, provider_specific_fields=None): import litellm.proxy.anthropic_endpoints.endpoints as ep @@ -428,7 +634,7 @@ class TestHttpExceptionDictDetail: ) as mock_logging, # test-quality-ok: module global imported at call time; no injection seam ): mock_logging.post_call_failure_hook = AsyncMock() - response = await ep.anthropic_response( + response: Final = await ep.anthropic_response( fastapi_response=MagicMock(), request=request, user_api_key_dict=UserAPIKeyAuth(), diff --git a/tests/unit/proxy/anthropic_endpoints/test_skill_upload_archive.py b/tests/unit/proxy/anthropic_endpoints/test_skill_upload_archive.py new file mode 100644 index 00000000000..04b81605e29 --- /dev/null +++ b/tests/unit/proxy/anthropic_endpoints/test_skill_upload_archive.py @@ -0,0 +1,64 @@ +import io +import json +import stat +import zipfile +from typing import Final + +import pytest + +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.skill_upload_archive import ( + MAX_SKILL_ARCHIVE_COMPRESSION_RATIO, + build_uploaded_skill_archive, +) + +MANIFEST: Final = b"---\nname: skill\ndescription: Notes\n---\nRead guide.md" + + +def _zip(files: tuple[tuple[str | zipfile.ZipInfo, bytes], ...], compression: int = zipfile.ZIP_DEFLATED) -> bytes: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=compression) as archive: + for name, content in files: + archive.writestr(name, content) + return buffer.getvalue() + + +def _entry(name: str, mode: int, system: int = 3) -> zipfile.ZipInfo: + entry: Final = zipfile.ZipInfo(name) + entry.create_system = system + entry.external_attr = mode << 16 + return entry + + +@pytest.mark.parametrize("prefix, manifest", [ + ("", MANIFEST), ("folder/", MANIFEST), + ("folder/", b"---\nnotes: " + b"[" * 1000 + b"0" + b"]" * 1000 + b"\n---\nRead guide.md"), +]) +@pytest.mark.parametrize("script, compression", [ + (b"#!/bin/sh\necho ready\n", zipfile.ZIP_DEFLATED), (b"echo ready\n" * 10000, zipfile.ZIP_STORED), +]) +def test_installable_archive_preserves_files_and_safe_modes( + prefix: str, manifest: bytes, script: bytes, compression: int, +) -> None: + files: Final = ( + ("SKILL.md", manifest, 0o644, 3, 0o644), + ("guide.md", b"Keep filenames unchanged", 0o666, 3, 0o644), + ("scripts/run.sh", script, stat.S_IFREG | 0o7777, 3, 0o755), + ("scripts/owner.sh", script, stat.S_IFREG | 0o700, 3, 0o744), + ("scripts/no-type.sh", script, 0o755, 3, 0o755), + ("scripts/dos.sh", script, stat.S_IFREG | 0o7777, 0, 0o644), + ("scripts/dos-link.sh", script, stat.S_IFLNK | 0o777, 0, 0o644), + ) + upload: Final = _zip(tuple( + (_entry(prefix + name, mode, system), data) for name, data, mode, system, _ in files + ) + ((prefix + ".DS_Store", b"Finder"), ("__MACOSX/._SKILL.md", b"Finder"))) + result: Final = build_uploaded_skill_archive(upload, "registered-name") + expected: Final = {".claude-plugin/plugin.json": (json.dumps({"name": "registered-name"}).encode(), 0o644)} | { + f"skills/registered-name/{name}": (data, mode) for name, data, _, _, mode in files + } + assert result is not None + with zipfile.ZipFile(io.BytesIO(result)) as archive: + assert {entry.filename: (archive.read(entry), entry.external_attr >> 16) for entry in archive.infolist()} == expected + assert {entry.create_system for entry in archive.infolist()} == {3} + assert {entry.compress_type for entry in archive.infolist()} == {compression} + assert sum(entry.file_size for entry in archive.infolist()) <= len(result) * MAX_SKILL_ARCHIVE_COMPRESSION_RATIO + assert build_uploaded_skill_archive(upload, "registered-name") == result diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 7a060a214f7..024d00d5d95 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -4023,11 +4023,11 @@ async def test_common_checks_rejects_a_spent_mapped_tag_whatever_non_list_tags_t async def fake_find_many(**kwargs): if "where" not in kwargs: - return [_tag_registry_row("usecase:spent")] + return [_tag_registry_row("app:spent")] return [_tag_db_row(name, max_budget=1.0) for name in kwargs["where"]["tag_name"]["in"]] async def spent_tag_spend(counter_key, fallback_spend, max_budget=None, **kwargs): - return 1.5 if counter_key == "spend:tag:usecase:spent" else fallback_spend + return 1.5 if counter_key == "spend:tag:app:spent" else fallback_spend async def receive(): return { @@ -4047,7 +4047,7 @@ async def test_common_checks_rejects_a_spent_mapped_tag_whatever_non_list_tags_t "scheme": "http", "server": ("localhost", 4000), "query_string": b"", - "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=spent")], + "headers": [(b"content-type", b"application/json"), (b"baggage", b"service=spent")], }, receive, ) @@ -4055,7 +4055,7 @@ async def test_common_checks_rejects_a_spent_mapped_tag_whatever_non_list_tags_t await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, request_body, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) @@ -4077,7 +4077,7 @@ async def test_common_checks_rejects_a_spent_mapped_tag_whatever_non_list_tags_t valid_token=UserAPIKeyAuth(token="test-token", metadata=key_metadata), request=request, ) - assert exc_info.value.entity_id == "usecase:spent" + assert exc_info.value.entity_id == "app:spent" @pytest.mark.asyncio @@ -11789,8 +11789,8 @@ def _client_tag_request(client_tags, baggage): { "type": "http", "headers": [ - *([(b"baggage", b"entrypoint=a")] if baggage else []), - *([(b"x-litellm-tags", b"usecase:a")] if client_tags == "header" else []), + *([(b"baggage", b"service=a")] if baggage else []), + *([(b"x-litellm-tags", b"app:a")] if client_tags == "header" else []), ], } ) @@ -11814,7 +11814,7 @@ def _client_tag_rejection(route, data, request): @pytest.mark.asyncio @pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/messages", "/v1/responses"]) -@pytest.mark.parametrize("client_tags", [None, [], ["usecase:a"], '{"tags":["usecase:a"]}', "header"]) +@pytest.mark.parametrize("client_tags", [None, [], ["app:a"], '{"tags":["app:a"]}', "header"]) @pytest.mark.parametrize("client_slot", ["metadata", "litellm_metadata"]) @pytest.mark.parametrize("baggage", [True, False]) async def test_baggage_mapping_rejects_exactly_the_client_tags_rejected_without_it( @@ -11833,7 +11833,7 @@ async def test_baggage_mapping_rejects_exactly_the_client_tags_rejected_without_ await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( mapped_request, mapped, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) @@ -11843,19 +11843,19 @@ async def test_baggage_mapping_rejects_exactly_the_client_tags_rejected_without_ route, unmapped, unmapped_request ) if client_tags is None and baggage: - assert "usecase:a" in mapped["metadata" if "chat/completions" in route else "litellm_metadata"]["tags"] + assert "app:a" in mapped["metadata" if "chat/completions" in route else "litellm_metadata"]["tags"] @pytest.mark.asyncio async def test_baggage_mapping_still_rejects_client_metadata_tags(): from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - request = _client_tag_request(["usecase:b"], baggage=True) - data = {"metadata": {"tags": ["usecase:b"]}} + request = _client_tag_request(["app:b"], baggage=True) + data = {"metadata": {"tags": ["app:b"]}} await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) diff --git a/tests/unit/proxy/auth/test_fallback_model_access.py b/tests/unit/proxy/auth/test_fallback_model_access.py index feeb2d26ff3..5c6a98d9e42 100644 --- a/tests/unit/proxy/auth/test_fallback_model_access.py +++ b/tests/unit/proxy/auth/test_fallback_model_access.py @@ -1,4 +1,4 @@ -from typing import Final +from typing import Final, NoReturn import pytest @@ -50,14 +50,15 @@ async def test_is_model_authorized_for_token_follows_the_key_access_groups(): assert await is_model_authorized_for_token(model="secret-model", valid_token=token, llm_router=router) is False -class _RouterWithBrokenAccessGroupLookup(Router): - def get_model_access_groups(self, *args, **kwargs): - raise RuntimeError("access group store unavailable") +def _broken_access_group_lookup(*args: object, **kwargs: object) -> NoReturn: + raise RuntimeError("access group store unavailable") @pytest.mark.asyncio -async def test_is_model_authorized_for_token_fails_closed_when_the_lookup_breaks(): - router = _RouterWithBrokenAccessGroupLookup(model_list=_router().model_list) +@pytest.mark.usefixtures("python_router_core") # the Rust core keeps this lookup on its host registry +async def test_is_model_authorized_for_token_fails_closed_when_the_lookup_breaks(monkeypatch: pytest.MonkeyPatch): + router = _router() + monkeypatch.setattr(router, "_core_get_model_access_groups", _broken_access_group_lookup) assert ( await is_model_authorized_for_token( diff --git a/tests/unit/proxy/auth/test_model_access_group_budgets.py b/tests/unit/proxy/auth/test_model_access_group_budgets.py index 7506bd031d9..d1b381907fa 100644 --- a/tests/unit/proxy/auth/test_model_access_group_budgets.py +++ b/tests/unit/proxy/auth/test_model_access_group_budgets.py @@ -8,7 +8,8 @@ unioned rather than ranked. """ from types import SimpleNamespace -from unittest.mock import patch +from typing import NoReturn +from unittest.mock import MagicMock, patch import pytest @@ -66,18 +67,6 @@ class _ExplodingPrismaClient: raise AssertionError(f"unexpected database access: {name}") -class _CountingRouter(Router): - """Counts access-group lookups, so a test can prove the registry gate skipped them.""" - - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - self.access_group_lookups = 0 - - def get_model_access_groups(self, *args, **kwargs): - self.access_group_lookups += 1 - return super().get_model_access_groups(*args, **kwargs) - - async def _cache( budgeted_groups: tuple[str, ...] = BUDGETED_GROUPS, member_allowed_models: tuple[str, ...] = (), @@ -233,14 +222,17 @@ async def test_group_without_a_budget_is_not_attributed(): @pytest.mark.asyncio -async def test_empty_registry_skips_the_access_group_matching_entirely(): - router = _CountingRouter(model_list=MODEL_LIST) +@pytest.mark.usefixtures("python_router_core") # the Rust core keeps this lookup on its host registry +async def test_empty_registry_skips_the_access_group_matching_entirely(monkeypatch: pytest.MonkeyPatch): + router = Router(model_list=MODEL_LIST) + access_group_lookups = MagicMock(wraps=router._core_get_model_access_groups) + monkeypatch.setattr(router, "_core_get_model_access_groups", access_group_lookups) assert await _matched(key_models=["tier-a"], cache=await _cache(budgeted_groups=()), llm_router=router) == () - assert router.access_group_lookups == 0 + assert access_group_lookups.call_count == 0 assert await _matched(key_models=["tier-a"], llm_router=router) == ("tier-a",) - assert router.access_group_lookups == 1 + assert access_group_lookups.call_count == 1 @pytest.mark.asyncio @@ -262,22 +254,24 @@ async def test_stamp_records_the_matched_groups_on_the_auth_object(): assert valid_token.matched_model_access_groups == ["tier-a", "tier-b"] -class _BrokenRouter(Router): - def get_model_access_groups(self, *args, **kwargs): - raise RuntimeError("access group store unavailable") +def _broken_access_group_lookup(*args: object, **kwargs: object) -> NoReturn: + raise RuntimeError("access group store unavailable") @pytest.mark.asyncio -async def test_stamp_does_not_break_auth_when_the_access_group_lookup_fails(): +@pytest.mark.usefixtures("python_router_core") # the Rust core keeps this lookup on its host registry +async def test_stamp_does_not_break_auth_when_the_access_group_lookup_fails(monkeypatch: pytest.MonkeyPatch): cache = await _cache() valid_token = UserAPIKeyAuth(api_key="hashed", models=["tier-a"], team_id=TEAM_ID, user_id=USER_ID) + router = Router(model_list=MODEL_LIST) + monkeypatch.setattr(router, "_core_get_model_access_groups", _broken_access_group_lookup) await stamp_matched_model_access_groups( model="gpt-4o", valid_token=valid_token, team_object=None, project_object=None, - llm_router=_BrokenRouter(model_list=MODEL_LIST), + llm_router=router, prisma_client=_ExplodingPrismaClient(), user_api_key_cache=cache, proxy_logging_obj=ProxyLogging(user_api_key_cache=cache), diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index d0156ff0195..bc9366aa607 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,13 +1,25 @@ -from collections.abc import Generator, Mapping +import datetime +import ipaddress +import json +import ssl +import threading +from collections.abc import Generator, Iterator, Mapping from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Final from unittest.mock import patch import httpx import pytest import respx +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID from litellm import Router +from litellm.utils import AZURE_MODEL_PLACEHOLDER def test_get_team_models_for_all_models_and_team_only_models(): @@ -430,6 +442,262 @@ def test_wildcard_credential_hydration_preserves_deployment_params( } +@respx.mock +def test_hosted_vllm_wildcard_lists_the_server_models_without_an_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-without-key.test:8000/v1/models").mock( + return_value=httpx.Response( + 200, + json={ + "data": [ + {"id": "meta-llama/Llama-3.1-8B-Instruct"}, + {"id": "qwen2.5"}, + {"id": "openai/gpt-oss-20b"}, + ] + }, + ) + ) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-without-key.test:8000/v1"), + ) + + assert listed == [ + "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", + "hosted_vllm/qwen2.5", + "hosted_vllm/openai/gpt-oss-20b", + ] + assert "authorization" not in models_route.calls.last.request.headers + + +@respx.mock +def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get( + "http://vllm-with-key.test:8000/v1/models", headers={"Authorization": "Bearer vllm-test-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-with-key.test:8000/v1", api_key="vllm-test-key"), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert models_route.call_count == 1 + + +@respx.mock +def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key") + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000") + passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]}) + ) + respx.get( + "http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm") + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert not passthrough_server_route.called + + +@respx.mock +def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.constants import NON_LLM_CONNECTION_TIMEOUT + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + + get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1") + ) + + read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"] + assert read_timeout is not None + assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT + + +@respx.mock +def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get( + "http://vllm-behind-gateway.test:8000/v1/models", + headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"}, + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://vllm-behind-gateway.test:8000/v1", + api_key="vllm-test-key", + extra_headers={"X-Tenant": "team-a"}, + ), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + + + +@respx.mock +def test_vllm_wildcard_never_lists_the_ids_of_a_configured_vllm_server(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-library-env.test:8000") + monkeypatch.setenv("VLLM_API_KEY", "vllm-library-env-key") + server_route: Final = respx.get(url__regex=r"http://vllm-library(-env)?\.test:8000/.*").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-only-a-server-serves"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "vllm/*", + LiteLLM_Params(model="vllm/*", api_base="http://vllm-library.test:8000", api_key="vllm-library-key"), + ) + + assert "vllm/model-only-a-server-serves" not in listed + assert not server_route.called + +_SELF_SIGNED_CERT_NOT_VALID_BEFORE: Final = datetime.datetime(2000, 1, 1, tzinfo=datetime.UTC) +_SELF_SIGNED_CERT_NOT_VALID_AFTER: Final = datetime.datetime(9999, 12, 31, tzinfo=datetime.UTC) + + +def _self_signed_cert(directory: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(_SELF_SIGNED_CERT_NOT_VALID_BEFORE) + .not_valid_after(_SELF_SIGNED_CERT_NOT_VALID_AFTER) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False + ) + .sign(key, hashes.SHA256()) + ) + cert_path: Final = directory / "cert.pem" + key_path: Final = directory / "key.pem" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return cert_path, key_path + + +@pytest.fixture +def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]: + cert_path, key_path = _self_signed_cert(tmp_path) + + class Handler(BaseHTTPRequestHandler): + timeout = 5 + + def do_GET(self) -> None: + body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_path, key_path) + server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"https://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify( + monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx import http_handler + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + + verified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base) + ) + unverified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False), + ) + + assert verified == [] + assert unverified == ["hosted_vllm/qwen2.5"] + + +@respx.mock +def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get("http://upstream-proxy.test:4000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "team-proxy/*", + LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"), + ) + + assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. @@ -834,6 +1102,56 @@ def test_expand_wildcard_invalid_litellm_params_passthrough(): assert result == [deployment] + +@pytest.mark.parametrize( + ("allowed_model_names", "expected_names", "expected_listings"), + [ + pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"), + pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"), + pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"), + ], +) +@respx.mock +def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern( + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, + allowed_model_names: list[str] | None, + expected_names: list[str], + expected_listings: int, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + api_base: Final = f"http://vllm-alias-{request.node.callspec.id}.test:8000/v1" + listing: Final = respx.get(f"{api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + deployment: Final = {"model_name": "qwen-*", "litellm_params": {"model": "hosted_vllm/*", "api_base": api_base}} + + rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names) + + assert ([row["model_name"] for row in rows], listing.call_count) == (expected_names, expected_listings) + + +@pytest.mark.parametrize( + ("allowed_model_names", "keeps_the_alias"), + [ + pytest.param(["gpt-4o-mini"], True, id="granted-a-name-the-alias-matches"), + pytest.param(["claude-sonnet"], False, id="granted-only-other-names"), + ], +) +def test_expand_wildcard_alias_over_one_model_is_kept_only_for_keys_with_a_name_it_matches( + allowed_model_names: list[str], keeps_the_alias: bool +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + deployment: Final = {"model_name": "gpt-4o*", "litellm_params": {"model": "openai/gpt-4o"}} + + rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names) + + assert rows == ([deployment] if keeps_the_alias else []) + def test_get_complete_model_list_excludes_wildcard_routes_by_default(): """Regression (LIT-4108): a wildcard with a matching router deployment leaked into /v1/models.""" from litellm import Router @@ -1327,6 +1645,40 @@ def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_ assert (listed, catalog.call_count) == ([], 0) +@respx.mock +def test_key_granted_hosted_vllm_wildcard_without_a_hosted_vllm_deployment_lists_no_environment_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-environment-only.test:8000/v1") + environment_server: Final = respx.get("http://hosted-vllm-environment-only.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + router: Final = Router( + model_list=[ + { + "model_name": "vllm-team/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-team.test:8000/v1"}, + } + ] + ) + + listed: Final = get_complete_model_list( + key_models=["hosted_vllm/*"], + team_models=[], + proxy_model_list=router.model_names, + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert (listed, environment_server.call_count) == ([], 0) + + @contextmanager def _openai_catalog_behind_a_catch_all( monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str] @@ -1748,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} pytest.param( "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" ), + pytest.param( + "gpt-partial-route-*", + "openai/gpt-partial-route-*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-repeating-the-deployment-partial-word", + ), + pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"), + pytest.param( + "team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all" + ), + pytest.param( + "team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment" + ), + pytest.param( + "*partial-route-9*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-with-more-wildcards-than-its-deployment", + ), + pytest.param( + "gemini/*partial-route-9*", + "gemini/gemini-*", + None, + {_ROUTE_GEMINI_ID}, + id="slashed-name-with-more-wildcards-than-its-deployment", + ), pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"), pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"), pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"), @@ -1825,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( assert listed_model_info_names == sorted(reachable) +@pytest.mark.parametrize( + ("model_name", "expected_v1_models"), + [ + pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"), + pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"), + pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"), + ], +) +def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder( + model_name: str, expected_v1_models: list[str] +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}]) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert v1_models == expected_v1_models + assert model_info_names == (expected_v1_models or [model_name]) + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"), + pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"), + ], +) +def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog( + model_name: str, deployment_model: str +) -> None: + from fnmatch import fnmatchcase + + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + router: Final = Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"}, + } + ] + ) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai")) + routed_models: Final = { + listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)] + for listed_id in v1_models + } + + assert v1_models + assert [ + listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name) + ] == [] + assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {} + assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models) + assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == [] + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"), + pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"), + pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"), + ], +) +def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments( + model_name: str, deployment_model: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + with_provider: Final = get_known_models_from_wildcard( + model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai") + ) + without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model)) + + assert with_provider == without_provider + + def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None: from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 5df3fbab9a6..efb4451f2fc 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -818,6 +818,52 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/translations", + "/v1/realtime/translations", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + ], +) +def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["openai_routes"], + ) + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + ) + is True + ) + + +@pytest.mark.parametrize( + "route", + [ + "/openai/v1/realtime/translations", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ], +) +def test_openai_prefixed_realtime_translation_routes_denied_for_openai_virtual_keys(route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes"]) + + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) + + assert exc_info.value.status_code == 403 + assert f"Tried to call route: {route}" in str(exc_info.value.detail) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 81ea0e3f60b..19f9d0dcc73 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -931,6 +931,43 @@ async def test_user_api_key_auth_live_websocket_ignores_query_model(): mock_user_api_key_auth.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("auth_name", "path"), + [("user_api_key_auth_websocket", "/v1/responses"), ("user_api_key_auth_realtime_websocket", "/v1/realtime")], +) +async def test_websocket_without_a_model_is_refused_for_a_role_limited_to_other_models( + auth_name: str, path: str +) -> None: + from litellm.proxy._types import RoleBasedPermissions, UserAPIKeyAuth + from litellm.proxy.auth import user_api_key_auth as auth_module + from litellm.proxy.auth.handle_jwt import JWTAuthManager + + async def role_limited_auth(request, api_key: str) -> UserAPIKeyAuth: + JWTAuthManager.can_rbac_role_call_model( + rbac_role=LitellmUserRoles.INTERNAL_USER, + general_settings={ + "role_permissions": [RoleBasedPermissions(role=LitellmUserRoles.INTERNAL_USER, models=["chat"])] + }, + model=json.loads(await request.body())["model"], + ) + return UserAPIKeyAuth(api_key=api_key) + + websocket: Final = MagicMock(spec=WebSocket) + websocket.query_params = {} + websocket.headers = {"authorization": "Bearer some_api_key"} + websocket.scope = {"type": "websocket", "path": path, "headers": [(b"authorization", b"Bearer some_api_key")]} + websocket.url = URL(url=path) + websocket.close = AsyncMock() + + with patch("litellm.proxy.auth.user_api_key_auth.user_api_key_auth", side_effect=role_limited_auth): + with pytest.raises(HTTPException) as raised: + await getattr(auth_module, auth_name)(websocket) + + assert raised.value.status_code == 403 + websocket.close.assert_awaited_once_with(code=status.WS_1008_POLICY_VIOLATION) + + @pytest.mark.asyncio async def test_accepted_websocket_reauthorizes_the_frame_model_without_closing(): from litellm.proxy._types import ProxyException @@ -2166,7 +2203,7 @@ async def test_websocket_inference_is_denied_when_a_required_baggage_mapping_is_ { "request_tag_mappings": [ {"key": "region", "tag_prefix": "region:"}, - {"key": "entrypoint", "tag_prefix": "usecase:", "required": True}, + {"key": "service", "tag_prefix": "app:", "required": True}, ] }, ), @@ -2190,7 +2227,7 @@ async def test_websocket_invalid_key_is_rejected_before_the_required_baggage_map with ( patch.dict( litellm.proxy.proxy_server.general_settings, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "required": True}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "required": True}]}, ), patch( "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", @@ -2211,7 +2248,7 @@ async def test_websocket_inference_is_allowed_when_no_baggage_mapping_is_require with ( patch.dict( litellm.proxy.proxy_server.general_settings, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}]}, ), patch( "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", new=AsyncMock(return_value=authorized) @@ -2236,8 +2273,8 @@ async def _auth_then_serve_chat(monkeypatch, general_settings, body, headers): model_list=[ { "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4o", "tags": ["usecase:a"], "mock_response": "hi"}, - "model_info": {"id": "usecase-deployment"}, + "litellm_params": {"model": "gpt-4o", "tags": ["app:a"], "mock_response": "hi"}, + "model_info": {"id": "app-deployment"}, }, { "model_name": "gpt-4", @@ -2284,30 +2321,30 @@ async def test_auth_header_tag_merge_keeps_json_string_metadata_tags_routable(mo body = { "model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], - "metadata": json.dumps({"tags": ["usecase:a"]}), + "metadata": json.dumps({"tags": ["app:a"]}), } cached_body, served_by = await _auth_then_serve_chat( monkeypatch, general_settings, body, [(b"x-litellm-tags", b"other"), (b"baggage", b"region=sg")] ) - assert served_by == "usecase-deployment" - assert cached_body["metadata"] == {"tags": ["usecase:a", "other"]} + assert served_by == "app-deployment" + assert cached_body["metadata"] == {"tags": ["app:a", "other"]} @pytest.mark.asyncio @pytest.mark.parametrize("client_metadata", ["run-123", 42, ["x"]]) @pytest.mark.parametrize( "baggage", - [[], [(b"baggage", b"region=sg")], [(b"baggage", b"entrypoint=a")]], + [[], [(b"baggage", b"region=sg")], [(b"baggage", b"service=a")]], ids=["no_baggage", "unmapped_baggage", "mapped_baggage"], ) async def test_non_object_metadata_is_served_like_unset_mappings_whatever_the_baggage_maps( monkeypatch, client_metadata, baggage ): body = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "metadata": client_metadata} - headers = [(b"x-litellm-tags", b"usecase:a"), *baggage] + headers = [(b"x-litellm-tags", b"app:a"), *baggage] unset = await _auth_then_serve_chat(monkeypatch, {}, body, headers) mapped = await _auth_then_serve_chat( - monkeypatch, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, body, headers + monkeypatch, {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, body, headers ) assert mapped == unset - assert mapped[1] == "usecase-deployment" + assert mapped[1] == "app-deployment" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 10738eabddf..17e6c004566 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -12872,6 +12872,188 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): assert websocket.scope["state"]["budget_reservation"] is reservation +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime/translations", "/v1/realtime/translations")) +@pytest.mark.parametrize("query", (b"", b"model=gpt-realtime-translate"), ids=("default_model", "explicit_model")) +async def test_translation_websocket_auth_enforces_the_default_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str, query: bytes +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": query, + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_realtime_websocket(websocket) + + assert denied.value.status_code == 403 + assert model in str(denied.value.detail) + assert "budget" in str(denied.value.detail).lower() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets")) +async def test_translation_client_secret_auth_enforces_the_session_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + body: Final = json.dumps({"session": {"type": "translation", "model": model}}).encode() + request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test"), (b"content-type", b"application/json")], + "query_string": b"", + }, + receive=AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}), + ) + + with pytest.raises(ProxyException) as denied: + await user_api_key_auth(request=request, api_key="Bearer sk-test") + + assert denied.value.type == "budget_exceeded" + assert model in denied.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) +async def test_managed_agent_transcription_websocket_without_a_model_is_refused( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket + from litellm.types.agents import AgentResponse + + token: Final = UserAPIKeyAuth(token="test-key") + token.managed_agent_policy = AgentResponse.model_validate( + {"agent_id": "agent", "agent_name": "Agent", "agent_card_params": {}, "execution_mode": "both"} + ) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = {**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), "prisma_client": None} + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_realtime_websocket(websocket) + + assert denied.value.status_code == 403 + assert "Managed inference requires an explicit or configured model" in str(denied.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/openai_passthrough/v1/realtime", "/v1/responses", "/vertex_ai/live")) +async def test_non_realtime_websocket_auth_checks_no_default_transcription_model( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + + exhausted_model: Final = "gpt-realtime-whisper" + token: Final = UserAPIKeyAuth( + token="test-key", + model_max_budget={exhausted_model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={exhausted_model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=exhausted_model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + assert (await user_api_key_auth_websocket(websocket)).token == token.token + + @pytest.mark.asyncio async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget(): from fastapi import Request diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 5b37cee88c5..92114b00398 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -24,12 +24,12 @@ import httpx import pytest import respx from fastapi import HTTPException +from litellm_enterprise.proxy.common_utils.check_batch_cost import GIVE_UP_POLLING_AFTER import litellm from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch -from litellm_enterprise.proxy.common_utils.check_batch_cost import GIVE_UP_POLLING_AFTER if TYPE_CHECKING: from litellm.batches.batch_utils import BatchCostUsageResult @@ -254,7 +254,6 @@ class TestCheckBatchCost: @pytest.mark.asyncio async def test_column_absence_cached_across_cycles(self, check_batch_cost_instance, mock_prisma_client): """After column absence is discovered, subsequent cycles skip the primary query entirely.""" - from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) # Simulate column already known absent from a previous cycle @@ -2387,8 +2386,8 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: @pytest.mark.asyncio async def test_target_model_names_comes_from_input_file_not_provider_model(self): from litellm.proxy.openai_files_endpoints.common_utils import ( - is_base64_encoded_unified_file_id, get_models_from_unified_file_id, + is_base64_encoded_unified_file_id, ) output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP))) @@ -2422,8 +2421,8 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: @pytest.mark.asyncio async def test_falls_back_to_deployment_model_group_without_managed_input_file(self): from litellm.proxy.openai_files_endpoints.common_utils import ( - is_base64_encoded_unified_file_id, get_models_from_unified_file_id, + is_base64_encoded_unified_file_id, ) output_file_id = await self._run(self._job("file-raw-provider-input")) @@ -3309,7 +3308,9 @@ class TestMultiPodBatchCostClaim: patch( "litellm.batches.batch_utils.calculate_batch_cost_and_usage", new_callable=AsyncMock, - return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]), + return_value=_batch_cost_result( + 0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"] + ), ), patch( "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", @@ -3329,15 +3330,15 @@ class TestMultiPodBatchCostClaim: from datetime import datetime import httpx - import litellm import respx + import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger - from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.utils import PrismaClient + from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.types.utils import SpecialEnums @@ -3383,6 +3384,14 @@ class TestMultiPodBatchCostClaim: database_url="postgresql://unused:unused@localhost:5432/unused", proxy_logging_obj=dependency ) create_spend_rows: Final = AsyncMock(return_value=1) + from litellm.proxy.db.prisma_client import PrismaWrapper + + spend_prisma.db = PrismaWrapper( + original_prisma=MagicMock( + query_raw=AsyncMock(return_value=[]), + litellm_spendlogs=MagicMock(create_many=create_spend_rows), + ) + ) input_rate: Final = 0.000002 litellm.register_model( { @@ -3434,11 +3443,6 @@ class TestMultiPodBatchCostClaim: respx.mock, patch("litellm.proxy.proxy_server.proxy_logging_obj", dependency), patch("litellm.proxy.proxy_server.prisma_client", spend_prisma), - patch.object( - spend_prisma.writer_db._original_prisma, - "litellm_spendlogs", - MagicMock(create_many=create_spend_rows), - ), ): respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock( return_value=httpx.Response(200, content=content) @@ -3483,9 +3487,9 @@ class TestMultiPodBatchCostClaim: from datetime import datetime import httpx - import litellm import respx + import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.types.utils import LiteLLMBatch, SpecialEnums @@ -3587,9 +3591,9 @@ class TestMultiPodBatchCostClaim: import base64 import httpx - import litellm import respx + import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import SpecialEnums @@ -4423,9 +4427,10 @@ class TestBatchLineItemsFromThePoller: @pytest.mark.asyncio async def test_a_batch_bigger_than_one_part_reaches_the_callbacks_as_numbered_parts(self, monkeypatch): - import litellm from litellm_enterprise.proxy.common_utils import check_batch_cost + import litellm + journal: list = [] _, instance = self._setup(journal) logger = self._recording_logger(journal) @@ -4933,9 +4938,10 @@ async def test_failed_retirement_keeps_batch_available() -> None: @pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl")) @pytest.mark.parametrize("other_id", ("{id}4", "prefix-{id}", "{id}/child", "{id}.backup", "{id}?generation=2")) def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other_id: str) -> None: - from litellm.exceptions import NotFoundError from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + from litellm.exceptions import NotFoundError + error: Final = NotFoundError( message=f"No object found with id '{other_id.format(id=resource_id)}'.", model="test", llm_provider="openai" ) @@ -4958,9 +4964,10 @@ def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other ), ) def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, message: str) -> None: - from litellm.exceptions import NotFoundError from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + from litellm.exceptions import NotFoundError + error: Final = NotFoundError(message=message.format(id=resource_id), model="test", llm_provider="openai") assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is True assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True @@ -4970,9 +4977,10 @@ def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, mes ("missing_key", "gone"), (("out/job-123/input.jsonl.out", True), ("out/job-123/other.jsonl.out", False)) ) def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: str, gone: bool) -> None: - from litellm.llms.bedrock.common_utils import BedrockError from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + from litellm.llms.bedrock.common_utils import BedrockError + # S3 GetObject 404 body shape, checked live against a missing key on 2026-10-20 error: Final = BedrockError( status_code=404, @@ -4985,9 +4993,10 @@ def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: @pytest.mark.parametrize("output_file_id", (None, "")) def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None: - from litellm.exceptions import NotFoundError from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + from litellm.exceptions import NotFoundError + error: Final = NotFoundError(message="Output file not found", model="test", llm_provider="openai") assert CheckBatchCost._is_output_file_gone_at_provider(error, output_file_id) is False @@ -5003,11 +5012,12 @@ async def test_batch_cost_reaches_configured_success_callbacks( initialized: bool, claimed: bool, ) -> None: + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import LiteLLMBatch - from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost events: Final[SimpleQueue[tuple[str, object, int, int]]] = SimpleQueue() diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..a72772df492 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,120 @@ +import _hashlib +from collections.abc import Mapping +from contextlib import nullcontext +from typing import Final +from unittest.mock import Mock + +import pytest +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + openssl_enforces_fips, +) + + +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new + + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's Python OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") + assert cryptography_enforces_fips() is False + + +@pytest.mark.parametrize( + "general_settings, openssl_fips, cryptography_fips, refusal", + [ + ({}, False, False, None), + ({"fips_mode": False}, False, False, None), + ({"fips_mode": "false"}, False, False, None), + ({"fips_mode": True}, True, True, None), + ({"fips_mode": "true"}, True, True, None), + ({"fips_mode": True}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": " TRUE "}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"), + ({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"), + ({"fips_mode": None}, True, True, "general_settings.fips_mode has no value"), + ], + ids=[ + "unset", + "false", + "false-from-a-string", + "fips", + "fips-from-a-string", + "openssl-not-fips", + "openssl-not-fips-from-a-padded-string", + "cryptography-not-fips", + "not-a-boolean", + "a-number", + "no-value", + ], +) +def test_enforce_fips_mode( + monkeypatch: pytest.MonkeyPatch, + general_settings: Mapping[str, object], + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode(general_settings) diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 62189f5b0e7..0fc7bb15a32 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -65,6 +65,24 @@ def _starlette_request( return Request(scope, receive) +@pytest.mark.asyncio +@pytest.mark.parametrize("path, path_params", [ + ("/claude-code/plugins/skill/archive/digest.zip", {"plugin_name": "skill"}), + ("/vector_stores/vs-safe/files", {}), +]) +async def test_request_vector_store_values_cannot_forge_debug_log_lines( + path: str, path_params: Mapping[str, str], caplog: pytest.LogCaptureFixture, +) -> None: + untrusted: Final = "private-value\r\nFORGED LOG LINE" + request: Final = _starlette_request(b"{}", "application/json", path) + request.scope["path_params"] = dict(path_params) + with caplog.at_level("DEBUG", logger=http_parsing_utils.verbose_proxy_logger.name): + result: Final = populate_request_with_path_params({"vector_store_ids": [untrusted]}, request) + assert untrusted in result["vector_store_ids"] + assert "private-value" not in caplog.text + assert "FORGED LOG LINE" not in caplog.text + + @pytest.mark.asyncio @pytest.mark.parametrize( "body", @@ -377,7 +395,7 @@ async def test_form_data_with_json_metadata(): @pytest.mark.asyncio -async def test_form_data_with_invalid_json_metadata(): +async def test_form_data_with_invalid_json_metadata() -> None: """ Test that form data with invalid JSON in metadata field raises an exception. @@ -398,9 +416,10 @@ async def test_form_data_with_invalid_json_metadata(): mock_request.scope = {} mock_request.state._cached_headers = None - # Should raise JSONDecodeError when trying to parse invalid JSON metadata - with pytest.raises(json.JSONDecodeError): + with pytest.raises(ProxyException) as exc_info: await read_request_body(mock_request) + assert exc_info.value.code == "400" + assert exc_info.value.param == "metadata" @pytest.mark.asyncio @@ -786,6 +805,17 @@ async def test_get_form_data(): assert result["temperature"] == "0.2" assert result["include"] == ["logprobs"] assert result["timestamp_granularities"] == ["word", "segment"] + assert tuple(result) == ( + "file", + "model", + "include", + "language", + "prompt", + "response_format", + "stream", + "temperature", + "timestamp_granularities", + ) def test_get_tags_from_request_body_with_metadata_tags(): diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index 7e96ad591b7..a0fe560e6fb 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -467,6 +467,52 @@ class TestFormatDeprecationAlertMessage: class TestSuccessorModel: + def test_unmatched_successor_logs_warning_by_default(self, monkeypatch, caplog): + monkeypatch.setattr(litellm, "model_cost", {}) + router = Router( + model_list=[ + { + "model_name": "old", + "litellm_params": {"model": "openai/old-model"}, + "model_info": { + "id": "1", + "deprecation_date": "2026-06-10", + "successor_model": "missing-model", + }, + } + ] + ) + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)) + + successor_records: Final = tuple( + record for record in caplog.records if "names successor missing-model" in record.getMessage() + ) + assert len(successor_records) == 1 + assert successor_records[0].levelno == logging.WARNING + + def test_empty_team_public_model_name_is_normalized_to_none(self, monkeypatch): + monkeypatch.setattr(litellm, "model_cost", {}) + router = Router( + model_list=[ + { + "model_name": "team-model", + "litellm_params": {"model": "openai/team-model"}, + "model_info": { + "id": "1", + "team_id": "team-1", + "team_public_model_name": "", + "deprecation_date": "2026-06-10", + }, + } + ] + ) + + snapshot: Final = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)) + + assert snapshot.imminent[0].team_public_model_name is None + def test_should_surface_the_successor_set_on_the_deployment(self, monkeypatch): monkeypatch.setattr(litellm, "model_cost", {}) router = Router( @@ -572,6 +618,8 @@ class TestSuccessorModel: "typo-old": "gpt-nwe", "wildcard-old": "anthropic/x", } + team_deprecation: Final = next(info for info in snapshot.imminent if info.model_name == "model_name_t1_old") + assert team_deprecation.team_public_model_name == "team-old" warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()] assert len(warnings) == 2 assert any("global-old names successor team-new" in warning for warning in warnings) @@ -638,3 +686,70 @@ class TestSuccessorModel: assert "migrate to `<https://evil.example|new>`" in message assert "" not in message assert message.count("migrate to") == 1 + + +class TestRecommendationSources: + def test_should_return_the_cost_map_recommendation_and_provider_notice(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + "old-model": { + "deprecation_date": "2026-06-10", + "successor_model": "new-model", + "deprecation_source": "https://provider.example/model-retirements", + } + }, + ) + router = _make_router( + [{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "new-model" + assert info.successor_model_source == "model_cost_map" + assert info.deprecation_source == "https://provider.example/model-retirements" + + def test_should_keep_configured_recommendation_provenance_when_merging_deployments(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + {"old-model": {"deprecation_date": "2026-06-10", "successor_model": "catalog-pick"}}, + ) + router = _make_router( + [ + {"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}, + { + "model_name": "support", + "litellm_params": {"model": "openai/old-model"}, + "model_info": {"successor_model": "configured-pick"}, + }, + ] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "configured-pick" + assert info.successor_model_source == "deployment_config" + assert info.deprecation_source is None + + def test_should_carry_the_notice_from_the_same_deployment_as_the_selected_recommendation(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + "a": {"deprecation_date": "2026-06-10", "deprecation_source": "https://provider.example/a"}, + "b": { + "deprecation_date": "2026-06-10", + "successor_model": "b-next", + "deprecation_source": "https://provider.example/b", + }, + }, + ) + router = _make_router( + [ + {"model_name": "support", "litellm_params": {"model": "openai/a"}, "model_info": {}}, + {"model_name": "support", "litellm_params": {"model": "openai/b"}, "model_info": {}}, + ] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "b-next" + assert info.successor_model_source == "model_cost_map" + assert info.deprecation_source == "https://provider.example/b" diff --git a/tests/unit/proxy/common_utils/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py index 9e76caff054..893e39b309a 100644 --- a/tests/unit/proxy/common_utils/test_realtime_cache.py +++ b/tests/unit/proxy/common_utils/test_realtime_cache.py @@ -1,7 +1,9 @@ import json +from collections.abc import Mapping from typing import Any, Final, cast import pytest +from pydantic import TypeAdapter from litellm.proxy.common_utils.realtime_utils import realtime_request_body from litellm.proxy.proxy_server import _realtime_query_params_template @@ -41,6 +43,13 @@ def test_realtime_request_body_caches_each_model_separately(): assert gpt4o_body_first is not gpt4o_mini_body +@pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline")) +def test_realtime_request_body_preserves_model_names(model: str | None) -> None: + body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(model)) + + assert body["model"] == model + + def test_realtime_request_body_keeps_a_model_name_with_quotes_inside_its_own_field(): body: Final = realtime_request_body('gpt-4o", "user": "someone-else') diff --git a/tests/unit/proxy/common_utils/test_request_tag_mappings.py b/tests/unit/proxy/common_utils/test_request_tag_mappings.py index e1644544c46..e8e7d881e1d 100644 --- a/tests/unit/proxy/common_utils/test_request_tag_mappings.py +++ b/tests/unit/proxy/common_utils/test_request_tag_mappings.py @@ -26,7 +26,7 @@ def test_accounting_enrichment_without_request_state_preserves_legacy_metadata() @pytest.mark.asyncio -@pytest.mark.parametrize("header", [b"", b"entrypoint=a,entrypoint=b"]) +@pytest.mark.parametrize("header", [b"", b"service=a,service=b"]) @pytest.mark.parametrize("client_metadata", [None, '{"requester_metadata":{"guardrails":["qa-guard"]}}']) async def test_mapping_rejection_keeps_string_metadata_safe_for_auth_failure_logging(header, client_metadata): from litellm.utils import get_request_guardrails @@ -37,7 +37,7 @@ async def test_mapping_rejection_keeps_string_metadata_safe_for_auth_failure_log await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "required": True}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "required": True}]}, None, MagicMock(), ) @@ -59,7 +59,7 @@ async def test_invalid_mapper_metadata_maps_like_empty_metadata_and_keeps_the_cl return {"type": "http.request", "body": json.dumps(body).encode()} request = Request( - {"type": "http", "method": "POST", "path": "/v1/responses", "headers": [(b"baggage", b"entrypoint=a")]}, + {"type": "http", "method": "POST", "path": "/v1/responses", "headers": [(b"baggage", b"service=a")]}, receive, ) data, parse_error = await _read_request_body_deferring_parse_failure(request) @@ -67,11 +67,11 @@ async def test_invalid_mapper_metadata_maps_like_empty_metadata_and_keeps_the_cl await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) - assert "usecase:a" in data["litellm_metadata"]["tags"] + assert "app:a" in data["litellm_metadata"]["tags"] assert "tags" not in data[slot] or slot == "litellm_metadata" assert get_request_guardrails(data) == [] assert await read_request_body(request) == body @@ -80,60 +80,60 @@ async def test_invalid_mapper_metadata_maps_like_empty_metadata_and_keeps_the_cl @pytest.mark.parametrize( "header,value", [ - ("entrypoint=a", "a"), - ("entrypoint = a", "a"), - ("region=sg, entrypoint=customer%20support;version=1", "customer support"), + ("service=a", "a"), + ("service = a", "a"), + ("region=sg, service=customer%20support;version=1", "customer support"), ], ) def test_variable_mapper_uses_decoded_baggage_value(header, value): - rule = RequestTagMapping(key="entrypoint", tag_prefix="usecase:") - assert resolve_baggage_tag_mappings(header, [rule]) == ((f"usecase:{value}", False),) + rule = RequestTagMapping(key="service", tag_prefix="app:") + assert resolve_baggage_tag_mappings(header, [rule]) == ((f"app:{value}", False),) @pytest.mark.parametrize( "header,expected", - [("entrypoint=a", "department:support"), ("entrypoint=b", "department:sales"), ("entrypoint=x", "usecase:x")], + [("service=a", "department:support"), ("service=b", "department:sales"), ("service=x", "app:x")], ) def test_exact_rules_override_variable_fallback_independent_of_order(header, expected): rules = [ - RequestTagMapping(key="entrypoint", tag_prefix="usecase:"), - RequestTagMapping(key="entrypoint", value="a", tag="department:support"), - RequestTagMapping(key="entrypoint", value="b", tag="department:sales"), + RequestTagMapping(key="service", tag_prefix="app:"), + RequestTagMapping(key="service", value="a", tag="department:support"), + RequestTagMapping(key="service", value="b", tag="department:sales"), ] assert resolve_baggage_tag_mappings(header, rules) == ((expected, False),) assert resolve_baggage_tag_mappings(header, list(reversed(rules))) == ((expected, False),) -@pytest.mark.parametrize("header", ["entrypoint=%00", "entrypoint=a%0Ab", "entrypoint=b,entry%70oint=a"]) +@pytest.mark.parametrize("header", ["service=%00", "service=a%0Ab", "service=b,serv%69ce=a"]) def test_variable_mapper_skips_control_characters_and_percent_encoded_duplicate_keys(header): - rule = RequestTagMapping(key="entrypoint", tag_prefix="usecase:") + rule = RequestTagMapping(key="service", tag_prefix="app:") assert resolve_baggage_tag_mappings(header, [rule]) == () @pytest.mark.parametrize( "header,expected", [ - ("entry+point=a", (("usecase:a", False),)), - ("entry%2Bpoint=a", (("usecase:a", False),)), - ("entry+point=a,entry%2Bpoint=b", ()), + ("serv+ice=a", (("app:a", False),)), + ("serv%2Bice=a", (("app:a", False),)), + ("serv+ice=a,serv%2Bice=b", ()), ], ) def test_key_with_a_plus_matches_both_wire_spellings_and_rejects_both_at_once(header, expected): - rule = RequestTagMapping(key="entry+point", tag_prefix="usecase:") + rule = RequestTagMapping(key="serv+ice", tag_prefix="app:") assert resolve_baggage_tag_mappings(header, [rule]) == expected def test_exact_rules_work_without_a_variable_rule(): - rule = RequestTagMapping(key="entrypoint", value="a", tag="usecase:manual") - assert resolve_baggage_tag_mappings("entrypoint=a", [rule]) == (("usecase:manual", False),) - assert resolve_baggage_tag_mappings("entrypoint=x", [rule]) == () + rule = RequestTagMapping(key="service", value="a", tag="app:manual") + assert resolve_baggage_tag_mappings("service=a", [rule]) == (("app:manual", False),) + assert resolve_baggage_tag_mappings("service=x", [rule]) == () @pytest.mark.parametrize( - "header", ["", "region=sg", "entrypoint=", "entrypoint=a,entrypoint=b", "entrypoint=b,entry%70oint=a", "entrypoint=x"] + "header", ["", "region=sg", "service=", "service=a,service=b", "service=b,serv%69ce=a", "service=x"] ) def test_required_mapping_rejects_missing_ambiguous_or_unmapped_values(header): - rule = RequestTagMapping(key="entrypoint", value="a", tag="usecase:a", required=True) + rule = RequestTagMapping(key="service", value="a", tag="app:a", required=True) with pytest.raises(ProxyException) as error: resolve_baggage_tag_mappings(header, [rule]) assert error.value.code == "400" @@ -142,10 +142,10 @@ def test_required_mapping_rejects_missing_ambiguous_or_unmapped_values(header): @pytest.mark.parametrize( "rule", [ - {"key": "entrypoint", "tag": "usecase:a"}, - {"key": "entrypoint", "value": "a", "tag_prefix": "usecase:"}, - {"key": "entrypoint", "value": "a", "tag": "usecase:a", "tag_prefix": "usecase:"}, - {"key": "", "tag_prefix": "usecase:"}, + {"key": "service", "tag": "app:a"}, + {"key": "service", "value": "a", "tag_prefix": "app:"}, + {"key": "service", "value": "a", "tag": "app:a", "tag_prefix": "app:"}, + {"key": "", "tag_prefix": "app:"}, ], ) def test_invalid_rule_is_rejected_during_config_validation(rule): @@ -158,8 +158,8 @@ def test_duplicate_fallbacks_are_rejected(): ConfigGeneralSettings.model_validate( { "request_tag_mappings": [ - {"key": "entrypoint", "tag_prefix": "one:"}, - {"key": "entrypoint", "tag_prefix": "two:"}, + {"key": "service", "tag_prefix": "one:"}, + {"key": "service", "tag_prefix": "two:"}, ] } ) @@ -171,23 +171,23 @@ def test_duplicate_fallbacks_are_rejected(): [("/chat/completions", "metadata"), ("/v1/messages", "litellm_metadata"), ("/v1/responses", "litellm_metadata")], ) async def test_mapped_tag_merges_with_existing_tags_in_the_budget_metadata_slot(route, slot): - data = {slot: {"tags": ["existing", "usecase:a"]}} - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + data = {slot: {"tags": ["existing", "app:a"]}} + request = Request({"type": "http", "headers": [(b"baggage", b"service=a")]}) LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(data, route) await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) - assert data[slot]["tags"] == ["existing", "usecase:a"] + assert data[slot]["tags"] == ["existing", "app:a"] @pytest.mark.asyncio async def test_budgeted_mapper_requires_a_database(): - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) - config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + request = Request({"type": "http", "headers": [(b"baggage", b"service=a")]}) + config = {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}]} with pytest.raises(ProxyException) as error: await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, {}, config, None, MagicMock() @@ -197,10 +197,10 @@ async def test_budgeted_mapper_requires_a_database(): @pytest.mark.asyncio async def test_budgeted_mapper_answers_503_when_the_tag_budget_lookup_fails(): - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + request = Request({"type": "http", "headers": [(b"baggage", b"service=a")]}) prisma_client = MagicMock() prisma_client.db.litellm_tagtable.find_many = AsyncMock(side_effect=Exception("database unreachable")) - config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + config = {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}]} with pytest.raises(ProxyException, match="could not read tag budgets") as error: await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, {"metadata": {}}, config, prisma_client, UserApiKeyCache() @@ -210,16 +210,16 @@ async def test_budgeted_mapper_answers_503_when_the_tag_budget_lookup_fails(): @pytest.mark.asyncio async def test_budgeted_mapper_serves_through_a_db_outage_when_requests_are_allowed_on_db_unavailable(): - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + request = Request({"type": "http", "headers": [(b"baggage", b"service=a")]}) prisma_client = MagicMock() prisma_client.db.litellm_tagtable.find_many = AsyncMock(side_effect=Exception("database unreachable")) data = {"metadata": {}} - config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + config = {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}]} with patch.dict("litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": True}): await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, config, prisma_client, UserApiKeyCache() ) - assert data["metadata"]["tags"] == ["usecase:a"] + assert data["metadata"]["tags"] == ["app:a"] async def _cache_with_tag(tag: LiteLLM_TagTable) -> UserApiKeyCache: @@ -234,11 +234,11 @@ async def _cache_with_tag(tag: LiteLLM_TagTable) -> UserApiKeyCache: [None, LiteLLM_BudgetTable(budget_id="qa-budget", max_budget=None, tpm_limit=10)], ) async def test_budgeted_mapper_rejects_a_tag_without_a_max_budget(budget_table): - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + request = Request({"type": "http", "headers": [(b"baggage", b"service=a")]}) data = {"metadata": {}} - cache = await _cache_with_tag(LiteLLM_TagTable(tag_name="usecase:a", litellm_budget_table=budget_table)) - config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} - with pytest.raises(ProxyException, match="usecase:a requires an existing tag budget") as error: + cache = await _cache_with_tag(LiteLLM_TagTable(tag_name="app:a", litellm_budget_table=budget_table)) + config = {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}]} + with pytest.raises(ProxyException, match="app:a requires an existing tag budget") as error: await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, config, MagicMock(), cache ) @@ -248,23 +248,23 @@ async def test_budgeted_mapper_rejects_a_tag_without_a_max_budget(budget_table): @pytest.mark.asyncio async def test_budgeted_mapper_admits_a_tag_with_a_max_budget_and_unbudgeted_rules_need_none(): - request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a,region=sg")]}) + request = Request({"type": "http", "headers": [(b"baggage", b"service=a,region=sg")]}) data = {"metadata": {}} cache = await _cache_with_tag( LiteLLM_TagTable( - tag_name="usecase:a", litellm_budget_table=LiteLLM_BudgetTable(budget_id="qa-budget", max_budget=1.0) + tag_name="app:a", litellm_budget_table=LiteLLM_BudgetTable(budget_id="qa-budget", max_budget=1.0) ) ) config = { "request_tag_mappings": [ - {"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}, + {"key": "service", "tag_prefix": "app:", "require_budgeted_tag": True}, {"key": "region", "tag_prefix": "region:"}, ] } await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, config, MagicMock(), cache ) - assert data["metadata"]["tags"] == ["usecase:a", "region:sg"] + assert data["metadata"]["tags"] == ["app:a", "region:sg"] @pytest.mark.parametrize( @@ -325,7 +325,7 @@ async def test_mapping_survives_cached_body_read_without_leaking_auth_query_fiel "scheme": "http", "server": ("localhost", 4000), "query_string": query, - "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + "headers": [(b"content-type", b"application/json"), (b"baggage", b"service=a")], }, receive, ) @@ -337,7 +337,7 @@ async def test_mapping_survives_cached_body_read_without_leaking_auth_query_fiel await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( request, data, - {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock(), ) @@ -350,7 +350,7 @@ async def test_mapping_survives_cached_body_read_without_leaking_auth_query_fiel proxy_config=ProxyConfig(), general_settings={}, ) - assert enriched[slot]["tags"] == ["usecase:a"] + assert enriched[slot]["tags"] == ["app:a"] assert "key" not in enriched assert "api-version" not in enriched assert "key" not in enriched["proxy_server_request"]["body"] @@ -360,7 +360,7 @@ async def test_mapping_survives_cached_body_read_without_leaking_auth_query_fiel @pytest.mark.asyncio @pytest.mark.parametrize( "client_metadata", - [{"trace_name": "qa"}, {"trace_name": "qa", "tags": ["client:x"]}, {"trace_name": "qa", "tags": ["usecase:a"]}], + [{"trace_name": "qa"}, {"trace_name": "qa", "tags": ["client:x"]}, {"trace_name": "qa", "tags": ["app:a"]}], ) async def test_mapped_tags_stay_out_of_requester_metadata(client_metadata): from litellm.proxy._types import UserAPIKeyAuth @@ -380,14 +380,14 @@ async def test_mapped_tags_stay_out_of_requester_metadata(client_metadata): "scheme": "http", "server": ("localhost", 4000), "query_string": b"", - "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + "headers": [(b"content-type", b"application/json"), (b"baggage", b"service=a")], }, receive, ) data, parse_error = await _read_request_body_deferring_parse_failure(request) assert parse_error is None await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( - request, data, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, None, MagicMock() + request, data, {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock() ) enriched = await add_litellm_data_to_request( data=await read_request_body(request), @@ -397,7 +397,7 @@ async def test_mapped_tags_stay_out_of_requester_metadata(client_metadata): general_settings={}, ) requester_metadata = enriched["metadata"]["requester_metadata"] - assert "usecase:a" in enriched["metadata"]["tags"] + assert "app:a" in enriched["metadata"]["tags"] assert requester_metadata["trace_name"] == "qa" assert requester_metadata.get("tags") == client_metadata.get("tags") assert BAGGAGE_MAPPED_TAGS_METADATA_KEY not in requester_metadata @@ -407,9 +407,9 @@ async def test_mapped_tags_stay_out_of_requester_metadata(client_metadata): @pytest.mark.parametrize( "headers,body_fields", [ - ([(b"x-litellm-tags", b"usecase:a")], {}), - ([], {"tags": ["usecase:a"]}), - ([], {"litellm_metadata": {"tags": ["usecase:a"]}}), + ([(b"x-litellm-tags", b"app:a")], {}), + ([], {"tags": ["app:a"]}), + ([], {"litellm_metadata": {"tags": ["app:a"]}}), ], ids=["header", "body", "litellm_metadata"], ) @@ -432,14 +432,14 @@ async def test_client_sent_tag_still_routes_when_baggage_maps_to_it(headers, bod "scheme": "http", "server": ("localhost", 4000), "query_string": b"", - "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a"), *headers], + "headers": [(b"content-type", b"application/json"), (b"baggage", b"service=a"), *headers], }, receive, ) data, parse_error = await _read_request_body_deferring_parse_failure(request) assert parse_error is None await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( - request, data, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, None, MagicMock() + request, data, {"request_tag_mappings": [{"key": "service", "tag_prefix": "app:"}]}, None, MagicMock() ) enriched = await add_litellm_data_to_request( data=await read_request_body(request), @@ -452,8 +452,8 @@ async def test_client_sent_tag_still_routes_when_baggage_maps_to_it(headers, bod model_list=[ { "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4o", "tags": ["usecase:a"]}, - "model_info": {"id": "usecase-deployment"}, + "litellm_params": {"model": "gpt-4o", "tags": ["app:a"]}, + "model_info": {"id": "app-deployment"}, }, { "model_name": "gpt-4", @@ -470,4 +470,4 @@ async def test_client_sent_tag_still_routes_when_baggage_maps_to_it(headers, bod metadata=dict(enriched["metadata"]), mock_response="hi", ) - assert response._hidden_params["model_id"] == "usecase-deployment" + assert response._hidden_params["model_id"] == "app-deployment" diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 3d8e92988d9..cdd18fd4eef 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -3166,18 +3166,34 @@ class TestCredentialDisplayName: repository.update_by_name.assert_not_awaited() assert litellm.credential_list == [_labeled_credential()] - def test_patch_accepts_the_same_credential_name_echoed_back(self, restore_credential_list, monkeypatch): + @pytest.mark.parametrize( + ("patch_info", "expected_info"), + [ + ({}, {"custom_llm_provider": "openai"}), + ({"description": "y"}, {"custom_llm_provider": "openai", "description": "y"}), + ({"custom_llm_provider": "anthropic"}, {"custom_llm_provider": "anthropic"}), + ], + ) + def test_patch_accepts_the_same_credential_name_echoed_back( + self, restore_credential_list, monkeypatch, patch_info: dict[str, str], expected_info: dict[str, str] + ): monkeypatch.setattr(litellm, "credential_list", [_labeled_credential()]) with _repository_holding(_labeled_credential()) as repository: response = _patch_credential( "openai-prod", - {"credential_name": "openai-prod", "credential_values": {"api_key": "sk-new"}, "credential_info": {}}, + { + "credential_name": "openai-prod", + "credential_values": {"api_key": "sk-new"}, + "credential_info": patch_info, + }, ) assert response.status_code == 200, response.text assert [(c.credential_name, c.credential_values) for c in litellm.credential_list] == [ ("openai-prod", {"api_key": "sk-new"}) ] + assert json.loads(repository.update_by_name.await_args.kwargs["data"]["credential_info"]) == expected_info + assert litellm.credential_list[0].credential_info == expected_info def test_patch_on_a_config_credential_is_rejected_as_config_owned(self, restore_credential_list, monkeypatch): config_credential = _labeled_credential(display_name=None).model_copy(update={"source": "config"}) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py index 0a015255f2e..814fdcb4eda 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py @@ -2,7 +2,6 @@ Unit tests for ToolDiscoveryQueue. """ - import pytest from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( @@ -63,3 +62,19 @@ def test_aggregate_sums_call_counts_across_workers_flushes(): {"tool_name": "get_weather", "origin": "user_defined", "call_count": 5}, {"tool_name": "search", "call_count": 1}, ) + + +def test_requeued_rows_go_ahead_of_and_merge_with_sightings_since_the_flush(queue: ToolDiscoveryQueue) -> None: + queue.add_update({"tool_name": "get_weather", "origin": "mcp_server"}) + queue.add_update({"tool_name": "search", "origin": "user_defined"}) + failed_write = queue.flush() + queue.add_update({"tool_name": "translate", "origin": "user_defined"}) + queue.add_update({"tool_name": "search", "origin": "user_defined"}) + + queue.requeue(failed_write) + + assert queue.flush() == [ + {"tool_name": "get_weather", "origin": "mcp_server", "call_count": 1}, + {"tool_name": "search", "origin": "user_defined", "call_count": 2}, + {"tool_name": "translate", "origin": "user_defined", "call_count": 1}, + ] diff --git a/tests/unit/proxy/db/fake_prisma_engine.py b/tests/unit/proxy/db/fake_prisma_engine.py index 4221eeced5a..082659a3adc 100644 --- a/tests/unit/proxy/db/fake_prisma_engine.py +++ b/tests/unit/proxy/db/fake_prisma_engine.py @@ -2,8 +2,10 @@ marking the DB I/O witness the way ``_TrackedPrismaEngine`` does, so the producer under test emits its service event.""" -from typing import TypeVar -from unittest.mock import AsyncMock +from typing import Final, TypeVar +from unittest.mock import AsyncMock, MagicMock + +from prisma.errors import DataError from litellm.proxy.db.log_db_metrics import record_db_io @@ -16,3 +18,73 @@ def engine_call(return_value: _T | None = None) -> AsyncMock: return return_value return AsyncMock(side_effect=run) + + +class RecordingReadTransaction: + """The transaction ``query_raw_within_statement_timeout`` opens on ``read_db``, keeping its ``SET LOCAL`` statements.""" + + def __init__(self, query_raw: AsyncMock) -> None: + self.statements: tuple[str, ...] = () + self.query_raw: AsyncMock = query_raw + + async def execute_raw(self, sql: str) -> int: + self.statements = (*self.statements, sql) + return 0 + + async def __aenter__(self) -> "RecordingReadTransaction": + return self + + async def __aexit__(self, *exc_info: object) -> None: + pass + + +def statement_timeout_reads(prisma: MagicMock, query_raw: AsyncMock) -> RecordingReadTransaction: + """Make ``prisma.read_db.tx()`` hand out one recording transaction whose raw reads answer with ``query_raw``.""" + transaction: Final = RecordingReadTransaction(query_raw) + prisma.read_db.tx = MagicMock(return_value=transaction) + return transaction + + +def statement_timeout_error() -> DataError: + """The error prisma raises when Postgres cancels a statement under ``statement_timeout`` (SQLSTATE 57014).""" + message: Final = "canceling statement due to statement timeout" + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": f"Error querying the database: {message}", + "meta": {"code": "57014", "message": message}, + } + } + ) + + +class _DelegatingReadTransaction: + """``read_db.tx()`` of a hand-written prisma fake: raw reads go to the fake's ``db.query_raw``.""" + + def __init__(self, db: object) -> None: + self.db: Final = db + self.statements: tuple[str, ...] = () + + async def execute_raw(self, sql: str) -> int: + self.statements = (*self.statements, sql) + return 0 + + async def query_raw(self, sql: str, *args: object) -> object: + return await getattr(self.db, "query_raw")(sql, *args) + + async def __aenter__(self) -> "_DelegatingReadTransaction": + return self + + async def __aexit__(self, *exc_info: object) -> None: + pass + + +class ReadTargetOf: + """What a prisma fake returns from ``read_db``: ``tx()`` opens a transaction over the fake's ``db``.""" + + def __init__(self, db: object) -> None: + self.db: Final = db + + def tx(self, **kwargs: object) -> _DelegatingReadTransaction: + return _DelegatingReadTransaction(self.db) diff --git a/tests/unit/proxy/db/test_accounting_ledger.py b/tests/unit/proxy/db/test_accounting_ledger.py deleted file mode 100644 index 6b301146301..00000000000 --- a/tests/unit/proxy/db/test_accounting_ledger.py +++ /dev/null @@ -1,149 +0,0 @@ -import json -from datetime import datetime, timezone -from decimal import Decimal -from typing import Final, Literal, NoReturn - -import pytest - -from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome -from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingLedger, UsageWrite -from litellm.types.utils import CallTypes - - -def _no_transaction() -> NoReturn: - raise AssertionError("Invalid accounting input must not open a database transaction") - - -@pytest.fixture -def event() -> InboundUsageEvent: - return InboundUsageEvent( - event_id="caller", - request_id="caller", - call_type=CallTypes.completion, - started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), - ended_at=datetime(2026, 1, 1, tzinfo=timezone.utc), - outcome=Outcome.SUCCEEDED, - dimensions=(), - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("envelope", ["{}", "[]", '{"event":null}', '{"event":{}}', "not json"]) -async def test_invalid_envelopes_fail_before_database_access(event: InboundUsageEvent, envelope: str) -> None: - with pytest.raises(ValueError, match=r"Accounting envelope|validation error"): - await AccountingLedger(_no_transaction).insert_event(event, (), envelope=envelope) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("version", [0, -1, True, 1.5]) -@pytest.mark.parametrize("field", ["version", "schema_version"]) -async def test_invalid_versions_fail_before_database_access( - event: InboundUsageEvent, version: int | float, field: Literal["version", "schema_version"] -) -> None: - with pytest.raises(ValueError, match="positive integers"): - await AccountingLedger(_no_transaction).insert_event( - event, - (), - envelope="{}", - **{field: version}, # pyright: ignore[reportArgumentType] # Exercise invalid runtime versions - ) - - -@pytest.mark.asyncio -async def test_constructed_invalid_events_are_revalidated(event: InboundUsageEvent) -> None: - invalid: Final = event.model_copy(update={"event_id": ""}) - with pytest.raises(ValueError, match="event_id"): - await AccountingLedger(_no_transaction).insert_event(invalid, (), envelope="{}") - - -@pytest.mark.asyncio -@pytest.mark.parametrize("limit", [0, -1, 1001, True, 1.5]) -async def test_retention_rejects_invalid_batch_bounds(limit: int | float) -> None: - with pytest.raises(ValueError, match="batch size"): - await AccountingLedger(_no_transaction).purge_completed( - limit=limit # pyright: ignore[reportArgumentType] # Exercise invalid runtime batch sizes - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("missing", ["both", "redelivery", "recovery"]) -async def test_retention_requires_both_safety_watermarks(missing: Literal["both", "redelivery", "recovery"]) -> None: - cutoff: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - deleted: Final = await AccountingLedger(_no_transaction).purge_completed( - redelivery_before=cutoff if missing == "recovery" else None, - recovery_before=cutoff if missing == "redelivery" else None, - ) - assert deleted == 0 - - -@pytest.mark.asyncio -async def test_retention_rejects_ambiguous_naive_watermarks() -> None: - with pytest.raises(ValueError, match="timezone"): - await AccountingLedger(_no_transaction).purge_completed( - redelivery_before=datetime(2026, 1, 1), recovery_before=datetime(2026, 1, 1, tzinfo=timezone.utc) - ) - - -@pytest.mark.asyncio -async def test_nonfinite_context_is_rejected_before_database_access(event: InboundUsageEvent) -> None: - envelope: Final = json.dumps({"event": event.model_dump(mode="json"), "context": {"price": float("nan")}}) - with pytest.raises(ValueError, match="Out of range float"): - await AccountingLedger(_no_transaction).insert_event(event, (), envelope=envelope) - - -@pytest.mark.asyncio -async def test_unknown_delivery_target_is_rejected_before_database_access(event: InboundUsageEvent) -> None: - envelope: Final = json.dumps({"event": event.model_dump(mode="json")}) - with pytest.raises(ValueError, match="not a valid DeliveryTarget"): - await AccountingLedger(_no_transaction).insert_event( - event, - ("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target - envelope=envelope, - ) - - -@pytest.mark.asyncio -async def test_empty_usage_batches_do_not_open_a_transaction() -> None: - ledger: Final = AccountingLedger(_no_transaction) - await ledger.insert_usage_batch(()) - assert await ledger.load_usage_batch(()) == () - - -@pytest.mark.asyncio -async def test_conflicting_usage_within_one_batch_is_rejected_before_database_access(event: InboundUsageEvent) -> None: - original: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) - changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}})) - with pytest.raises(AccountingConflictError, match="conflicting copies"): - await AccountingLedger(_no_transaction).insert_usage_batch((original, changed)) - - -@pytest.mark.asyncio -async def test_loaded_usage_preserves_decimal_metrics_from_jsonb(event: InboundUsageEvent) -> None: - from collections.abc import AsyncIterator - from contextlib import asynccontextmanager - from typing import cast - from unittest.mock import AsyncMock - - from prisma import Prisma - - from litellm.accounting import OutboundUsageEvent - - outbound: Final = OutboundUsageEvent( - **event.model_dump(exclude={"metrics"}), - attempt_id="attempt", - retry_of_attempt_id=None, - metrics=(Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),), - ) - payload: Final = {"event": outbound.model_dump(mode="json"), "usage": {"prompt_tokens": 120000}} - database: Final = AsyncMock(spec=Prisma) - database.query_raw.return_value = [{"payload": payload, "targets": []}] - - @asynccontextmanager - async def transaction() -> AsyncIterator[Prisma]: - yield cast(Prisma, database) - - recovered: Final = await AccountingLedger(transaction).load_usage_batch((outbound.event_id,)) - assert len(recovered) == 1 - assert recovered[0].event == outbound - assert json.loads(recovered[0].envelope) == payload diff --git a/tests/unit/proxy/db/test_budget_window_spend_writer.py b/tests/unit/proxy/db/test_budget_window_spend_writer.py index 27f1472c183..7315d51c5dd 100644 --- a/tests/unit/proxy/db/test_budget_window_spend_writer.py +++ b/tests/unit/proxy/db/test_budget_window_spend_writer.py @@ -407,9 +407,7 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o @pytest.mark.asyncio -@pytest.mark.parametrize( - ("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))] -) +@pytest.mark.parametrize(("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]) async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param): db = _FakeDB() @@ -425,7 +423,9 @@ async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(re ((query, params),) = db.execute_raw_calls normalized = " ".join(query.split()) assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized - assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')") + assert normalized.endswith( + "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')" + ) assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param) @@ -561,7 +561,12 @@ async def test_seed_aggregate_sql_splits_the_window_at_the_batch_start(entity_ty ((query, params),) = db.query_raw_calls normalized = " ".join(query.split()) assert expected_column in normalized - assert "FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC'))" in normalized + assert ( + 'FILTER (WHERE EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery ' + 'WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id) ' + "OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC'))" + in normalized + ) assert 'FROM "LiteLLM_SpendLogs"' in normalized # startTime is TIMESTAMP(3): the bound is floored to the second so the # batch's own earliest row cannot round under it. @@ -635,3 +640,21 @@ async def test_rolling_a_window_row_renders_a_postgres_update_span( new_window_start=WINDOW_B, ) assert await postgres_span_names() == ("postgres.update LiteLLM_BudgetWindowSpend",) + + +@pytest.mark.asyncio +async def test_window_reads_inside_an_enclosing_transaction_use_its_connection(): + client_db = _FakeDB() + transaction = _FakeDB(existing_rows=[]) + + await commit_window_spend_updates( + prisma_client=_FakePrismaClient(client_db), + transactions=( + build_window_spend_transaction("key", "k1", "30d", WINDOW_A, 1.0, started_at=BATCH_STARTED_AT), + ), + outer_transaction=transaction, + ) + + assert client_db.query_raw_calls == [] + assert len(transaction.query_raw_calls) == 2 + assert len(transaction.batcher.calls) == 1 diff --git a/tests/unit/proxy/db/test_db_lookup_gate.py b/tests/unit/proxy/db/test_db_lookup_gate.py index 68903170840..3450d3e05b8 100644 --- a/tests/unit/proxy/db/test_db_lookup_gate.py +++ b/tests/unit/proxy/db/test_db_lookup_gate.py @@ -46,21 +46,37 @@ async def test_bounded_db_lookup_returns_a_prompt_answer_without_recording_a_sta assert tracker.stalled_within(30) is False +class _PendingLookups: + def __init__(self) -> None: + self.pending = 0 + self.peak = 0 + + async def never_answers(self) -> None: + self.pending += 1 + self.peak = max(self.peak, self.pending) + try: + await asyncio.Event().wait() + finally: + self.pending -= 1 + + @pytest.mark.asyncio async def test_bounded_db_lookup_fails_a_whole_stalled_burst_within_one_deadline(): tracker: Final = DBLookupStallTracker() burst: Final = 200 - started: Final = time.monotonic() + lookups: Final = _PendingLookups() results: Final = await asyncio.gather( *( - bounded_db_lookup(_never_answers(), name=f"key-{i}", deadline_seconds=0.1, tracker=tracker) + bounded_db_lookup(lookups.never_answers(), name=f"key-{i}", deadline_seconds=0.1, tracker=tracker) for i in range(burst) ), return_exceptions=True, ) - assert time.monotonic() - started < 2 + assert lookups.peak == burst, ( + f"only {lookups.peak} of {burst} stalled lookups were waiting on their deadline together" + ) assert len(results) == burst assert all(isinstance(result, DBLookupDeadlineExceeded) for result in results) assert tracker.stalled_within(30) is True diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 298b153f621..d90c7f6d21d 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -1,10 +1,12 @@ import asyncio import copy +import functools +import itertools import json import logging import re -from collections.abc import AsyncIterator, Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace @@ -19,6 +21,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger + from litellm._service_logger import ServiceTypes from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem @@ -4380,7 +4383,6 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bri assert prisma.spend_log_flush_requested.is_set() is expects_flush - @pytest.mark.asyncio @pytest.mark.timeout(30) async def test_a_chatgpt_responses_row_queues_without_a_default_chatgpt_login( @@ -4422,6 +4424,8 @@ def _batch_cost_payload() -> dict: def _spend_logs_prisma(inserted: int, existing: object, taken_over: int = 1) -> MagicMock: prisma = _tool_usage_prisma() prisma.jsonify_object = lambda data: dict(data) + prisma.writer_db = prisma.db + prisma.db.query_raw = AsyncMock(return_value=[]) prisma.db.litellm_spendlogs.create_many = AsyncMock(return_value=inserted) prisma.db.litellm_spendlogs.find_unique = AsyncMock(return_value=existing) prisma.db.litellm_spendlogs.update_many = AsyncMock(return_value=taken_over) @@ -4456,7 +4460,7 @@ async def _update_database_with( end_user_id=None, team_id=None, org_id=None, - kwargs={"model": "gpt-5.6-luna", "call_type": "aretrieve_batch"}, + kwargs={"model": "gpt-5.6-luna", "call_type": payload["call_type"]}, completion_response=None, start_time=datetime.now(timezone.utc), end_time=datetime.now(timezone.utc), @@ -4707,6 +4711,392 @@ async def test_update_database_queues_every_other_spend_row_for_the_next_flush(p assert db_writer._batch_database_updates.await_count == 1 +def _video_status_payload(request_id: str = "video_abc_video_cost") -> dict: + return { + **_minimal_spend_payload(), + "request_id": request_id, + "call_type": "avideo_status", + "status": "success", + } + + +class _SpendLogsTable: + """LiteLLM_SpendLogs keyed by its request_id primary key, as the writer's three queries see it.""" + + def __init__(self) -> None: + self.rows: dict[str, dict] = {} + + async def create_many(self, data: list[dict], skip_duplicates: bool) -> int: + fresh = {row["request_id"]: dict(row) for row in data if row["request_id"] not in self.rows} + self.rows.update(fresh) + return len(fresh) + + async def find_unique(self, where: dict) -> SimpleNamespace | None: + row = self.rows.get(where["request_id"]) + return None if row is None else SimpleNamespace(**row) + + async def update_many(self, data: dict, where: dict) -> int: + row = self.rows.get(where["request_id"]) + if row is None or any(row.get(field) != value for field, value in where.items()): + return 0 + row.update(data) + return 1 + + +def _spend_logs_table_prisma() -> MagicMock: + prisma = _tool_usage_prisma() + prisma.jsonify_object = lambda data: dict(data) + prisma.db.litellm_spendlogs = _SpendLogsTable() + prisma.writer_db.query_raw = AsyncMock(return_value=[]) + return prisma + + +async def _flush_queued_spend_logs(prisma: MagicMock) -> None: + await prisma.db.litellm_spendlogs.create_many(data=prisma.spend_log_transactions, skip_duplicates=True) + prisma.spend_log_transactions.clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "flushes_before_completion", + [0, 1], + ids=["rendering_poll_still_queued", "rendering_poll_already_written"], +) +async def test_update_database_charges_a_video_once_from_the_poll_that_sees_it_completed( + flushes_before_completion: int, +): + """ + A custom video provider reports the video's cost on the status poll that sees it completed, + and clients poll a finished video again before they download it, so only that first completed + poll may charge, and its spend row has to carry the charge (LIT-10404). The $0 poll written + while the video was still rendering keeps the bare video id, so it lands on the create row. + """ + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + rendering = await _update_database_with(db_writer, prisma, _video_status_payload("video_abc"), response_cost=0.0) + for _ in range(flushes_before_completion): + await _flush_queued_spend_logs(prisma) + completed = await _update_database_with(db_writer, prisma, _video_status_payload(), response_cost=0.42) + repolled = await _update_database_with(db_writer, prisma, _video_status_payload(), response_cost=0.42) + await _flush_queued_spend_logs(prisma) + + assert (rendering, completed, repolled) == (True, True, False) + assert db_writer._batch_database_updates.await_count == 2 + rows = prisma.db.litellm_spendlogs.rows + assert {request_id: (row["call_type"], row["spend"]) for request_id, row in rows.items()} == { + "video_abc": ("avideo_status", 0.0), + "video_abc_video_cost": ("avideo_status", 0.42), + } + + +def _indexed_guardrail_request_ids(prisma: MagicMock, index_table: str) -> tuple[str, ...]: + calls: Final = getattr(prisma.db, index_table).create_many.await_args_list + batches: Final = tuple(call.kwargs["data"] for call in calls) + return tuple(row["request_id"] for row in itertools.chain.from_iterable(batches)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_spend_logs", [False, True], ids=["spend_logs_on", "spend_logs_off"]) +@pytest.mark.parametrize("left_uncharged", [False, True], ids=["claimed_row", "taken_over_row"]) +async def test_a_job_cost_row_counts_its_polls_guardrail_run_in_guardrail_usage_once( + disable_spend_logs: bool, left_uncharged: bool +): + """The job's cost row is written by its claim, not the spend log queue whose flush counts guardrail runs, so a + finished video's guardrail run must still reach guardrail usage, once, and not when spend logs are off.""" + from litellm.repositories.table_repositories import ( + DailyGuardrailMetricsRepository, + SpendLogGuardrailIndexRepository, + ) + + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + getattr(prisma.db, SpendLogGuardrailIndexRepository.table_name).create_many = AsyncMock(return_value=1) + getattr(prisma.db, DailyGuardrailMetricsRepository.table_name).upsert = AsyncMock() + if left_uncharged: + prisma.db.litellm_spendlogs.rows["video_abc_video_cost"] = {**_video_status_payload(), "spend": 0.0} + guarded_poll: Final = { + **_video_status_payload(), + "metadata": json.dumps( + {"guardrail_information": [{"guardrail_name": "priced", "guardrail_status": "success"}]} + ), + } + + charged = [ + await _update_database_with( + db_writer, prisma, guarded_poll, disable_spend_logs=disable_spend_logs, response_cost=0.43 + ) + for _ in range(2) + ] + + assert charged == [True, False] + assert _indexed_guardrail_request_ids(prisma, SpendLogGuardrailIndexRepository.table_name) == ( + () if disable_spend_logs else ("video_abc_video_cost",) + ) + + +def _job_poll( + litellm_call_id: str, + response_cost: float, + status: str = "completed", + polled_video_id: str = "video_abc", + guardrail_cost: float = 0.0, + call_type: str = "avideo_status", +) -> tuple[dict[str, object], object]: + """The logging kwargs a job poll reaches the spend writer with: litellm logs the provider's own cost as + response_cost and the guardrails' on top of it only in the standard logging payload.""" + from litellm.types.utils import LiteLLMBatch + from litellm.types.videos.main import VideoObject + + polled_job = ( + VideoObject(id="video_abc", object="video", status=status) + if call_type == "avideo_status" + else LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="f", + object="batch", + status=status, + output_file_id="file-out", + ) + ) + kwargs: Final[dict[str, object]] = { + "model": "sora-2", + "call_type": call_type, + "litellm_call_id": litellm_call_id, + "response_cost": response_cost - guardrail_cost, + "polled_video_status": status, + "litellm_params": {"metadata": {"user_api_key": "test-key"}, "video_id": polled_video_id}, + **( + { + "standard_logging_object": { + "response_cost": response_cost, + "guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": guardrail_cost}], + "metadata": {}, + "hidden_params": {}, + "request_tags": [], + "model_map_information": None, + } + } + if guardrail_cost + else {} + ), + } + return kwargs, polled_job + + +def _spend_writer_call(db_writer: DBSpendUpdateWriter, polled_job: object) -> Callable[..., Awaitable[bool]]: + return functools.partial( + db_writer.update_database, + token="test-token", + user_id="test-user", + end_user_id=None, + team_id=None, + org_id=None, + completion_response=polled_job, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), + ) + + +@contextmanager +def _proxy_globals(prisma: MagicMock) -> Iterator[None]: + with ( + patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam + "litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget" + ), + ): + yield + + +async def _update_database_for_video_poll( + db_writer: DBSpendUpdateWriter, + prisma: MagicMock, + litellm_call_id: str, + response_cost: float, + status: str = "completed", + polled_video_id: str = "video_abc", + guardrail_cost: float = 0.0, + call_type: str = "avideo_status", +) -> bool: + kwargs, polled_job = _job_poll(litellm_call_id, response_cost, status, polled_video_id, guardrail_cost, call_type) + with _proxy_globals(prisma): + charged = await _spend_writer_call(db_writer, polled_job)(kwargs=kwargs, response_cost=response_cost) + await asyncio.sleep(0) + return charged + + +@pytest.mark.asyncio +async def test_update_database_charges_the_completed_polls_cost_and_not_a_running_polls(): + """A provider that also prices a running video must still have the completed poll's cost billed, once.""" + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + running = await _update_database_for_video_poll(db_writer, prisma, "poll-1", 0.1, status="in_progress") + completed = await _update_database_for_video_poll(db_writer, prisma, "poll-2", 0.42) + repolled = await _update_database_for_video_poll(db_writer, prisma, "poll-3", 0.42) + await _flush_queued_spend_logs(prisma) + + assert (running, completed, repolled) == (True, True, False) + assert prisma.db.litellm_spendlogs.rows["video_abc_video_cost"]["spend"] == 0.42 + + +@pytest.mark.asyncio +async def test_a_poll_paying_only_its_guardrail_cannot_take_the_jobs_charge_whatever_call_id_the_client_sent(): + """A client picks its own x-litellm-call-id, so a guardrail-only video poll naming the job's charge row, once or + twice, must still be charged each time and leave the finished job's charge to the poll that sees it done. An + unfinished batch retrieve never reaches the writer, and its row is named by the batch id the provider sent.""" + call_type: Final = "avideo_status" + charge_row_id: Final = "video_abc_video_cost" + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + charged = [ + await _update_database_for_video_poll( + db_writer, prisma, call_id, cost, status=status, guardrail_cost=0.01, call_type=call_type + ) + for call_id, cost, status in ( + (charge_row_id, 0.01, "in_progress"), + (charge_row_id, 0.01, "in_progress"), + ("poll-3", 0.43, "completed"), + ) + ] + await _flush_queued_spend_logs(prisma) + + assert charged == [True, True, True] + assert prisma.db.litellm_spendlogs.rows[charge_row_id]["spend"] == 0.43 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "charge_row_id"), + [("avideo_status", "video_abc_video_cost"), ("aretrieve_batch", "batch_abc_batch_cost")], +) +async def test_a_finished_job_cheaper_than_its_polls_guardrail_is_still_charged_once(call_type, charge_row_id): + """A batch of a few cheap requests costs less than a priced guardrail on its retrieve, and the finished job's + cost must still settle under its charge row once instead of being billed again on every poll.""" + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + charged = [ + await _update_database_for_video_poll( + db_writer, prisma, call_id, 0.01002775, guardrail_cost=0.01, call_type=call_type + ) + for call_id in ("poll-1", "poll-2") + ] + await _flush_queued_spend_logs(prisma) + + assert charged == [True, False] + assert {request_id: row["spend"] for request_id, row in prisma.db.litellm_spendlogs.rows.items()} == { + charge_row_id: 0.01002775 + } + + +@pytest.mark.asyncio +async def test_a_finished_batch_that_cost_nothing_itself_settles_its_retrieves_guardrail_once(): + """A batch whose every request failed costs nothing itself, and its finished retrieves must still settle under + the batch's charge row once, as they always have, instead of each paying the retrieve's guardrail again.""" + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + charged = [ + await _update_database_for_video_poll( + db_writer, prisma, call_id, 0.01, status="failed", guardrail_cost=0.01, call_type="aretrieve_batch" + ) + for call_id in ("poll-1", "poll-2", "poll-3") + ] + await _flush_queued_spend_logs(prisma) + + assert charged == [True, False, False] + assert {request_id: row["spend"] for request_id, row in prisma.db.litellm_spendlogs.rows.items()} == { + "batch_abc_batch_cost": 0.01 + } + + +@pytest.mark.asyncio +async def test_a_repeat_poll_of_a_finished_video_charges_its_guardrail_on_a_row_of_its_own(): + from litellm.proxy.hooks.proxy_track_cost_callback import _charge_request_cost + + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + await _update_database_for_video_poll(db_writer, prisma, "poll-1", 0.43, guardrail_cost=0.01) + kwargs, polled_job = _job_poll("poll-2", 0.43, guardrail_cost=0.01) + + async def charge_spend( + *, kwargs: dict[str, object], response_cost: float, budget_reservation: dict[str, object] | None + ) -> bool: + return await _spend_writer_call(db_writer, polled_job)(kwargs=kwargs, response_cost=response_cost) + + with _proxy_globals(prisma): + repeat_charge = await _charge_request_cost(charge_spend, kwargs, 0.43, None) + await asyncio.sleep(0) + await _flush_queued_spend_logs(prisma) + + assert repeat_charge == 0.01 + assert {request_id: row["spend"] for request_id, row in prisma.db.litellm_spendlogs.rows.items()} == { + "video_abc_video_cost": 0.43, + "poll-2_guardrail_cost": 0.01, + } + + +@pytest.mark.asyncio +async def test_update_database_charges_each_deployments_video_though_their_provider_ids_match(): + """A custom provider's own ids can repeat across deployments, so the cost row keys off the id the client polled.""" + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + + charged = [ + await _update_database_for_video_poll(db_writer, prisma, call_id, 0.42, polled_video_id=polled_id) + for call_id, polled_id in (("poll-1", "video_dep_a"), ("poll-2", "video_dep_b"), ("poll-3", "video_dep_a")) + ] + + assert charged == [True, True, False] + assert { + request_id: row["spend"] + for request_id, row in prisma.db.litellm_spendlogs.rows.items() + if request_id.endswith("_video_cost") + } == {"video_dep_a_video_cost": 0.42, "video_dep_b_video_cost": 0.42} + + +@pytest.mark.asyncio +async def test_update_database_writes_no_row_of_its_own_for_a_video_poll_that_reports_no_cost(): + """ + Built-in video providers charge at create and report nothing on the status poll, so those + polls keep the ordinary queued write under the create row's id and still count as requests + in the daily rollups, leaving the video with its one create row. + """ + db_writer = DBSpendUpdateWriter() + db_writer._batch_database_updates = AsyncMock() + prisma = _spend_logs_table_prisma() + await prisma.db.litellm_spendlogs.create_many( + data=[{**_minimal_spend_payload(), "request_id": "video_abc", "call_type": "avideo_generation", "spend": 0.3}], + skip_duplicates=True, + ) + + assert await _update_database_for_video_poll(db_writer, prisma, "poll-1", response_cost=0.0) is True + assert await _update_database_for_video_poll(db_writer, prisma, "poll-2", response_cost=0.0) is True + await _flush_queued_spend_logs(prisma) + + assert db_writer._batch_database_updates.await_count == 2 + rows = prisma.db.litellm_spendlogs.rows + assert {request_id: (row["call_type"], row["spend"]) for request_id, row in rows.items()} == { + "video_abc": ("avideo_generation", 0.3) + } + + @pytest.mark.asyncio @pytest.mark.parametrize( "injected_deployment, attributed", @@ -5452,3 +5842,36 @@ async def test_completed_spend_survives_error_log_suppression( assert len(rows) == 1 assert rows[0]["spend"] == pytest.approx(0.23) assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (10, 5) + + +@pytest.mark.parametrize("ledger_enabled", [False, True]) +@pytest.mark.asyncio +async def test_request_error_counts_survive_the_ledger_path_switch(monkeypatch: pytest.MonkeyPatch, ledger_enabled: bool): + from litellm.proxy import proxy_server + from litellm.proxy.db import db_spend_update_writer as writer_module + from litellm.proxy.db.request_error_tracking import RequestErrorAccumulator + from litellm.types.proxy.request_errors import RequestErrorKey + + accumulator = RequestErrorAccumulator() + prisma = _tool_usage_prisma() + prisma.get_request_status.return_value = "failure" + monkeypatch.setattr(writer_module, "request_error_accumulator", accumulator) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger_enabled}) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + now = datetime(2026, 11, 2, tzinfo=timezone.utc) + await DBSpendUpdateWriter().update_database( + token="key", user_id="user", team_id=None, org_id=None, end_user_id=None, + kwargs={ + "model": "ledger-test", "call_type": "acompletion", "litellm_call_id": "failed-request", + "litellm_params": {"metadata": { + "user_api_key": "key", "user_api_key_user_id": "user", "status": "failure", + "error_information": {"error_code": "503"}, + }}, + }, + completion_response=None, start_time=now, end_time=now, response_cost=0, + ) + await drain_logging_tasks() + assert accumulator.drain() == {RequestErrorKey( + date="2026-11-02", api_key=prisma.spend_log_transactions[0]["api_key"], team_id="", user_id="user", model_group="ledger-test", status_code=503, + ): 1} diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index d70d4403962..abc0850194d 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -22,7 +22,7 @@ from prisma.errors import ( import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException -from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_never_sent # Test is_database_connection_error method @@ -953,6 +953,21 @@ def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False +@pytest.mark.parametrize( + ("error", "expected"), + [ + (_lock_timeout_error(message="canceling statement due to statement timeout", code="57014"), True), + (_lock_timeout_error(message="canceling statement due to statement timeout", code=None), True), + (_lock_timeout_error(), False), + (_capacity_error("sorry, too many clients already"), False), + (RuntimeError("canceling statement due to statement timeout"), False), + ], + ids=["meta_57014", "message_only", "lock_timeout_55P03", "capacity_53300", "not_prisma"], +) +def test_is_statement_timeout_error(error: Exception, expected: bool) -> None: + assert PrismaDBExceptionHandler.is_statement_timeout_error(error) is expected + + @pytest.mark.parametrize( ("body", "retryable"), [ @@ -981,3 +996,137 @@ def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None: ) assert PrismaDBExceptionHandler.is_database_transport_error(error) is True assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False + + +def _connector_error(message: str) -> DataError: + return DataError( + data={"user_facing_error": {"message": f"Error in connector: Error querying the database: FATAL: {message}"}} + ) + + +def _batched_postgres_error(sqlstate: str, message: str) -> DataError: + return DataError( + data={ + "user_facing_error": { + "message": "Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: " + f'None, kind: QueryError(PostgresError {{ code: "{sqlstate}", message: "{message}", severity: "FATAL" }}) }})' + } + } + ) + + +def _localized_error(sqlstate: str) -> DataError: + return DataError(data={"user_facing_error": {"message": "FATAL: Datenbank", "meta": {"code": sqlstate}}}) + + +_NOT_ACCEPTING_CONNECTIONS: Final = ( + _connector_error("the database system is starting up"), + _connector_error("the database system is shutting down"), + _connector_error("the database system is not yet accepting connections"), + _connector_error("the database system is not accepting connections"), + _connector_error("the database system is in recovery mode"), + _localized_error("57P03"), + _batched_postgres_error("57P03", "the database system is starting up"), + EngineRequestError( + MagicMock(status=500), + '{"is_panic":false,"message":"Error in connector: Error querying the database: FATAL: the database system is ' + 'starting up","backtrace":null}', + ), + RawQueryError( + data={ + "user_facing_error": { + "message": "Raw query failed. Code: `57P03`. Message: `db error: FATAL: the database system is starting up`", + "meta": {"code": "57P03", "message": "FATAL: the database system is starting up"}, + "error_code": "P2010", + } + } + ), +) +_SERVER_TERMINATED: Final = ( + _connector_error("terminating connection due to administrator command"), + _connector_error("terminating connection due to immediate shutdown command"), + _connector_error("terminating connection because of unexpected SIGQUIT signal"), + _connector_error("terminating connection because of crash of another server process"), + _connector_error("terminating connection due to idle-session timeout"), + _localized_error("57P01"), + _localized_error("57P02"), + _localized_error("57P05"), + _batched_postgres_error("57P01", "terminating connection due to administrator command"), +) + + +@pytest.mark.parametrize( + ("error", "connection_error"), + [ + *((error, True) for error in _NOT_ACCEPTING_CONNECTIONS), + *((error, False) for error in _SERVER_TERMINATED), + ], +) +def test_postgres_lifecycle_errors_mean_the_database_is_unavailable(error: Exception, connection_error: bool) -> None: + """A server that is restarting, shutting down or cutting sessions is a transient outage: + auth answers 503 rather than 401, and the spend-log writer must not take the rows for poison. + Only a server refusing connections opens the ``allow_requests_on_db_unavailable`` fallback, + one terminated session does not prove the database is down.""" + assert { + "lifecycle": PrismaDBExceptionHandler.is_database_lifecycle_error(error), + "service_unavailable": PrismaDBExceptionHandler.is_database_service_unavailable_error(error), + "connection_error": PrismaDBExceptionHandler.is_database_connection_error(error), + "permanent": PrismaDBExceptionHandler.is_permanent_database_fault(error), + } == { + "lifecycle": True, + "service_unavailable": True, + "connection_error": connection_error, + "permanent": False, + } + + +@pytest.mark.parametrize( + ("error", "never_sent"), + [ + *((error, True) for error in _NOT_ACCEPTING_CONNECTIONS), + *((error, False) for error in _SERVER_TERMINATED), + (DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}}), True), + (DataError(data={"user_facing_error": {"error_code": "P1001", "message": "unreachable"}}), True), + (_capacity_error("sorry, too many clients already"), True), + (_localized_error("53300"), True), + ( + DataError( + data={ + "user_facing_error": { + "error_code": "P2024", + "message": "Timed out fetching a new connection from the connection pool", + } + } + ), + True, + ), + (httpx.ReadTimeout("read timed out"), False), + (_localized_error("57014"), False), + ], +) +def test_db_write_never_sent_only_for_failures_before_any_statement_ran(error: Exception, never_sent: bool) -> None: + """A refused, unreachable or full server provably applied nothing, while a terminated session + or a timeout may have committed before the failure, so only the former lets a non-idempotent + write be replayed.""" + assert db_write_never_sent(error) is never_sent + + +@pytest.mark.parametrize( + "error", + [ + _localized_error("57P04"), + _localized_error("57014"), + _localized_error("23505"), + DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}), + RuntimeError("the database system is starting up"), + ], +) +def test_other_failures_are_not_postgres_lifecycle_errors(error: Exception) -> None: + assert PrismaDBExceptionHandler.is_database_lifecycle_error(error) is False + assert db_write_never_sent(error) is False + + +def test_lifecycle_error_is_seen_through_a_wrapping_exception() -> None: + wrapped: Final = RuntimeError("spend flush failed") + wrapped.__cause__ = _connector_error("the database system is starting up") + assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True diff --git a/tests/unit/proxy/db/test_prisma_query_span.py b/tests/unit/proxy/db/test_prisma_query_span.py index 9f2ba28ea7e..55ba03d9f2c 100644 --- a/tests/unit/proxy/db/test_prisma_query_span.py +++ b/tests/unit/proxy/db/test_prisma_query_span.py @@ -19,6 +19,7 @@ from litellm.proxy.db.prisma_query_span import UNKNOWN_PRISMA_QUERY, parse_prism _REPO: Final = Path(__file__).resolve().parents[4] _SOURCE_ROOTS: Final = ("litellm", "enterprise", "litellm-proxy-extras") _RAW_METHODS: Final = frozenset({"query_first", "query_raw", "execute_raw"}) +_CROSS_MODULE_FORWARDERS: Final = frozenset({"query_raw_within_statement_timeout"}) _MODEL_METHODS: Final = frozenset( { "find_unique", @@ -297,7 +298,7 @@ def _parameter_site( """A statement that arrives as a parameter: a ``query_raw`` forwarder adds no round trip of its own, any other helper is named by what its callers in the module hand it.""" function: Final = next(p for p in parents if isinstance(p, (ast.AsyncFunctionDef, ast.FunctionDef))) - if function.name in _RAW_METHODS: + if function.name in _RAW_METHODS or function.name in _CROSS_MODULE_FORWARDERS: return _PrismaCallSite(location, method, "forwarder", f"(callers of {function.name})") callers: Final = tuple( node diff --git a/tests/unit/proxy/db/test_tool_registry_writer.py b/tests/unit/proxy/db/test_tool_registry_writer.py index 76898b0eeab..a5ad897913e 100644 --- a/tests/unit/proxy/db/test_tool_registry_writer.py +++ b/tests/unit/proxy/db/test_tool_registry_writer.py @@ -69,9 +69,7 @@ def _make_prisma( prisma.db.litellm_tooltable = MagicMock() prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return) prisma.db.execute_raw = AsyncMock(return_value=1) - prisma.db.litellm_tooltable.find_many = AsyncMock( - return_value=find_many_rows if find_many_rows is not None else [] - ) + prisma.db.litellm_tooltable.find_many = AsyncMock(return_value=find_many_rows if find_many_rows is not None else []) prisma.db.litellm_tooltable.find_unique = AsyncMock(return_value=find_unique_row) prisma.db.litellm_verificationtoken = MagicMock() prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(key_rows)) @@ -227,9 +225,7 @@ async def test_get_tool_found(): result = await get_tool(prisma, "my_tool") assert result is not None assert result.tool_name == "my_tool" - prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with( - where={"tool_name": "my_tool"} - ) + prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(where={"tool_name": "my_tool"}) @pytest.mark.asyncio @@ -266,9 +262,7 @@ async def test_update_tool_policy_calls_upsert_then_get_tool(): updated_by="admin", ) prisma = _make_prisma(find_unique_row=row) - result = await update_tool_policy( - prisma, "my_tool", updated_by="admin", input_policy="blocked" - ) + result = await update_tool_policy(prisma, "my_tool", updated_by="admin", input_policy="blocked") assert result is not None assert result.input_policy == "blocked" prisma.db.litellm_tooltable.upsert.assert_awaited_once() @@ -567,3 +561,42 @@ async def test_failed_tool_table_write_requeues_the_rows_only_when_they_provably left_in_redis = [json.loads(entry) for entry in redis.lists.get(REDIS_TOOL_DISCOVERY_BUFFER_KEY, [])] expected = [[{"tool_name": "get_weather", "origin": "user_defined", "call_count": 1}]] if requeued else [] assert left_in_redis == expected, f"requeued={requeued} but redis holds {left_in_redis}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "requeued"), + [ + (_postgres_rejection("57P03"), True), + (_postgres_rejection("53300"), True), + (_postgres_rejection("57P01"), False), + (_postgres_rejection("54000"), False), + (_postgres_rejection("23502"), False), + ], + ids=[ + "restarting_server_keeps_the_rows", + "full_server_keeps_the_rows", + "terminated_session_may_have_counted_so_the_rows_are_dropped", + "oversized_value_is_dropped", + "rejected_row_is_dropped", + ], +) +async def test_failed_write_without_redis_keeps_the_rows_in_the_queue_only_when_they_provably_did_not_apply( + failure: Exception, requeued: bool +): + queue = ToolDiscoveryQueue() + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.execute_raw = AsyncMock(side_effect=failure) + + await commit_tool_discoveries( + prisma_client=prisma_client, + tool_discovery_queue=queue, + redis_update_buffer=RedisUpdateBuffer(redis_cache=_InMemoryRedis()), + pod_lock_manager=PodLockManager(redis_cache=_InMemoryRedis()), + use_redis_buffer=False, + ) + queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + + expected = [{"tool_name": "get_weather", "origin": "user_defined", "call_count": 2 if requeued else 1}] + assert queue.flush() == expected diff --git a/tests/unit/proxy/discovery_endpoints/test_agent_skills_archive.py b/tests/unit/proxy/discovery_endpoints/test_agent_skills_archive.py index dd5ac230aac..a3d968263e5 100644 --- a/tests/unit/proxy/discovery_endpoints/test_agent_skills_archive.py +++ b/tests/unit/proxy/discovery_endpoints/test_agent_skills_archive.py @@ -1,6 +1,8 @@ import hashlib import io +import stat import zipfile +from typing import Final from litellm.proxy.discovery_endpoints.agent_skills_archive import ( MAX_ARCHIVE_ENTRIES, @@ -16,7 +18,7 @@ Read the PDF, then write the brief. """ -def zip_bytes(files: dict[str, bytes]) -> bytes: +def zip_bytes(files: dict[str | zipfile.ZipInfo, bytes]) -> bytes: buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: for name, content in files.items(): @@ -104,3 +106,15 @@ def test_an_upload_over_the_entry_cap_is_not_installable(): } assert build_skill_archive(zip_bytes(files)) is None + + +def test_discovery_keeps_its_permission_and_finder_metadata_policy() -> None: + executable: Final = zipfile.ZipInfo("scripts/run.sh") + executable.create_system = 3 + executable.external_attr = (stat.S_IFREG | 0o7777) << 16 + result: Final = build_skill_archive(zip_bytes({"SKILL.md": MANIFEST, executable: b"echo ready\n"})) + assert build_skill_archive(zip_bytes({"folder/SKILL.md": MANIFEST, "__MACOSX/._SKILL.md": b"Finder"})) is None + assert result is not None + with zipfile.ZipFile(io.BytesIO(result.content)) as archive: + assert {entry.create_system for entry in archive.infolist()} == {3} + assert {entry.external_attr >> 16 for entry in archive.infolist()} == {0o644} diff --git a/tests/unit/proxy/enterprise_billing/test_billing_metrics.py b/tests/unit/proxy/enterprise_billing/test_billing_metrics.py index 0446cfeeab0..14a8fea04f7 100644 --- a/tests/unit/proxy/enterprise_billing/test_billing_metrics.py +++ b/tests/unit/proxy/enterprise_billing/test_billing_metrics.py @@ -1,398 +1,379 @@ -""" -Tests for the enterprise billing-metrics recorder and its factory. - -These verify the license gate, the missing-config and missing-cert disable -paths, the OTLP/HTTP exporter wiring (client cert+key authenticate us to the -collector's mTLS-terminating front end; CA override optional for private -collectors), the metric attribute mapping, and that recording produces the -expected OTLP counter via an in-memory reader. -""" - -import os -import socket -import stat -from pathlib import Path -from typing import Dict, List, Optional +from collections.abc import Generator, Mapping +from typing import Final import pytest +import requests +import requests_mock from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.metrics.export import InMemoryMetricReader, Sum +from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.proxy._types import EnterpriseLicenseData from litellm.proxy.enterprise_billing import billing_metrics as bm from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory -_ENV_VARS = ( + +class _RecordedRequest(BaseModel): + model_config = ConfigDict(from_attributes=True, frozen=True) + + url: str | None + body: str | bytes | None + headers: dict[str, str] + + +def _request_history(http_mock: requests_mock.Mocker) -> tuple[_RecordedRequest, ...]: + return TypeAdapter(tuple[_RecordedRequest, ...]).validate_python(http_mock.request_history) + + +def _metric_category(attributes: Mapping[str, object] | None) -> str: + if attributes is None: + return "" + return str(attributes.get("litellm.endpoint.category")) + + +_ENV_VARS: Final = ( bm.ENDPOINT_ENV, - bm.CLIENT_CERT_ENV, - bm.CLIENT_KEY_ENV, - bm.CA_CERT_ENV, + bm.CLIENT_ID_ENV, + bm.CLIENT_SECRET_ENV, + bm.TOKEN_URL_ENV, bm.EXPORT_INTERVAL_ENV, ) @pytest.fixture(autouse=True) -def clear_env(monkeypatch): +def clear_env(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: for name in _ENV_VARS: monkeypatch.delenv(name, raising=False) yield bm.shutdown_billing_metrics_recorder() -def _write_certs(tmp_path: Path) -> Dict[str, str]: - files = { - bm.CA_CERT_ENV: ("ca.pem", b"ca-bytes"), - bm.CLIENT_CERT_ENV: ("client.pem", b"client-cert-bytes"), - bm.CLIENT_KEY_ENV: ("client.key", b"client-key-bytes"), - } - paths = {} - for env_name, (filename, content) in files.items(): - path = tmp_path / filename - path.write_bytes(content) - paths[env_name] = str(path) - return paths +def _set_full_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4318") + monkeypatch.setenv(bm.CLIENT_ID_ENV, "client-id") + monkeypatch.setenv(bm.CLIENT_SECRET_ENV, "client-secret") + monkeypatch.setenv(bm.TOKEN_URL_ENV, "https://auth.example/oauth2/token") -def _set_full_env(monkeypatch, tmp_path: Path) -> Dict[str, str]: - paths = _write_certs(tmp_path) - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CA_CERT_ENV, paths[bm.CA_CERT_ENV]) - monkeypatch.setenv(bm.CLIENT_CERT_ENV, paths[bm.CLIENT_CERT_ENV]) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, paths[bm.CLIENT_KEY_ENV]) - return paths - - -def _config(tmp_path: Path, license_id: Optional[str] = "org-1") -> bm.BillingMetricsConfig: - paths = _write_certs(tmp_path) +def _config(license_id: str | None = "org-1") -> bm.BillingMetricsConfig: return bm.BillingMetricsConfig( - endpoint="https://collector.example:4317", - client_cert_path=paths[bm.CLIENT_CERT_ENV], - client_key_path=paths[bm.CLIENT_KEY_ENV], - ca_cert_path=paths[bm.CA_CERT_ENV], + endpoint="https://collector.example:4318", + token_url="https://auth.example/oauth2/token", + client_id="client-id", + client_secret="client-secret", export_interval_ms=60_000, litellm_version="1.2.3", license_id=license_id, ) -# ── Factory gating ──────────────────────────────────────────────────────────── +@pytest.mark.parametrize("missing_env", (bm.ENDPOINT_ENV, bm.CLIENT_ID_ENV, bm.CLIENT_SECRET_ENV, bm.TOKEN_URL_ENV)) +def test_missing_required_env_disables_metrics_without_logging_values( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + missing_env: str, +) -> None: + _set_full_env(monkeypatch) + monkeypatch.setenv(bm.CLIENT_SECRET_ENV, "secret-for-test") + monkeypatch.delenv(missing_env) + + assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None + assert f"licensed deployment missing config ({missing_env})" in caplog.text + assert "secret-for-test" not in caplog.text -def test_not_premium_returns_none(tmp_path, monkeypatch): - _set_full_env(monkeypatch, tmp_path) +def test_empty_required_env_disables_metrics(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: + _set_full_env(monkeypatch) + monkeypatch.setenv(bm.ENDPOINT_ENV, "") + + assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None + assert bm.ENDPOINT_ENV in caplog.text + + +@pytest.mark.parametrize( + ("env_name", "url"), + ( + (bm.ENDPOINT_ENV, "http://collector.example:4318"), + (bm.TOKEN_URL_ENV, "http://auth.example/oauth2/token"), + ), +) +def test_http_urls_disable_metrics( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + env_name: str, + url: str, +) -> None: + _set_full_env(monkeypatch) + monkeypatch.setenv(env_name, url) + + assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None + assert f"{env_name} must use HTTPS" in caplog.text + + +def test_valid_config_contains_bearer_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + _set_full_env(monkeypatch) + + license_data: Final[EnterpriseLicenseData] = {"user_id": "org-1"} + config: Final = bm.load_billing_metrics_config(license_data=license_data, litellm_version="1.0") + assert config is not None + + assert config == bm.BillingMetricsConfig( + endpoint="https://collector.example:4318", + token_url="https://auth.example/oauth2/token", + client_id="client-id", + client_secret="client-secret", + export_interval_ms=bm.DEFAULT_EXPORT_INTERVAL_MS, + litellm_version="1.0", + license_id="org-1", + ) + assert "client-secret" not in repr(config) + assert "license-secret" not in repr(config) + + +def test_not_premium_returns_none_without_warning( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _set_full_env(monkeypatch) + assert bm.build_billing_metrics_recorder(premium=False, license_data=None, litellm_version="1.0") is None + assert "Enterprise billing metrics disabled" not in caplog.text -def test_premium_without_config_returns_none(monkeypatch): +def test_premium_without_config_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: assert bm.build_billing_metrics_recorder(premium=True, license_data={"user_id": "x"}, litellm_version="1.0") is None -def test_premium_with_missing_cert_files_returns_none(monkeypatch, tmp_path): - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CA_CERT_ENV, str(tmp_path / "missing-ca.pem")) - monkeypatch.setenv(bm.CLIENT_CERT_ENV, str(tmp_path / "missing-cert.pem")) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, str(tmp_path / "missing-key.pem")) - assert bm.build_billing_metrics_recorder(premium=True, license_data=None, litellm_version="1.0") is None +def test_premium_builds_recorder_and_logs_enabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _set_full_env(monkeypatch) + caplog.set_level("INFO", logger=bm.verbose_proxy_logger.name) + token_url: Final = "https://auth.example/oauth2/token" + export_url: Final = "https://collector.example:4318/v1/metrics" + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post(token_url, json={"access_token": "access-token", "expires_in": 3600}) + http_mock.post(export_url, status_code=200) + recorder: Final = bm.build_billing_metrics_recorder( + premium=True, license_data={"user_id": "org-1"}, litellm_version="1.0" + ) + assert recorder is not None + assert "Enterprise billing metrics enabled: exporting to https://collector.example:4318 every 60000 ms" in caplog.text + bm.shutdown_billing_metrics_recorder() -def test_premium_with_full_config_builds_recorder(monkeypatch, tmp_path): - """Builds a real MeterProvider, so the exporter is stubbed: the live one - resolves the collector and opens a TLS connection during the shutdown flush. - The getaddrinfo spy keeps that stub from being quietly dropped later.""" - _set_full_env(monkeypatch, tmp_path) - monkeypatch.setattr(bm, "OTLPMetricExporter", _fake_exporter_class({})) +class _ShutdownCountingMeterProvider(MeterProvider): + def __init__(self) -> None: + super().__init__() + self.shutdown_calls = 0 - resolved: List[str] = [] - real_getaddrinfo = socket.getaddrinfo + def shutdown(self, timeout_millis: float = 30_000) -> None: + self.shutdown_calls += 1 + super().shutdown(timeout_millis=timeout_millis) - def _spy_getaddrinfo(host, port, *args, **kwargs): - resolved.append(str(host)) - return real_getaddrinfo(host, port, *args, **kwargs) - monkeypatch.setattr(socket, "getaddrinfo", _spy_getaddrinfo) +def test_shutdown_flushes_active_recorder_once(monkeypatch: pytest.MonkeyPatch) -> None: + provider: Final = _ShutdownCountingMeterProvider() + _set_full_env(monkeypatch) - recorder = bm.build_billing_metrics_recorder( - premium=True, license_data={"user_id": "org-1"}, litellm_version="1.0" + def create_provider(config: bm.BillingMetricsConfig) -> MeterProvider: + return provider + + monkeypatch.setattr(bm, "build_bearer_meter_provider", create_provider) + recorder: Final = bm.build_billing_metrics_recorder(premium=True, license_data=None, litellm_version="1.0") + assert recorder is not None + + bm.shutdown_billing_metrics_recorder() + bm.shutdown_billing_metrics_recorder() + + assert provider.shutdown_calls == 1 + + +class _FakeClock: + def __init__(self, now: float) -> None: + self._now = now + + def __call__(self) -> float: + return self._now + + def set(self, now: float) -> None: + self._now = now + + +def test_bearer_auth_fetches_and_caches_client_credentials_token() -> None: + token_url: Final = "https://auth.example/oauth2/token" + export_url: Final = "https://collector.example/v1/metrics" + auth: Final = bm.BillingMetricsBearerAuth( + client_id="client-id", + client_secret="client-secret", + token_url=token_url, + clock=lambda: 100.0, ) - assert isinstance(recorder, bm.BillingMetricsRecorder) - recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200, model_id=None) - bm.shutdown_billing_metrics_recorder() + session: Final = requests.Session() + session.auth = auth + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post(token_url, json={"access_token": "access-token", "expires_in": 3600}) + http_mock.post(export_url, status_code=200) + session.post(export_url, data=b"metrics") + session.post(export_url, data=b"metrics") - assert [host for host in resolved if "collector.example" in host] == [] + request_records: Final = _request_history(http_mock) + token_requests: Final = tuple(request for request in request_records if request.url == token_url) + export_requests: Final = tuple(request for request in request_records if request.url == export_url) + assert len(token_requests) == 1 + assert token_requests[0].headers["Authorization"].startswith("Basic ") + assert token_requests[0].body == "grant_type=client_credentials&scope=telemetry%2Fwrite" + assert tuple(request.headers["Authorization"] for request in export_requests) == ( + "Bearer access-token", + "Bearer access-token", + ) -def test_building_the_recorder_logs_an_affirmative_line(monkeypatch, tmp_path): - """ - Every disable path logs; a successful build must log too. Otherwise an - operator cannot tell a metering component from one that silently returned - None, which is how an unlicensed component looks healthy while exporting - nothing. - """ - _set_full_env(monkeypatch, tmp_path) +def test_bearer_auth_refreshes_before_token_expiry() -> None: + token_url: Final = "https://auth.example/oauth2/token" + export_url: Final = "https://collector.example/v1/metrics" + clock: Final = _FakeClock(1000.0) + auth: Final = bm.BillingMetricsBearerAuth( + client_id="client-id", + client_secret="client-secret", + token_url=token_url, + clock=clock, + ) + session: Final = requests.Session() + session.auth = auth + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post( + token_url, + [ + {"json": {"access_token": "first-token", "expires_in": 120}}, + {"json": {"access_token": "second-token", "expires_in": 120}}, + ], + ) + http_mock.post(export_url, status_code=200) + session.post(export_url, data=b"metrics") + clock.set(1059.0) + session.post(export_url, data=b"metrics") + clock.set(1060.0) + session.post(export_url, data=b"metrics") + + request_records: Final = _request_history(http_mock) + token_requests: Final = tuple(request for request in request_records if request.url == token_url) + export_requests: Final = tuple(request for request in request_records if request.url == export_url) + assert len(token_requests) == 2 + assert tuple(request.headers["Authorization"] for request in export_requests) == ( + "Bearer first-token", + "Bearer first-token", + "Bearer second-token", + ) + + +def test_bearer_export_sets_authorization_header() -> None: + config: Final = _config() + token_url: Final = config.token_url + export_url: Final = f"{config.endpoint}/v1/metrics" + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post(token_url, json={"access_token": "export-token", "expires_in": 3600}) + http_mock.post(export_url, status_code=200) + provider: Final = bm.build_bearer_meter_provider(config) + try: + provider.get_meter("test").create_counter("requests").add(1) + assert provider.force_flush(timeout_millis=1000) + finally: + provider.shutdown(timeout_millis=1000) + + export_requests: Final = tuple(request for request in _request_history(http_mock) if request.url == export_url) + assert export_requests + assert all(request.headers["Authorization"] == "Bearer export-token" for request in export_requests) + + +def test_unauthorized_export_refetches_token_before_next_export() -> None: + config: Final = _config() + token_url: Final = config.token_url + export_url: Final = f"{config.endpoint}/v1/metrics" + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post( + token_url, + [ + {"json": {"access_token": "first-token", "expires_in": 3600}}, + {"json": {"access_token": "second-token", "expires_in": 3600}}, + ], + ) + http_mock.post(export_url, [{"status_code": 401}, {"status_code": 200}]) + provider: Final = bm.build_bearer_meter_provider(config) + try: + counter: Final = provider.get_meter("test").create_counter("requests") + counter.add(1) + provider.force_flush(timeout_millis=1000) + counter.add(1) + provider.force_flush(timeout_millis=1000) + finally: + provider.shutdown(timeout_millis=1000) + + requests: Final = _request_history(http_mock) + token_requests: Final = tuple(request for request in requests if request.url == token_url) + export_requests: Final = tuple(request for request in requests if request.url == export_url) + assert len(token_requests) == 2 + assert tuple(request.headers["Authorization"] for request in export_requests[:2]) == ( + "Bearer first-token", + "Bearer second-token", + ) + + +def test_failed_bearer_token_fetch_does_not_raise_from_export() -> None: + config: Final = _config() + token_url: Final = config.token_url + export_url: Final = f"{config.endpoint}/v1/metrics" + http_mock: Final = requests_mock.Mocker() + with http_mock: + http_mock.post(token_url, status_code=400) + provider: Final = bm.build_bearer_meter_provider(config) + try: + provider.get_meter("test").create_counter("requests").add(1) + assert provider.force_flush(timeout_millis=1000) + finally: + provider.shutdown(timeout_millis=1000) + + request_urls: Final = tuple(request.url for request in _request_history(http_mock)) + assert token_url in request_urls + assert export_url not in request_urls + + +def test_export_interval_default_and_override(monkeypatch: pytest.MonkeyPatch) -> None: + _set_full_env(monkeypatch) + default_config: Final = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") + assert default_config is not None + assert default_config.export_interval_ms == bm.DEFAULT_EXPORT_INTERVAL_MS monkeypatch.setenv(bm.EXPORT_INTERVAL_ENV, "5000") - monkeypatch.setattr(bm, "OTLPMetricExporter", _fake_exporter_class({})) - - infos: List[str] = [] - monkeypatch.setattr(bm.verbose_proxy_logger, "info", lambda msg, *args: infos.append(msg % args if args else msg)) - - recorder = bm.build_billing_metrics_recorder(premium=True, license_data={"user_id": "org-1"}, litellm_version="1.0") - - assert recorder is not None - joined = "\n".join(infos) - assert "https://collector.example:4317" in joined - assert "5000" in joined - - -def test_unlicensed_build_does_not_warn(monkeypatch, tmp_path): - """Unlicensed is the common OSS case; warning there would be pure noise.""" - _set_full_env(monkeypatch, tmp_path) - - warnings: List[str] = [] - monkeypatch.setattr(bm.verbose_proxy_logger, "warning", lambda msg, *args: warnings.append(str(msg))) - - assert bm.build_billing_metrics_recorder(premium=False, license_data=None, litellm_version="1.0") is None - assert warnings == [] - - -def test_shutdown_flushes_active_recorder_once(monkeypatch, tmp_path): - """The shutdown hook must flush the recorder the factory built (buffered - counts are lost on restart otherwise) and be idempotent for repeat calls.""" - _set_full_env(monkeypatch, tmp_path) - shutdowns = [] - - class _SpyProvider: - def get_meter(self, name): - return MeterProvider().get_meter(name) - - def shutdown(self, timeout_millis=None): - shutdowns.append(timeout_millis) - - monkeypatch.setattr(bm, "build_mtls_meter_provider", lambda config: _SpyProvider()) - recorder = bm.build_billing_metrics_recorder(premium=True, license_data=None, litellm_version="1.0") - assert recorder is not None - - bm.shutdown_billing_metrics_recorder() - bm.shutdown_billing_metrics_recorder() - assert shutdowns == [bm.SHUTDOWN_FLUSH_TIMEOUT_MS] - - -def test_shutdown_without_active_recorder_is_noop(): - bm.shutdown_billing_metrics_recorder() - - -# ── Config loading ──────────────────────────────────────────────────────────── - - -def test_load_config_carries_license_id(monkeypatch, tmp_path): - _set_full_env(monkeypatch, tmp_path) - config = bm.load_billing_metrics_config(license_data={"user_id": "org-42"}, litellm_version="9.9") - assert config is not None and config.license_id == "org-42" and config.litellm_version == "9.9" - - -def test_load_config_with_empty_string_env_is_disabled(monkeypatch, tmp_path): - """An env var set to the empty string is as unusable as an unset one and - must disable metering rather than produce a config with a blank endpoint.""" - paths = _write_certs(tmp_path) - monkeypatch.setenv(bm.ENDPOINT_ENV, "") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, paths[bm.CLIENT_CERT_ENV]) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, paths[bm.CLIENT_KEY_ENV]) - assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None - - -_CLIENT_CERT_PEM = "-----BEGIN CERTIFICATE-----\nclient-cert-body\n-----END CERTIFICATE-----" -_CLIENT_KEY_PEM = "-----BEGIN PRIVATE KEY-----\nclient-key-body\n-----END PRIVATE KEY-----" -_CA_CERT_PEM = "-----BEGIN CERTIFICATE-----\nca-body\n-----END CERTIFICATE-----" - - -def test_load_config_materializes_inline_pem_content(monkeypatch): - """ - ECS and Cloud Run inject secrets as env content, not as mounted files, so the - cert env vars must accept PEM directly. The exporter takes paths, so the PEM - is written to disk and the config points at those files. - """ - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, _CLIENT_CERT_PEM) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, _CLIENT_KEY_PEM) - monkeypatch.setenv(bm.CA_CERT_ENV, _CA_CERT_PEM) - - config = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") - - assert config is not None - assert config.ca_cert_path is not None - written = { - config.client_cert_path: _CLIENT_CERT_PEM, - config.client_key_path: _CLIENT_KEY_PEM, - config.ca_cert_path: _CA_CERT_PEM, - } - for path, pem in written.items(): - assert path != pem, "config must carry a file path, not the PEM itself" - assert os.path.isfile(path) - assert Path(path).read_text(encoding="utf-8") == f"{pem}\n" - - # The private key must not be world- or group-readable. - assert stat.S_IMODE(os.stat(config.client_key_path).st_mode) == 0o600 - - -def test_load_config_accepts_a_mix_of_pem_content_and_file_paths(monkeypatch, tmp_path): - """A deployment may mount the CA but inject the client credentials inline.""" - paths = _write_certs(tmp_path) - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, _CLIENT_CERT_PEM) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, _CLIENT_KEY_PEM) - monkeypatch.setenv(bm.CA_CERT_ENV, paths[bm.CA_CERT_ENV]) - - config = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") - - assert config is not None - assert config.ca_cert_path == paths[bm.CA_CERT_ENV] - assert Path(config.client_cert_path).read_text(encoding="utf-8") == f"{_CLIENT_CERT_PEM}\n" - - -def test_load_config_leaves_file_paths_untouched(monkeypatch, tmp_path): - """Path-valued env vars keep working; nothing is copied or rewritten.""" - paths = _set_full_env(monkeypatch, tmp_path) - - config = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") - - assert config is not None - assert config.client_cert_path == paths[bm.CLIENT_CERT_ENV] - assert config.client_key_path == paths[bm.CLIENT_KEY_ENV] - assert config.ca_cert_path == paths[bm.CA_CERT_ENV] - - -def test_load_config_with_inline_pem_disabled_when_unwritable(monkeypatch): - """A failure to materialize the PEM disables metering instead of raising.""" - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, _CLIENT_CERT_PEM) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, _CLIENT_KEY_PEM) - - def _explode(prefix=None): - raise OSError("read-only filesystem") - - monkeypatch.setattr(bm.tempfile, "mkdtemp", _explode) - - assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None - - -def test_load_config_never_logs_credential_values(monkeypatch): - """ - A value that is neither a readable path nor `-----BEGIN`-prefixed PEM is - still secret material. The disable warning must name the env vars, never - echo their contents, or a malformed key lands in the proxy logs. - """ - secret_material = "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQ-not-pem-prefixed" - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, secret_material) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, secret_material) - - logged: List[str] = [] - - def _capture(msg, *args): - logged.append(msg % args if args else msg) - - monkeypatch.setattr(bm.verbose_proxy_logger, "warning", _capture) - - assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None - - joined = "\n".join(logged) - assert secret_material not in joined - assert bm.CLIENT_CERT_ENV in joined and bm.CLIENT_KEY_ENV in joined - - -def test_load_config_with_empty_pem_env_is_disabled(monkeypatch): - """Empty stays empty: an unset secret must not be mistaken for inline PEM.""" - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://collector.example:4317") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, "") - monkeypatch.setenv(bm.CLIENT_KEY_ENV, "") - - assert bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") is None - - -def test_export_interval_default_and_override(monkeypatch): - assert bm._export_interval_ms() == bm.DEFAULT_EXPORT_INTERVAL_MS - monkeypatch.setenv(bm.EXPORT_INTERVAL_ENV, "5000") - assert bm._export_interval_ms() == 5000 + override_config: Final = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") + assert override_config is not None + assert override_config.export_interval_ms == 5000 monkeypatch.setenv(bm.EXPORT_INTERVAL_ENV, "not-a-number") - assert bm._export_interval_ms() == bm.DEFAULT_EXPORT_INTERVAL_MS + invalid_config: Final = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") + assert invalid_config is not None + assert invalid_config.export_interval_ms == bm.DEFAULT_EXPORT_INTERVAL_MS -# ── OTLP/HTTP exporter wiring ───────────────────────────────────────────────── +def test_resource_attributes_include_license_id() -> None: + attributes: Final = bm.resource_attributes(_config(license_id="org-7")) + assert attributes == { + "service.name": "litellm-proxy", + "litellm.version": "1.2.3", + "litellm.license.id": "org-7", + } -def test_metrics_endpoint_appends_signal_path(): - assert bm._metrics_endpoint("https://telemetry.example.com") == "https://telemetry.example.com/v1/metrics" - assert bm._metrics_endpoint("https://telemetry.example.com/") == "https://telemetry.example.com/v1/metrics" - assert bm._metrics_endpoint("https://telemetry.example.com/v1/metrics") == "https://telemetry.example.com/v1/metrics" +def test_resource_attributes_omit_license_id_when_absent() -> None: + assert "litellm.license.id" not in bm.resource_attributes(_config(license_id=None)) -def _fake_exporter_class(captured: Dict[str, object]) -> type: - """A no-network stand-in for OTLPMetricExporter. Tests that build a real - MeterProvider must install this: the real exporter resolves the collector - host and opens a TLS connection on the reader's first export and on the - shutdown flush.""" - - class _FakeExporter: - # PeriodicExportingMetricReader probes these on the exporter it wraps. - _preferred_temporality: dict = {} - _preferred_aggregation: dict = {} - - def __init__(self, **kwargs): - captured.update(kwargs) - - def export(self, *args, **kwargs): - return None - - def shutdown(self, *args, **kwargs): - return None - - def force_flush(self, *args, **kwargs): - return True - - return _FakeExporter - - -def test_meter_provider_wires_client_cert_into_http_exporter(tmp_path, monkeypatch): - """Client cert+key authenticate us at the collector's mTLS front end; CA override rides certificate_file.""" - captured: Dict[str, object] = {} - - monkeypatch.setattr(bm, "OTLPMetricExporter", _fake_exporter_class(captured)) - config = _config(tmp_path) - provider = bm.build_mtls_meter_provider(config) - provider.shutdown() - - assert captured["endpoint"] == "https://collector.example:4317/v1/metrics" - assert captured["client_certificate_file"] == config.client_cert_path - assert captured["client_key_file"] == config.client_key_path - assert captured["certificate_file"] == config.ca_cert_path - - -def test_load_config_without_ca_is_valid(monkeypatch, tmp_path): - """The production collector presents a public web-PKI cert: no CA override required.""" - paths = _write_certs(tmp_path) - monkeypatch.setenv(bm.ENDPOINT_ENV, "https://telemetry.example.com") - monkeypatch.setenv(bm.CLIENT_CERT_ENV, paths[bm.CLIENT_CERT_ENV]) - monkeypatch.setenv(bm.CLIENT_KEY_ENV, paths[bm.CLIENT_KEY_ENV]) - config = bm.load_billing_metrics_config(license_data=None, litellm_version="1.0") - assert config is not None and config.ca_cert_path is None - - -# ── Resource and metric attributes ──────────────────────────────────────────── - - -def test_resource_attributes_include_license_id(tmp_path): - attrs = bm._resource_attributes(_config(tmp_path, license_id="org-7")) - assert attrs["service.name"] == "litellm-proxy" - assert attrs["litellm.version"] == "1.2.3" - assert attrs["litellm.license.id"] == "org-7" - - -def test_resource_attributes_omit_license_id_when_absent(tmp_path): - attrs = bm._resource_attributes(_config(tmp_path, license_id=None)) - assert "litellm.license.id" not in attrs - - -def test_billable_attributes_with_model_id(): - attrs = bm._billable_attributes(BillableCategory.LLM, "/chat/completions", 200, "deploy-3") - assert attrs == { +def test_billable_attributes_include_model_id() -> None: + assert bm.billable_attributes( + BillableCategory.LLM, "/chat/completions", 200, "deploy-3" + ) == { "litellm.endpoint.category": "llm", "http.route": "/chat/completions", "http.response.status_code": 200, @@ -400,33 +381,35 @@ def test_billable_attributes_with_model_id(): } -def test_billable_attributes_omit_model_id_when_none(): - attrs = bm._billable_attributes(BillableCategory.MCP, "/mcp", 200, None) - assert "litellm.model_id" not in attrs +def test_billable_attributes_omit_model_id_when_none() -> None: + attributes: Final = bm.billable_attributes( + BillableCategory.MCP, "/mcp", 200, None + ) + assert "litellm.model_id" not in attributes -# ── End-to-end recording via in-memory reader ───────────────────────────────── +def _counter_points(reader: InMemoryMetricReader) -> tuple[tuple[str, int], ...]: + data: Final = reader.get_metrics_data() + if data is None: + return () + return tuple( + (_metric_category(point.attributes), int(point.value)) + for resource_metric in data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + if metric.name == bm.METRIC_NAME and isinstance(metric.data, Sum) + for point in metric.data.data_points + ) -def _counter_points(reader: InMemoryMetricReader): - data = reader.get_metrics_data() - for resource_metric in data.resource_metrics: - for scope_metric in resource_metric.scope_metrics: - for metric in scope_metric.metrics: - if metric.name == bm.METRIC_NAME: - return list(metric.data.data_points) - return [] - - -def test_record_increments_counter_with_attributes(): - reader = InMemoryMetricReader() - recorder = bm.BillingMetricsRecorder(MeterProvider(metric_readers=[reader])) +def test_record_increments_counter_with_attributes() -> None: + reader: Final = InMemoryMetricReader() + recorder: Final = bm.BillingMetricsRecorder(MeterProvider(metric_readers=[reader])) recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200, model_id="m1") recorder.record(category=BillableCategory.LLM, route="/chat/completions", status_code=200, model_id="m1") recorder.record(category=BillableCategory.MCP, route="/mcp", status_code=200, model_id=None) - points = _counter_points(reader) - by_category = {point.attributes["litellm.endpoint.category"]: point.value for point in points} - assert by_category["llm"] == 2 - assert by_category["mcp"] == 1 + points: Final = _counter_points(reader) + by_category: Final = dict(points) + assert by_category == {"llm": 2, "mcp": 1} diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py index d2e5b407e30..1baf9504208 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py @@ -200,3 +200,78 @@ async def test_write_back_keeps_real_tool_results_under_modify_params(): assert "skipped" not in serialized assert "Please continue" not in serialized assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] + + +@pytest.mark.asyncio +async def test_write_back_sends_the_client_url_blocks_as_the_client_sent_them(async_only_image_fetch): + image_link = "https://img.example/presskit/page.pdf?fm=png" + document_link = "https://files.example/reports/q3.pdf" + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu_1", "name": "fetch_report", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [ + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 attachment"} + ], + }, + {"type": "text", "text": "compare these"}, + {"type": "image", "source": {"type": "url", "url": image_link}}, + { + "type": "document", + "source": {"type": "url", "url": document_link}, + "title": "Q3 report", + "context": "Board copy", + "citations": {"enabled": True}, + "cache_control": {"type": "ephemeral"}, + }, + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 report, again"}, + ], + }, + ] + + written = await _write_back_identity([dict(m) for m in messages]) + + assert async_only_image_fetch.fetched == [] + assert [m["content"] for m in written[1:]] == [messages[1]["content"][:1], messages[1]["content"][1:]] + + +@pytest.mark.asyncio +async def test_write_back_keeps_each_url_document_fields_when_a_tool_result_after_it_has_the_same_link( + async_only_image_fetch, +): + document_link = "https://files.example/reports/q3.pdf" + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu_1", "name": "fetch_report", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "document", + "source": {"type": "url", "url": document_link}, + "title": "Q3 report", + "context": "Board copy", + }, + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [ + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 attachment"} + ], + }, + ], + }, + ] + + written = await _write_back_identity([dict(m) for m in messages]) + + assert [m["content"] for m in written[1:]] == [messages[1]["content"][1:], messages[1]["content"][:1]] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 91f00edfb39..46e766f753c 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -2,11 +2,12 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics) """ +import copy import json import logging import re from itertools import chain -from typing import Final +from typing import Final, Literal from unittest.mock import patch import pytest @@ -26,6 +27,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( PermissionError, ) from litellm.types.utils import ( + CallTypesLiteral, ChatCompletionCustomToolCallPayload, ChatCompletionDeltaToolCall, ChatCompletionMessageCustomToolCall, @@ -467,6 +469,346 @@ class TestToolPermissionGuardrail: ] assert kept == [[], ["Read"]] + @pytest.mark.asyncio + @pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) + async def test_post_call_judges_a_returned_namespace_member_by_its_qualified_name(self, stream: bool) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-mcp", + rules=[{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]} + data = {"guardrails": ["post-mcp"], "tools": [namespace]} + function = Function(name="lookup", arguments='{"city": "Paris"}') + + if stream: + + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_lookup", index=0, type="function", function=function + ) + ] + ), + ) + ] + ) + + chunks = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data=data + ) + ] + deltas = [choice.delta for choice in chain.from_iterable(chunk.choices for chunk in chunks)] + called = [call.function.name for call in chain.from_iterable(delta.tool_calls or () for delta in deltas)] + content = "".join(delta.content or "" for delta in deltas) + else: + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_lookup", "type": "function", "function": function}]}, + ) + ] + ) + await guardrail.async_post_call_success_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + message = response.choices[0].message + called = [call.function.name for call in message.tool_calls or ()] + content = message.content or "" + + assert called == [] + assert "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'" in content + + @pytest.mark.asyncio + async def test_post_call_judges_a_returned_namespace_member_by_the_rule_its_arguments_satisfy(self) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-exec", + rules=[ + { + "id": "allow_ls", + "tool_name": r"^exec$", + "decision": "allow", + "allowed_param_patterns": {"command": r"^ls$"}, + }, + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + {"id": "allow_ops", "tool_name": r"^ops__.*$", "decision": "allow"}, + ], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "ops", "tools": [{"type": "function", "name": "exec"}]} + data = {"guardrails": ["post-exec"], "tools": [namespace]} + function = Function(name="exec", arguments='{"command": "rm -rf /data"}') + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_exec", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response) + + message = response.choices[0].message + assert [call.function.name for call in message.tool_calls or ()] == [] + assert "Tool 'exec'" in (message.content or "") + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rules", "command", "kept"), + [ + ( + [ + { + "id": "allow_ls", + "tool_name": r"^exec$", + "decision": "allow", + "allowed_param_patterns": {"command": r"^ls$"}, + }, + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + ], + "ls", + ["run"], + ), + ( + [ + { + "id": "allow_ls", + "tool_name": r"^exec$", + "decision": "allow", + "allowed_param_patterns": {"command": r"^ls$"}, + }, + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + ], + "pwd", + [], + ), + ( + [ + {"id": "allow_run", "tool_name": r"^run$", "decision": "allow"}, + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + ], + "ls", + [], + ), + ], + ids=["namespace_rule_allows", "namespace_rule_denies", "namespace_deny_beats_member_allow"], + ) + async def test_post_call_judges_a_returned_namespace_member_by_a_rule_on_its_namespace_name( + self, rules: list[dict[str, object]], command: str, kept: list[str] + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-exec", + rules=rules, + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "exec", "tools": [{"type": "function", "name": "run"}]} + function = Function(name="run", arguments=json.dumps({"command": command})) + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_run", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-exec"], "tools": [namespace]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + message = response.choices[0].message + assert [call.function.name for call in message.tool_calls or ()] == kept + assert ("Tool 'exec'" in (message.content or "")) == (not kept) + + @pytest.mark.asyncio + @pytest.mark.parametrize("namespace_first", [True, False], ids=["namespace_first", "top_level_first"]) + async def test_post_call_judges_a_returned_name_a_top_level_function_carries_as_that_function( + self, namespace_first: bool + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-mcp", + rules=[{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]} + top_level = {"type": "function", "function": {"name": "lookup"}} + tools = [namespace, top_level] if namespace_first else [top_level, namespace] + function = Function(name="lookup", arguments='{"country": "France"}') + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_lookup", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-mcp"], "tools": tools}, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + assert [call.function.name for call in response.choices[0].message.tool_calls or ()] == ["lookup"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "same_named_tool", + [ + {"type": "mcp", "server_label": "lookup", "server_url": "https://mcp.example.com/sse"}, + {"type": "mcp_toolset", "mcp_server_name": "lookup"}, + {"ignored": {"name": "lookup"}}, + {"type": "custom", "custom": {"name": "lookup"}}, + {"functionDeclarations": [{"name": "lookup"}]}, + ], + ids=["mcp", "mcp_toolset", "unknown_keyed_block", "chat_custom", "native_declaration"], + ) + async def test_post_call_judges_a_returned_namespace_member_as_that_member_beside_a_same_named_non_function( + self, same_named_tool: dict[str, object] + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-mcp", + rules=[{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]} + function = Function(name="lookup", arguments='{"country": "France"}') + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_lookup", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-mcp"], "tools": [namespace, same_named_tool]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + assert [call.function.name for call in response.choices[0].message.tool_calls or ()] == [] + + @pytest.mark.asyncio + async def test_post_call_judges_a_call_tagged_with_a_namespace_as_that_member_beside_a_same_named_function( + self, + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-mcp", + rules=[{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]} + top_level = {"type": "function", "function": {"name": "lookup"}} + tagged_call = { + "id": "call_lookup", + "type": "function", + "function": Function(name="lookup", arguments='{"country": "France"}'), + "provider_specific_fields": {"namespace": "mcp__weather"}, + } + response = ModelResponse( + choices=[Choices(finish_reason="tool_calls", message={"tool_calls": [tagged_call]})] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-mcp"], "tools": [namespace, top_level]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + message = response.choices[0].message + assert [call.function.name for call in message.tool_calls or ()] == [] + assert "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'" in (message.content or "") + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("decision", "default_action", "kept"), + [("allow", "deny", ["weather__get_weather"]), ("deny", "allow", [])], + ids=["bare_name_allowed", "bare_name_denied"], + ) + async def test_post_call_judges_a_returned_qualified_member_name_by_the_member_names_pre_call_judged( + self, decision: Literal["allow", "deny"], default_action: Literal["allow", "deny"], kept: list[str] + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-weather", + rules=[{"id": "get_weather", "tool_name": r"^get_weather$", "decision": decision}], + default_action=default_action, + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "weather", "tools": [{"type": "function", "name": "get_weather"}]} + top_level = {"type": "function", "function": {"name": "get_weather"}} + function = Function(name="weather__get_weather", arguments='{"city": "Istanbul"}') + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_weather", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-weather"], "tools": [namespace, top_level]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + assert [call.function.name for call in response.choices[0].message.tool_calls or ()] == kept + + @pytest.mark.asyncio + async def test_post_call_judges_a_top_level_function_by_its_own_name_beside_a_member_qualified_the_same( + self, + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name="post-exec", + rules=[{"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + namespace = {"type": "namespace", "name": "ops", "tools": [{"type": "function", "name": "exec"}]} + top_level = {"type": "function", "function": {"name": "ops__exec"}} + function = Function(name="ops__exec", arguments="{}") + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={"tool_calls": [{"id": "call_ops", "type": "function", "function": function}]}, + ) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["post-exec"], "tools": [namespace, top_level]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + assert [call.function.name for call in response.choices[0].message.tool_calls or ()] == ["ops__exec"] + @pytest.mark.asyncio async def test_async_post_call_success_hook_missing_arguments_blocks_param_rule( self, @@ -1361,6 +1703,1723 @@ class TestToolPermissionGuardrailAnthropicMessages: def _tool_use(self, name, tool_id="tu_1"): return {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": "ls"}} + def _always_on_pre_call(self, on_disallowed_action: Literal["block", "rewrite"]) -> ToolPermissionGuardrail: + return ToolPermissionGuardrail( + guardrail_name=f"anthropic-pre-call-{on_disallowed_action}", + rules=self.rules, + default_action="deny", + on_disallowed_action=on_disallowed_action, + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "denied_tool", + [ + {"name": "Read", "input_schema": {"type": "object", "properties": {}}}, + {"type": "custom", "name": "Read", "input_schema": {"type": "object", "properties": {}}}, + {"type": "function", "name": "Read", "parameters": {"type": "object", "properties": {}}}, + ], + ids=["anthropic", "anthropic_custom_type", "responses_api_flat_function"], + ) + async def test_pre_call_blocks_denied_request_tool_in_flat_format(self, denied_tool: dict[str, object]) -> None: + data = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "tools": [denied_tool]} + + with pytest.raises(HTTPException) as excinfo: + await self._always_on_pre_call("block").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(default_in_memory_ttl=1), + data=data, + call_type="anthropic_messages", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.detail["detection_message"] == "Tool 'Read' denied by rule 'deny_read'" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("tool_fields", "rule_tool_type", "call_type"), + [ + ({"format": {"type": "text"}}, r"^custom$", "responses"), + ({"input_schema": {"type": "object", "properties": {}}}, r"^function$", "anthropic_messages"), + ({"input_schema": {"type": "object", "properties": {}}}, r"^function$", "pass_through_endpoint"), + ({"custom": {"name": "apply_patch"}, "input_schema": {"type": "object"}}, r"^custom$", "acompletion"), + ], + ids=[ + "responses_custom_stays_custom", + "anthropic_custom_reads_as_function", + "anthropic_custom_reads_as_function_on_a_pass_through_route", + "chat_custom_with_a_stray_input_schema_stays_custom", + ], + ) + async def test_pre_call_tool_type_rules_follow_the_request_format( + self, tool_fields: dict[str, object], rule_tool_type: str, call_type: CallTypesLiteral + ) -> None: + guardrail = ToolPermissionGuardrail( + guardrail_name=f"tool-type-{call_type}", + rules=[{"id": "deny_type", "tool_type": rule_tool_type, "decision": "deny"}], + default_action="allow", + on_disallowed_action="block", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "custom", "name": "apply_patch", **tool_fields}], + } + + with pytest.raises(HTTPException) as excinfo: + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(default_in_memory_ttl=1), + data=data, + call_type=call_type, + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.detail["detection_message"] == "Tool 'apply_patch' denied by rule 'deny_type'" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("tool_shape", "tool_choice", "call_type", "expected_tool_choice"), + [ + ( + {"input_schema": {"type": "object", "properties": {}}}, + {"type": "tool", "name": "Read"}, + "anthropic_messages", + {"type": "none"}, + ), + ( + {"type": "function", "parameters": {"type": "object", "properties": {}}}, + {"type": "function", "name": "Read"}, + "responses", + "none", + ), + ], + ids=["anthropic", "responses_api"], + ) + async def test_pre_call_rewrite_strips_denied_flat_tool_and_forced_choice( + self, + tool_shape: dict[str, object], + tool_choice: dict[str, str], + call_type: CallTypesLiteral, + expected_tool_choice: dict[str, str] | str, + ) -> None: + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": "Bash", **tool_shape}, {"name": "Read", **tool_shape}], + "tool_choice": tool_choice, + } + + result = await self._always_on_pre_call("rewrite").async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(default_in_memory_ttl=1), + data=data, + call_type=call_type, + ) + + assert [tool["name"] for tool in result["tools"]] == ["Bash"] + assert result["tool_choice"] == expected_tool_choice + + def _pre_call( + self, + rules: list[dict[str, str]], + on_disallowed_action: Literal["block", "rewrite"], + default_action: Literal["allow", "deny"] = "allow", + ) -> ToolPermissionGuardrail: + return ToolPermissionGuardrail( + guardrail_name=f"request-shapes-{on_disallowed_action}", + rules=rules, + default_action=default_action, + on_disallowed_action=on_disallowed_action, + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + + async def _run_pre_call( + self, guardrail: ToolPermissionGuardrail, data: dict[str, object], call_type: CallTypesLiteral + ) -> dict[str, object]: + return await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(default_in_memory_ttl=1), + data=data, + call_type=call_type, + ) + + def _namespace(self, *member_names: str) -> dict[str, object]: + return { + "type": "namespace", + "name": "mcp__everything", + "tools": [{"type": "function", "name": name, "parameters": {"type": "object"}} for name in member_names], + } + + @pytest.mark.asyncio + async def test_pre_call_blocks_a_denied_tool_inside_a_responses_namespace(self) -> None: + guardrail = self._pre_call( + [{"id": "deny_sum", "tool_name": r"^mcp__everything__get_sum$", "decision": "deny"}], "block" + ) + data = {"model": "gpt-5", "input": "add 2 and 3", "tools": [self._namespace("echo", "get_sum")]} + + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "responses") + + assert excinfo.value.detail["detection_message"] == "Tool 'mcp__everything__get_sum' denied by rule 'deny_sum'" + + @pytest.mark.asyncio + async def test_pre_call_rewrite_drops_denied_namespace_members_and_empty_namespaces(self) -> None: + guardrail = self._pre_call([{"id": "deny_sum", "tool_name": r"^.*get_sum$", "decision": "deny"}], "rewrite") + partly_denied = {**self._namespace("echo", "get_sum"), "name": "mcp__kept"} + data = {"model": "gpt-5", "input": "add", "tools": [partly_denied, self._namespace("get_sum")]} + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [{**partly_denied, "tools": partly_denied["tools"][:1]}] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call_type", "malformed"), + [ + ( + "agenerate_content", + {"toolConfig": {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": [{"a": 1}]}}}, + ), + ("responses", {"input": [{"type": {"x": 1}, "content": "hi"}]}), + ("acompletion", {"extra_body": {"input": [{"type": [1]}]}}), + ("acompletion", {"extra_body": {"function_call": {"name": 5}}}), + ], + ids=["unhashable_allowed_function_name", "unhashable_input_type", "extra_body_input_type", "int_function_call"], + ) + async def test_pre_call_rewrite_leaves_a_malformed_value_for_the_provider_to_reject( + self, call_type: CallTypesLiteral, malformed: dict[str, object] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], "rewrite", "deny" + ) + tools = [{"type": "function", "function": {"name": name}} for name in ("get_weather", "shell")] + + result = await self._run_pre_call(guardrail, {"model": "m", "tools": tools, **malformed}, call_type) + + assert result["tools"] == tools[:1] + assert {key: result[key] for key in malformed} == malformed + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rule", "tool", "call_type", "detection_message"), + [ + ( + {"id": "deny_mcp", "tool_type": r"^mcp$", "decision": "deny"}, + {"type": "mcp", "server_label": "weather", "server_url": "https://mcp.example.com"}, + "responses", + "Tool 'weather' denied by rule 'deny_mcp'", + ), + ( + {"id": "allow_bash", "tool_name": r"^Bash$", "decision": "allow"}, + {"type": "mcp_toolset", "mcp_server_name": "weather"}, + "anthropic_messages", + "Tool 'weather' denied by default action", + ), + ( + {"id": "deny_weather", "tool_name": r"^weather$", "decision": "deny"}, + {"type": "mcp", "server_label": "weather", "server_url": "https://mcp.example.com"}, + "responses", + "Tool 'weather' denied by rule 'deny_weather'", + ), + ( + {"id": "deny_lookup", "tool_name": r"^mcp__weather__lookup$", "decision": "deny"}, + {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "custom", "name": "lookup"}]}, + "responses", + "Tool 'mcp__weather__lookup' denied by rule 'deny_lookup'", + ), + ( + {"id": "deny_lookup", "tool_name": r"^mcp__weather__lookup$", "decision": "deny"}, + {"functionDeclarations": [{"name": "mcp__weather__lookup"}, {"name": "get_weather"}]}, + "agenerate_content", + "Tool 'mcp__weather__lookup' denied by rule 'deny_lookup'", + ), + ( + {"id": "allow_bash", "tool_name": r"^Bash$", "decision": "allow"}, + {"googleSearch": {}}, + "acompletion", + "Tool 'googleSearch' denied by default action", + ), + ( + {"id": "deny_custom", "tool_type": r"^custom$", "decision": "deny"}, + {"type": "custom", "custom": {"name": "grammar_tool"}}, + "acompletion", + "Tool 'grammar_tool' denied by rule 'deny_custom'", + ), + ( + {"id": "deny_read", "tool_name": r"^Read$", "decision": "deny"}, + {"type": "custom", "custom": {"name": "Read"}}, + "acompletion", + "Tool 'Read' denied by rule 'deny_read'", + ), + ], + ids=[ + "responses_mcp_by_type", + "anthropic_mcp_toolset_by_server_name", + "responses_mcp_by_server_label", + "responses_namespace_custom_member", + "gemini_function_declarations", + "gemini_type_only_tool_by_its_key", + "chat_custom_keeps_its_type", + "chat_custom_nested_name", + ], + ) + async def test_pre_call_checks_tools_that_carry_no_top_level_name( + self, rule: dict[str, str], tool: dict[str, object], call_type: CallTypesLiteral, detection_message: str + ) -> None: + guardrail = self._pre_call([rule], "block", default_action="deny") + data = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "input": "hi", "tools": [tool]} + + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, call_type) + + assert excinfo.value.detail["detection_message"] == detection_message + + @pytest.mark.asyncio + async def test_pre_call_lets_a_type_rule_allow_a_gemini_tool_by_its_key(self) -> None: + guardrail = self._pre_call( + [{"id": "allow_search", "tool_type": r"^googleSearch$", "decision": "allow"}], "block", default_action="deny" + ) + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [{"googleSearch": {}}]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [{"googleSearch": {}}] + + @pytest.mark.asyncio + async def test_pre_call_blocks_a_denied_tool_loaded_by_responses_tool_search(self) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "block") + loaded = {"type": "tool_search_output", "tools": [{"type": "function", "name": "mcp__weather__lookup"}]} + data = {"model": "gpt-5", "input": [loaded], "tools": []} + + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "responses") + + assert excinfo.value.detail["detection_message"] == "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'" + + @pytest.mark.asyncio + async def test_pre_call_rewrite_keeps_the_allowed_members_of_grouped_and_loaded_tools(self) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + get_weather = {"type": "function", "name": "get_weather"} + lookup = {"type": "function", "name": "mcp__weather__lookup"} + data = { + "model": "m", + "input": [{"role": "user", "content": "hi"}, {"type": "tool_search_output", "tools": [get_weather, lookup]}], + "tools": [{"functionDeclarations": [{"name": "get_weather"}, {"name": "mcp__weather__lookup"}]}], + } + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [{"functionDeclarations": [{"name": "get_weather"}]}] + assert result["input"] == [ + {"role": "user", "content": "hi"}, + {"type": "tool_search_output", "tools": [get_weather]}, + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call_type", "message", "rewritten"), + [ + ( + "anthropic_messages", + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [{"type": "tool_reference", "tool_name": "mcp__weather__lookup"}], + }, + {"type": "text", "text": "Tool loaded."}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [ + { + "type": "text", + "text": "Permission denied: tool mcp__weather__lookup is not available", + } + ], + }, + {"type": "text", "text": "Tool loaded."}, + ], + }, + ), + ( + "anthropic_messages", + { + "role": "assistant", + "content": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [ + {"type": "tool_reference", "tool_name": "mcp__weather__lookup"}, + {"type": "tool_reference", "tool_name": "get_weather"}, + ], + }, + } + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, + } + ], + }, + ), + ( + "acompletion", + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "tool_reference", "tool_name": "mcp__weather__lookup"}, + {"type": "tool_reference", "tool_name": "get_weather"}, + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, + ), + ], + ) + async def test_pre_call_rewrite_drops_references_to_a_removed_tool(self, call_type, message, rewritten) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + data = { + "model": "m", + "messages": [{"role": "user", "content": "weather in Istanbul"}, message], + "tools": [ + {"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True}, + {"name": "mcp__weather__lookup", "input_schema": {"type": "object"}, "defer_loading": True}, + ], + } + + result = await self._run_pre_call(guardrail, data, call_type) + + assert [tool["name"] for tool in result["tools"]] == ["get_weather"] + assert result["messages"] == [{"role": "user", "content": "weather in Istanbul"}, rewritten] + + @pytest.mark.asyncio + @pytest.mark.parametrize("tool_choice", ["required", {"type": "function", "name": "get_weather"}]) + async def test_pre_call_rewrite_keeps_tool_choice_when_only_loaded_tools_remain(self, tool_choice) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + get_weather = {"type": "function", "name": "get_weather"} + lookup = {"type": "function", "name": "mcp__weather__lookup"} + data = { + "model": "m", + "input": [{"role": "user", "content": "hi"}, {"type": "tool_search_output", "tools": [get_weather, lookup]}], + "tools": [lookup], + "tool_choice": tool_choice, + } + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [] + assert result["input"][1] == {"type": "tool_search_output", "tools": [get_weather]} + assert result["tool_choice"] == tool_choice + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rule", "tools", "tool_choice"), + [ + ( + {"id": "deny_search", "tool_type": r"^web_search$", "decision": "deny"}, + [{"type": "web_search"}, {"type": "function", "name": "get_weather"}], + {"type": "web_search"}, + ), + ( + {"id": "deny_weather", "tool_name": r"^weather$", "decision": "deny"}, + [ + {"type": "mcp", "server_label": "weather", "server_url": "https://a.example.com"}, + {"type": "mcp", "server_label": "files", "server_url": "https://b.example.com"}, + ], + {"type": "mcp", "server_label": "weather"}, + ), + ], + ids=["hosted_tool_type", "hosted_mcp_server"], + ) + async def test_pre_call_rewrite_relaxes_a_hosted_tool_choice_it_removed( + self, rule: dict[str, str], tools: list[dict[str, str]], tool_choice: dict[str, str] + ) -> None: + guardrail = self._pre_call([rule], "rewrite") + data = {"model": "gpt-5", "input": "hi", "tools": tools, "tool_choice": tool_choice} + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == tools[1:] + assert result["tool_choice"] == "none" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("tools", "tool_choice", "call_type", "expected_tool_choice"), + [ + ( + [{"type": "custom", "name": "Bash"}, {"type": "custom", "name": "Read"}], + {"type": "custom", "name": "Read"}, + "responses", + "none", + ), + ( + [{"type": "function", "name": "Bash"}, {"type": "function", "name": "Read"}], + { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "Read"}, {"type": "function", "name": "Bash"}], + }, + "responses", + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "Bash"}]}, + ), + ( + [{"type": "function", "name": "Bash"}, {"type": "function", "name": "Read"}], + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "Read"}]}, + "responses", + "none", + ), + ( + [ + {"type": "function", "function": {"name": "Bash"}}, + {"type": "function", "function": {"name": "Read"}}, + ], + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "Read"}}]}, + }, + "acompletion", + "none", + ), + ( + [{"name": "Read", "input_schema": {"type": "object"}}], + {"type": "any"}, + "anthropic_messages", + {"type": "none"}, + ), + ( + [ + {"name": "Bash", "input_schema": {"type": "object"}}, + {"name": "Read", "input_schema": {"type": "object"}}, + ], + {"type": "any"}, + "anthropic_messages", + {"type": "any"}, + ), + ( + [{"type": "custom", "custom": {"name": "Bash"}}, {"type": "custom", "custom": {"name": "Read"}}], + {"type": "custom", "custom": {"name": "Read"}}, + "acompletion", + "none", + ), + ], + ids=[ + "responses_custom_choice", + "responses_allowed_tools_keeps_the_rest", + "responses_allowed_tools_all_denied", + "chat_allowed_tools_all_denied", + "anthropic_any_with_no_tools_left", + "anthropic_any_with_a_tool_left", + "chat_custom_choice", + ], + ) + async def test_pre_call_rewrite_leaves_no_tool_choice_that_needs_a_removed_tool( + self, + tools: list[dict[str, object]], + tool_choice: dict[str, object], + call_type: CallTypesLiteral, + expected_tool_choice: dict[str, object] | str, + ) -> None: + guardrail = self._pre_call(self.rules, "rewrite") + data = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "tools": tools, + "tool_choice": tool_choice, + } + + result = await self._run_pre_call(guardrail, data, call_type) + + assert result["tool_choice"] == expected_tool_choice + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("tool", "tool_choice", "call_type", "allowed_tool_type"), + [ + ({"type": "custom", "name": "apply_patch"}, {"type": "custom", "name": "apply_patch"}, "responses", r"^custom$"), + ( + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "tool", "name": "web_search"}, + "anthropic_messages", + r"^web_search_\d+$", + ), + ], + ids=["responses_custom_tool", "anthropic_server_tool"], + ) + async def test_pre_call_checks_a_forced_choice_with_its_tool_type( + self, + tool: dict[str, str], + tool_choice: dict[str, str], + call_type: CallTypesLiteral, + allowed_tool_type: str, + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_type", "tool_type": allowed_tool_type, "decision": "allow"}], "block", default_action="deny" + ) + data = {"model": "m", "input": "hi", "tools": [tool], "tool_choice": tool_choice} + + result = await self._run_pre_call(guardrail, data, call_type) + + assert result["tools"] == [tool] + assert result["tool_choice"] == tool_choice + + @pytest.mark.asyncio + async def test_pre_call_rewrite_relaxes_a_choice_for_a_tool_a_type_rule_removed(self) -> None: + guardrail = self._pre_call([{"id": "deny_custom", "tool_type": r"^custom$", "decision": "deny"}], "rewrite") + data = { + "model": "gpt-5", + "input": "patch it", + "tools": [{"type": "function", "name": "Bash"}, {"type": "custom", "name": "apply_patch"}], + "tool_choice": {"type": "custom", "name": "apply_patch"}, + } + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [{"type": "function", "name": "Bash"}] + assert result["tool_choice"] == "none" + + async def _blocked_and_rewritten( + self, rules: list[dict[str, str]], request: dict[str, object], call_type: CallTypesLiteral + ) -> tuple[str, dict[str, object]]: + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(self._pre_call(rules, "block"), copy.deepcopy(request), call_type) + rewritten = await self._run_pre_call(self._pre_call(rules, "rewrite"), request, call_type) + return excinfo.value.detail["detection_message"], rewritten + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("denied_tool", "detection_message"), + [ + ( + {"type": "function", "name": "mcp__weather__lookup"}, + "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'", + ), + ( + {"type": "namespace", "name": "mcp__weather", "tools": [{"type": "function", "name": "lookup"}]}, + "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'", + ), + ({"type": "web_search"}, "Tool 'web_search' denied by rule 'deny_search'"), + ], + ids=["flat", "namespace_member", "hosted_type"], + ) + async def test_pre_call_checks_and_strips_tools_in_responses_additional_tools_items( + self, denied_tool: dict[str, object], detection_message: str + ) -> None: + rules = [ + {"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}, + {"id": "deny_search", "tool_type": r"^web_search$", "decision": "deny"}, + ] + get_weather = {"type": "function", "name": "get_weather"} + request = { + "model": "gpt-5", + "input": [ + {"role": "user", "content": "hi"}, + {"type": "additional_tools", "tools": [get_weather, denied_tool]}, + {"type": "additional_tools", "tools": [denied_tool]}, + ], + "tools": [], + "tool_choice": "required", + } + + message, rewritten = await self._blocked_and_rewritten(rules, request, "responses") + + assert message == detection_message + assert rewritten["input"] == [ + {"role": "user", "content": "hi"}, + {"type": "additional_tools", "tools": [get_weather]}, + ] + assert rewritten["tool_choice"] == "required" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rules", "default_action", "kept"), + [ + ([{"id": "allow_exec", "tool_name": r"^exec$", "decision": "allow"}], "deny", ["run"]), + ([{"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}], "allow", []), + ( + [ + {"id": "allow_run", "tool_name": r"^run$", "decision": "allow"}, + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + ], + "allow", + [], + ), + ( + [ + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + {"id": "allow_all", "tool_name": r"^.*$", "decision": "allow"}, + ], + "allow", + [], + ), + ( + [ + {"id": "allow_exec", "tool_name": r"^exec$", "decision": "allow"}, + {"id": "deny_all", "tool_name": r"^.*$", "decision": "deny"}, + ], + "deny", + ["run"], + ), + ], + ids=[ + "namespace_allowed", + "namespace_denied", + "namespace_deny_beats_member_allow", + "namespace_deny_beats_a_catch_all_allow", + "namespace_allow_beats_a_catch_all_deny", + ], + ) + async def test_pre_call_judges_a_namespace_member_by_a_rule_on_its_namespace_name( + self, rules: list[dict[str, str]], default_action: Literal["allow", "deny"], kept: list[str] + ) -> None: + namespace = {"type": "namespace", "name": "exec", "tools": [{"type": "function", "name": "run"}]} + request = {"model": "gpt-5", "input": "Run pwd", "tools": [namespace]} + + rewritten = await self._run_pre_call(self._pre_call(rules, "rewrite", default_action), request, "responses") + + tools = rewritten.get("tools") or [] + assert [member["name"] for member in chain.from_iterable(tool["tools"] for tool in tools)] == kept + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call_type", "bash", "read"), + [ + ( + "acompletion", + {"type": "function", "function": {"name": "Bash"}}, + {"type": "function", "function": {"name": "Read"}}, + ), + ("responses", {"type": "function", "name": "Bash"}, {"type": "function", "name": "Read"}), + ( + "anthropic_messages", + {"name": "Bash", "input_schema": {"type": "object"}}, + {"name": "Read", "input_schema": {"type": "object"}}, + ), + ], + ids=["chat", "responses", "anthropic"], + ) + async def test_pre_call_checks_and_strips_tools_in_extra_body( + self, call_type: CallTypesLiteral, bash: dict[str, object], read: dict[str, object] + ) -> None: + request = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "input": "hi", + "extra_body": {"tools": [bash, read], "seed": 7}, + } + + message, rewritten = await self._blocked_and_rewritten(self.rules, request, call_type) + + assert message == "Tool 'Read' denied by rule 'deny_read'" + assert rewritten["extra_body"] == {"tools": [bash], "seed": 7} + + @pytest.mark.asyncio + async def test_pre_call_checks_and_strips_tools_loaded_in_extra_body_input(self) -> None: + rules = [{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}] + get_weather = {"type": "function", "name": "get_weather"} + lookup = {"type": "function", "name": "mcp__weather__lookup"} + user = {"role": "user", "content": "hi"} + request = { + "model": "gpt-5", + "input": "hi", + "tools": [], + "extra_body": {"input": [user, {"type": "tool_search_output", "tools": [get_weather, lookup]}], "seed": 7}, + } + + message, rewritten = await self._blocked_and_rewritten(rules, request, "responses") + + assert message == "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'" + assert rewritten["extra_body"] == { + "input": [user, {"type": "tool_search_output", "tools": [get_weather]}], + "seed": 7, + } + + @pytest.mark.asyncio + async def test_rewrite_relaxes_an_extra_body_tool_choice_that_names_a_dropped_tool(self) -> None: + guardrail = self._pre_call([{"id": "deny_delete", "tool_name": r"^delete_db$", "decision": "deny"}], "rewrite") + weather = {"type": "function", "function": {"name": "get_weather"}} + delete_db = {"type": "function", "function": {"name": "delete_db"}} + data = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "extra_body": { + "tools": [weather, delete_db], + "tool_choice": {"type": "function", "function": {"name": "delete_db"}}, + "seed": 7, + }, + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["extra_body"] == {"tools": [weather], "tool_choice": "none", "seed": 7} + + @pytest.mark.asyncio + async def test_rewrite_relaxes_a_required_tool_choice_when_the_extra_body_tools_sent_lose_every_tool( + self, + ) -> None: + guardrail = self._pre_call([{"id": "deny_delete", "tool_name": r"^delete_db$", "decision": "deny"}], "rewrite") + weather = {"type": "function", "function": {"name": "get_weather"}} + delete_db = {"type": "function", "function": {"name": "delete_db"}} + data = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "tools": [weather], + "extra_body": {"tools": [delete_db], "tool_choice": "required", "seed": 7}, + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + assert result["extra_body"] == {"tools": [], "tool_choice": "none", "seed": 7} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call_type", "tool_choice"), + [ + ("acompletion", {"type": "function", "function": {"name": "Read"}}), + ("responses", {"type": "function", "name": "Read"}), + ], + ids=["chat", "responses"], + ) + async def test_pre_call_checks_an_extra_body_tool_choice_that_names_an_unlisted_tool( + self, call_type: CallTypesLiteral, tool_choice: dict[str, object] + ) -> None: + request = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "input": "hi", + "tools": [{"type": "function", "name": "Bash"}], + "extra_body": {"tool_choice": tool_choice, "seed": 7}, + } + + message, rewritten = await self._blocked_and_rewritten(self.rules, request, call_type) + + assert message == "Tool 'Read' denied by rule 'deny_read'" + assert rewritten["extra_body"] == {"tool_choice": "none", "seed": 7} + + @pytest.mark.asyncio + async def test_pre_call_checks_and_strips_legacy_functions_in_extra_body(self) -> None: + rules = [{"id": "deny_delete", "tool_name": r"^delete_db$", "decision": "deny"}] + weather = {"name": "get_weather", "parameters": {"type": "object"}} + delete_db = {"name": "delete_db", "parameters": {"type": "object"}} + request = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "extra_body": {"functions": [weather, delete_db], "function_call": {"name": "delete_db"}, "seed": 7}, + } + + message, rewritten = await self._blocked_and_rewritten(rules, request, "acompletion") + + assert message == "Tool 'delete_db' denied by rule 'deny_delete'" + assert rewritten["extra_body"] == {"functions": [weather], "function_call": "none", "seed": 7} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "fields", + [ + {"functions": [{"name": "get_weather"}, {"name": "delete_db"}], "function_call": "delete_db"}, + { + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + "mcp_servers": [{"type": "url", "url": "https://mcp.example.com/mcp", "name": "delete_db"}], + "tool_choice": "required", + }, + { + "tools": [{"functionDeclarations": [{"name": "get_weather"}, {"name": "delete_db"}]}], + "toolConfig": {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": ["delete_db"]}}, + }, + ], + ids=["legacy_functions", "mcp_servers", "gemini_tool_config"], + ) + async def test_extra_body_fields_are_judged_and_rewritten_like_the_same_top_level_fields( + self, fields: dict[str, object] + ) -> None: + rules = [{"id": "deny_delete", "tool_name": r"^delete_db$", "decision": "deny"}] + base = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + top_request = {**base, **copy.deepcopy(fields)} + extra_request = {**base, "extra_body": {**copy.deepcopy(fields), "seed": 7}} + + top_message, top = await self._blocked_and_rewritten(rules, top_request, "acompletion") + extra_message, extra = await self._blocked_and_rewritten(rules, extra_request, "acompletion") + + top_fields = {key: top[key] for key in fields} + assert top_fields != fields + assert extra_message == top_message + assert extra["extra_body"] == {**top_fields, "seed": 7} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "rule", + [ + {"id": "deny_deepwiki", "tool_name": r"^deepwiki$", "decision": "deny"}, + {"id": "deny_deepwiki", "tool_name": r"^deepwiki$", "tool_type": r"^mcp_toolset$", "decision": "deny"}, + ], + ids=["by_server_name", "by_server_name_and_mcp_toolset_type"], + ) + async def test_pre_call_checks_and_strips_anthropic_mcp_servers_like_an_mcp_toolset( + self, rule: dict[str, str] + ) -> None: + deepwiki = {"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"} + files = {"type": "url", "url": "https://files.example.com/mcp", "name": "files"} + request = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "mcp_servers": [deepwiki, files], + "tool_choice": {"type": "any"}, + } + + message, rewritten = await self._blocked_and_rewritten([rule], request, "anthropic_messages") + + assert message == "Tool 'deepwiki' denied by rule 'deny_deepwiki'" + assert rewritten["mcp_servers"] == [files] + assert rewritten["tool_choice"] == {"type": "any"} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rule", "tool", "detection_message", "kept_tools"), + [ + ( + {"id": "deny_code", "tool_type": r"^codeExecution$", "decision": "deny"}, + {"googleSearch": {}, "codeExecution": {}}, + "Tool 'codeExecution' denied by rule 'deny_code'", + [{"googleSearch": {}}], + ), + ( + {"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}, + {"functionDeclarations": [{"name": "mcp__weather__lookup"}], "googleSearch": {}}, + "Tool 'mcp__weather__lookup' denied by rule 'deny_mcp'", + [{"googleSearch": {}}], + ), + ( + {"id": "deny_search", "tool_type": r"^googleSearch$", "decision": "deny"}, + {"functionDeclarations": [{"name": "get_weather"}], "googleSearch": {}}, + "Tool 'googleSearch' denied by rule 'deny_search'", + [{"functionDeclarations": [{"name": "get_weather"}]}], + ), + ( + {"id": "deny_builtin", "tool_type": r"^(googleSearch|codeExecution)$", "decision": "deny"}, + {"googleSearch": {}, "codeExecution": {}}, + "Tool 'googleSearch' denied by rule 'deny_builtin'", + [], + ), + ], + ids=["denied_key_beside_a_key", "every_declaration_denied", "denied_key_beside_declarations", "all_denied"], + ) + async def test_pre_call_checks_and_strips_each_part_of_a_multi_key_gemini_tool( + self, + rule: dict[str, str], + tool: dict[str, object], + detection_message: str, + kept_tools: list[dict[str, object]], + ) -> None: + request = {"model": "gemini-3", "contents": [{"role": "user", "parts": [{"text": "hi"}]}], "tools": [tool]} + + message, rewritten = await self._blocked_and_rewritten([rule], request, "agenerate_content") + + assert message == detection_message + assert rewritten["tools"] == kept_tools + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + @pytest.mark.parametrize( + ("tool", "call_type"), + [({"type": "web_search"}, "responses"), ({"googleSearch": {}}, "acompletion")], + ids=["responses_hosted_type", "gemini_tool_key"], + ) + async def test_pre_call_matches_a_name_rule_against_a_nameless_tool_by_its_type( + self, + tool: dict[str, object], + call_type: CallTypesLiteral, + on_disallowed_action: Literal["block", "rewrite"], + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_all", "tool_name": r".*", "decision": "allow"}], on_disallowed_action, default_action="deny" + ) + data = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "input": "hi", "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, call_type) + + assert result["tools"] == [tool] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("rules", "default_action", "detection_message"), + [ + ( + [{"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}], + "allow", + "Tool 'exec' denied by rule 'deny_exec'", + ), + ([{"id": "allow_exec", "tool_name": r"^exec$", "decision": "allow"}], "deny", None), + ( + [ + {"id": "allow_exec", "tool_name": r"^exec$", "decision": "allow"}, + {"id": "deny_functions", "tool_name": r"^functions__.*$", "decision": "deny"}, + ], + "allow", + "Tool 'functions__exec' denied by rule 'deny_functions'", + ), + ( + [ + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + {"id": "allow_functions", "tool_type": r"^function$", "decision": "allow"}, + ], + "allow", + "Tool 'exec' denied by rule 'deny_exec'", + ), + ( + [ + {"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}, + {"id": "allow_all", "tool_name": r"^.*$", "decision": "allow"}, + ], + "deny", + "Tool 'exec' denied by rule 'deny_exec'", + ), + ( + [ + {"id": "allow_functions", "tool_name": r"^functions__.*$", "decision": "allow"}, + {"id": "deny_all", "tool_name": r"^.*$", "decision": "deny"}, + ], + "allow", + None, + ), + ], + ids=[ + "bare_name_deny_rule", + "bare_name_allow_rule", + "qualified_rule_wins_over_an_earlier_bare_rule", + "type_rule_keeps_an_earlier_bare_name_rule_in_charge", + "bare_name_deny_beats_a_later_catch_all_allow", + "qualified_allow_beats_a_later_catch_all_deny", + ], + ) + async def test_pre_call_judges_a_namespace_member_by_its_qualified_and_bare_names( + self, rules: list[dict[str, str]], default_action: Literal["allow", "deny"], detection_message: str | None + ) -> None: + guardrail = self._pre_call(rules, "block", default_action=default_action) + namespace = {"type": "namespace", "name": "functions", "tools": [{"type": "function", "name": "exec"}]} + data = {"model": "gpt-5", "input": "run date", "tools": [namespace]} + + if detection_message is None: + assert (await self._run_pre_call(guardrail, data, "responses"))["tools"] == [namespace] + return + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "responses") + assert excinfo.value.detail["detection_message"] == detection_message + + @pytest.mark.asyncio + async def test_pre_call_rewrite_drops_a_namespace_member_a_bare_name_rule_denies(self) -> None: + guardrail = self._pre_call([{"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}], "rewrite") + members = [{"type": "function", "name": "exec"}, {"type": "function", "name": "read_file"}] + namespace = {"type": "namespace", "name": "functions", "tools": members} + data = {"model": "gpt-5", "input": "run date", "tools": [namespace]} + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [{"type": "namespace", "name": "functions", "tools": members[1:]}] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("choice", "relaxed"), + [ + ({"tool_choice": {"type": "function", "function": {"name": "exec"}}}, {"tool_choice": "none"}), + ({"tool_choice": {"type": "function", "function": {"name": "ops"}}}, {"tool_choice": "none"}), + ( + { + "tool_choice": { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "exec"}}, + {"type": "function", "function": {"name": "get_weather"}}, + ], + }, + } + }, + { + "tool_choice": { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + }, + } + }, + ), + ( + {"toolConfig": {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": ["exec"]}}}, + {"toolConfig": {"functionCallingConfig": {"mode": "NONE"}}}, + ), + ], + ids=["bare_member_choice", "namespace_choice", "allowed_tools_entry", "allowed_function_names"], + ) + async def test_pre_call_rewrite_relaxes_a_bare_name_a_dropped_namespace_member_was_sent_under( + self, choice: dict[str, object], relaxed: dict[str, object] + ) -> None: + guardrail = self._pre_call([{"id": "deny_exec", "tool_name": r"^ops__exec$", "decision": "deny"}], "rewrite") + weather = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} + namespace = {"type": "namespace", "name": "ops", "tools": [{"type": "function", "name": "exec"}]} + data = { + "model": "gemini/gemini-2.5-flash", + "messages": [{"role": "user", "content": "hi"}], + "tools": [weather, namespace], + **choice, + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + assert {key: result[key] for key in relaxed} == relaxed + + @pytest.mark.asyncio + async def test_pre_call_rewrite_keeps_a_bare_name_another_kept_namespace_still_serves(self) -> None: + guardrail = self._pre_call([{"id": "deny_exec", "tool_name": r"^ops__exec$", "decision": "deny"}], "rewrite") + dropped_namespace = {"type": "namespace", "name": "ops", "tools": [{"type": "function", "name": "exec"}]} + kept_namespace = {"type": "namespace", "name": "dev", "tools": [{"type": "function", "name": "exec"}]} + tool_choice = {"type": "function", "function": {"name": "exec"}} + data = { + "model": "gemini/gemini-2.5-flash", + "messages": [{"role": "user", "content": "hi"}], + "tools": [dropped_namespace, kept_namespace], + "tool_choice": tool_choice, + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [kept_namespace] + assert result["tool_choice"] == tool_choice + + @pytest.mark.asyncio + @pytest.mark.parametrize("exec_first", [True, False], ids=["exec_namespace_first", "exec_namespace_last"]) + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_denies_a_bare_name_another_namespace_shares_a_qualified_name_with( + self, on_disallowed_action: Literal["block", "rewrite"], exec_first: bool + ) -> None: + guardrail = self._pre_call( + [{"id": "deny_exec", "tool_name": r"^exec$", "decision": "deny"}], on_disallowed_action + ) + exec_namespace = {"type": "namespace", "name": "mcp__ops", "tools": [{"type": "function", "name": "exec"}]} + twin_namespace = {"type": "namespace", "name": "mcp", "tools": [{"type": "function", "name": "ops__exec"}]} + tools = [exec_namespace, twin_namespace] if exec_first else [twin_namespace, exec_namespace] + data = {"model": "gpt-5", "input": "run date", "tools": tools} + + if on_disallowed_action == "rewrite": + assert (await self._run_pre_call(guardrail, data, "responses"))["tools"] == [] + return + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "responses") + assert excinfo.value.detail["detection_message"] == "Tool 'exec' denied by rule 'deny_exec'" + + @pytest.mark.asyncio + @pytest.mark.parametrize("placement", ["tools", "functions"]) + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + @pytest.mark.parametrize( + ("member", "detection_message"), + [ + ({"type": "function", "function": {"name": "delete_db"}}, "Tool 'delete_db' denied by rule 'deny_delete'"), + ( + {"type": "function", "name": "lookup", "function": {"name": "delete_db"}}, + "Tool 'delete_db' denied by rule 'deny_delete'", + ), + ({"googleSearch": {}}, "Tool 'googleSearch' denied by rule 'deny_search'"), + ( + { + "type": "namespace", + "name": "inner", + "tools": [{"type": "function", "name": "noop"}], + "function": {"name": "delete_db"}, + }, + "Tool 'delete_db' denied by rule 'deny_delete'", + ), + ], + ids=["chat_shaped_member", "function_object_behind_another_name", "hosted_key_member", "nested_namespace"], + ) + async def test_pre_call_judges_a_namespace_member_by_what_a_provider_reads_from_it( + self, + member: dict[str, object], + detection_message: str, + on_disallowed_action: Literal["block", "rewrite"], + placement: Literal["tools", "functions"], + ) -> None: + guardrail = self._pre_call( + [ + {"id": "deny_delete", "tool_name": r"^delete_db$", "decision": "deny"}, + {"id": "deny_search", "tool_type": r"^googleSearch$", "decision": "deny"}, + ], + on_disallowed_action, + ) + weather = {"type": "function", "name": "get_weather"} + data = { + "model": "gemini-3", + "messages": [{"role": "user", "content": "hi"}], + placement: [{"type": "namespace", "name": "ops", "tools": [weather, member]}], + } + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == detection_message + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result[placement] == [{"type": "namespace", "name": "ops", "tools": [weather]}] + + @pytest.mark.asyncio + async def test_pre_call_allows_a_chat_shaped_namespace_member_the_allowlist_names(self) -> None: + guardrail = self._pre_call( + [{"id": "allow_create", "tool_name": r"^github__create_issue$", "decision": "allow"}], + "block", + default_action="deny", + ) + namespace = { + "type": "namespace", + "name": "github", + "tools": [{"type": "function", "function": {"name": "create_issue"}}], + } + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [namespace]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [namespace] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("tool_config", "rewritten_tool_config"), + [ + ( + { + "functionCallingConfig": { + "mode": "ANY", + "allowedFunctionNames": ["mcp__weather__lookup", "get_weather"], + } + }, + {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": ["get_weather"]}}, + ), + ( + {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": ["mcp__weather__lookup"]}}, + {"functionCallingConfig": {"mode": "NONE"}}, + ), + ( + {"function_calling_config": {"mode": "any", "allowed_function_names": ["mcp__weather__lookup"]}}, + {"function_calling_config": {"mode": "NONE"}}, + ), + ({"functionCallingConfig": {"mode": "ANY"}}, {"functionCallingConfig": {"mode": "ANY"}}), + ( + {"functionCallingConfig": {"mode": "AUTO", "allowedFunctionNames": []}}, + {"functionCallingConfig": {"mode": "AUTO", "allowedFunctionNames": []}}, + ), + ], + ids=[ + "some_allowed_names_left", + "every_allowed_name_removed", + "snake_case_fields", + "no_allowed_names", + "empty_allowed_names_list", + ], + ) + async def test_pre_call_rewrite_drops_removed_functions_from_a_gemini_tool_config( + self, tool_config: dict[str, object], rewritten_tool_config: dict[str, object] + ) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + config_key = "tool_config" if "function_calling_config" in tool_config else "toolConfig" + data = { + "model": "gemini-3", + "contents": [{"role": "user", "parts": [{"text": "weather in Paris?"}]}], + "tools": [{"functionDeclarations": [{"name": "mcp__weather__lookup"}, {"name": "get_weather"}]}], + config_key: tool_config, + } + + result = await self._run_pre_call(guardrail, data, "agenerate_content") + + assert result["tools"] == [{"functionDeclarations": [{"name": "get_weather"}]}] + assert result[config_key] == rewritten_tool_config + + @pytest.mark.asyncio + async def test_pre_call_rewrite_turns_off_forced_gemini_calling_when_no_function_is_left(self) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + data = { + "model": "gemini-3", + "contents": [{"role": "user", "parts": [{"text": "weather in Paris?"}]}], + "tools": [{"functionDeclarations": [{"name": "mcp__weather__lookup"}]}, {"googleSearch": {}}], + "toolConfig": {"functionCallingConfig": {"mode": "ANY"}}, + } + + result = await self._run_pre_call(guardrail, data, "agenerate_content") + + assert result["tools"] == [{"googleSearch": {}}] + assert result["toolConfig"] == {"functionCallingConfig": {"mode": "NONE"}} + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_judges_a_named_hosted_tool_by_its_type_as_well_as_its_name( + self, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + weather = {"type": "function", "function": {"name": "get_weather"}} + data = { + "model": "gemini-3", + "messages": [{"role": "user", "content": "hi"}], + "tools": [weather, {"type": "web_search", "name": "get_weather"}], + } + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == "Tool 'web_search' denied by default action" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_judges_a_gemini_hosted_key_by_the_key_whatever_name_it_carries( + self, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + weather = {"type": "function", "function": {"name": "get_weather"}} + computer_use = {"computerUse": {"name": "get_weather", "environment": "browser"}} + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [weather, computer_use]} + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == "Tool 'computerUse' denied by default action" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "rules, tool", + [ + ( + [{"id": "allow_bash", "tool_name": r"^bash$", "decision": "allow"}], + {"type": "bash_20250124", "name": "bash"}, + ), + ( + [{"id": "allow_edit", "tool_name": r"^str_replace_based_edit_tool$", "decision": "allow"}], + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + ), + ( + [{"id": "allow_search", "tool_name": r"^web_search$", "decision": "allow"}], + {"type": "web_search_20250305", "name": "web_search"}, + ), + ( + [ + { + "id": "allow_search", + "tool_name": r"^web_search$", + "tool_type": r"^web_search_20250305$", + "decision": "allow", + } + ], + {"type": "web_search_20250305", "name": "web_search"}, + ), + ], + ids=["client_tool_by_name", "client_tool_with_its_own_name", "server_tool_by_name", "name_and_type_rule"], + ) + async def test_pre_call_allows_an_anthropic_defined_tool_that_an_allowlist_names( + self, rules: list[dict[str, object]], tool: dict[str, object] + ) -> None: + guardrail = self._pre_call(rules, "block", default_action="deny") + data = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [tool] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "tool, call_type", + [ + ({"type": "web_search_20250305", "name": "get_weather"}, "anthropic_messages"), + ( + { + "type": "computer_20251124", + "name": "get_weather", + "function": {"parameters": {"display_width_px": 1024, "display_height_px": 768}}, + }, + "acompletion", + ), + ({"type": "computer_use", "name": "get_weather", "environment": "browser"}, "acompletion"), + ({"type": "text_editor_20250728", "name": "get_weather"}, "anthropic_messages"), + ], + ids=["server_tool", "anthropic_computer_tool", "gemini_computer_use", "text_editor"], + ) + async def test_pre_call_blocks_a_provider_defined_tool_whose_name_is_not_its_own( + self, tool: dict[str, object], call_type: CallTypesLiteral + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + "block", + default_action="deny", + ) + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, call_type) + + assert excinfo.value.detail["detection_message"] == f"Tool '{tool['type']}' denied by default action" + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + @pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "delete_db"}, "functionDeclarations": [{"name": "get_weather"}]}, + {"type": "function", "function": {"name": "delete_db"}, "function_declarations": [{"name": "get_weather"}]}, + {"type": "custom", "name": "get_weather", "function": {"name": "delete_db"}}, + {"type": "mcp", "server_label": "get_weather", "function": {"name": "delete_db"}}, + { + "type": "namespace", + "name": "github", + "tools": [{"type": "function", "name": "create_issue"}], + "function": {"name": "delete_db"}, + }, + { + "type": "computer_20251124", + "name": "computer", + "function": {"name": "delete_db", "parameters": {"display_width_px": 1024, "display_height_px": 768}}, + }, + ], + ids=[ + "declarations_list", + "snake_case_declarations_list", + "custom_top_level_name", + "mcp_server_label", + "namespace_with_members", + "computer_top_level_name", + ], + ) + async def test_pre_call_judges_the_function_a_tool_runs_behind_an_allowed_name( + self, tool: dict[str, object], on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_known", "tool_name": r"^(get_weather|computer|github__create_issue)$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + weather = {"type": "function", "function": {"name": "get_weather"}} + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [weather, tool]} + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == "Tool 'delete_db' denied by default action" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + @pytest.mark.parametrize( + ("tool", "hosted_key"), + [ + ({"type": "get_weather", "googleSearch": {}}, "googleSearch"), + ({"type": "function", "codeExecution": {}}, "codeExecution"), + ({"type": "get_weather", "url_context": {}}, "url_context"), + ], + ids=["named_type_google_search", "function_type_code_execution", "named_type_url_context"], + ) + async def test_pre_call_judges_a_gemini_hosted_tool_key_whatever_the_tool_type( + self, tool: dict[str, object], hosted_key: str, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [ + {"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}, + {"id": "allow_functions", "tool_type": r"^function$", "decision": "allow"}, + ], + on_disallowed_action, + default_action="deny", + ) + weather = {"type": "function", "function": {"name": "get_weather"}} + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [weather, tool]} + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == f"Tool '{hosted_key}' denied by default action" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [weather] + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_leaves_a_gemini_hosted_key_that_sits_beside_a_function( + self, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + tool = {"type": "function", "function": {"name": "get_weather"}, "googleSearch": {}} + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [tool] + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + @pytest.mark.parametrize( + ("function", "denied"), + [ + ({"type": "function", "function": {"name": "delete_db"}}, "delete_db"), + ({"name": "get_weather", "function": {"name": "delete_db"}}, "delete_db"), + ({"googleSearch": {}}, "googleSearch"), + ], + ids=["function_object", "function_object_behind_an_allowed_name", "gemini_hosted_key"], + ) + async def test_pre_call_judges_a_legacy_functions_entry_like_a_tool( + self, function: dict[str, object], denied: str, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + weather = {"name": "get_weather"} + data = { + "model": "gemini-3", + "messages": [{"role": "user", "content": "hi"}], + "functions": [weather, function], + } + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == f"Tool '{denied}' denied by default action" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["functions"] == [weather] + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_judges_a_legacy_functions_entry_as_a_function_whatever_its_type( + self, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "deny_delete", "tool_name": r"^delete_db$", "tool_type": r"^function$", "decision": "deny"}], + on_disallowed_action, + ) + weather = {"name": "get_weather"} + data = { + "model": "gemini-3", + "messages": [{"role": "user", "content": "hi"}], + "functions": [weather, {"type": "custom", "name": "delete_db"}], + } + + if on_disallowed_action == "block": + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "acompletion") + assert excinfo.value.detail["detection_message"] == "Tool 'delete_db' denied by rule 'deny_delete'" + return + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["functions"] == [weather] + + @pytest.mark.asyncio + async def test_rewrite_keeps_an_allowed_function_whose_declaration_list_is_denied(self) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + "rewrite", + default_action="deny", + ) + tool = { + "type": "function", + "function": {"name": "get_weather"}, + "functionDeclarations": [{"name": "delete_db"}], + } + data = {"model": "gemini-3", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [{"type": "function", "function": {"name": "get_weather"}}] + + @pytest.mark.asyncio + async def test_pre_call_allows_a_custom_tool_by_its_function_name(self) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + "block", + default_action="deny", + ) + tool = {"type": "custom", "function": {"name": "get_weather"}} + data = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [tool] + + @pytest.mark.asyncio + async def test_pre_call_blocks_a_tool_whose_type_is_not_a_string_by_its_name(self) -> None: + guardrail = self._pre_call( + [ + {"id": "allow_search", "tool_type": r"^web_search$", "decision": "allow"}, + {"id": "deny_read", "tool_name": r"^Read$", "decision": "deny"}, + ], + "block", + ) + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": ["custom"], "name": "Read", "input_schema": {"type": "object"}}], + } + + with pytest.raises(HTTPException) as excinfo: + await self._run_pre_call(guardrail, data, "anthropic_messages") + + assert excinfo.value.status_code == 400 + assert excinfo.value.detail["detection_message"] == "Tool 'Read' denied by rule 'deny_read'" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("malformed_tool", "tool_choice", "call_type"), + [ + ({"type": ["custom"], "name": "Read", "input_schema": {"type": "object"}}, None, "anthropic_messages"), + ({"type": "function", "name": "Read"}, {"type": {"a": 1}}, "responses"), + ], + ids=["tool_type_is_a_list", "tool_choice_type_is_an_object"], + ) + async def test_pre_call_rewrite_strips_a_denied_tool_and_leaves_malformed_fields_for_the_provider( + self, malformed_tool: dict[str, object], tool_choice: dict[str, object] | None, call_type: CallTypesLiteral + ) -> None: + guardrail = self._pre_call([{"id": "deny_read", "tool_name": r"^Read$", "decision": "deny"}], "rewrite") + bash = {"type": "function", "name": "Bash"} + data = { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "tools": [bash, malformed_tool], + **({"tool_choice": tool_choice} if tool_choice is not None else {}), + } + + result = await self._run_pre_call(guardrail, data, call_type) + + assert result["tools"] == [bash] + assert result.get("tool_choice") == tool_choice + + @pytest.mark.asyncio + async def test_pre_call_judges_a_tool_with_a_null_type_by_its_nested_function_name(self) -> None: + guardrail = self._pre_call( + [{"id": "allow_read", "tool_name": r"^Read$", "decision": "allow"}], "block", default_action="deny" + ) + tool = {"type": None, "function": {"name": "Read", "parameters": {"type": "object"}}} + data = {"model": "gpt-5", "messages": [{"role": "user", "content": "hi"}], "tools": [tool]} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == [tool] + + @pytest.mark.asyncio + async def test_pre_call_rewrite_drops_a_removed_hosted_tool_from_allowed_tools(self) -> None: + guardrail = self._pre_call([{"id": "deny_search", "tool_type": r"^web_search$", "decision": "deny"}], "rewrite") + weather = {"type": "function", "name": "get_weather"} + tool_choice = {"type": "allowed_tools", "mode": "required", "tools": [weather, {"type": "web_search"}]} + data = {"model": "gpt-5", "input": "hi", "tools": [weather, {"type": "web_search"}], "tool_choice": tool_choice} + + result = await self._run_pre_call(guardrail, data, "responses") + + assert result["tools"] == [weather] + assert result["tool_choice"] == {"type": "allowed_tools", "mode": "required", "tools": [weather]} + + @pytest.mark.asyncio + @pytest.mark.parametrize("tools", [{"not": "a list"}, '[{"type": "function"}]'], ids=["object", "json_string"]) + async def test_pre_call_rewrite_leaves_a_tools_field_that_is_not_a_list_for_the_provider_to_reject( + self, tools: object + ) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + data = { + "model": "gpt-5", + "messages": [{"role": "user", "content": "hi"}], + "tools": tools, + "tool_choice": {"type": "function", "function": {"name": "mcp__weather__lookup"}}, + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == tools + assert result["tool_choice"] == "none" + + @pytest.mark.asyncio + @pytest.mark.parametrize("on_disallowed_action", ["block", "rewrite"]) + async def test_pre_call_reads_bedrock_tool_blocks_by_name_and_skips_cache_points( + self, on_disallowed_action: Literal["block", "rewrite"] + ) -> None: + guardrail = self._pre_call( + [{"id": "allow_weather", "tool_name": r"^get_weather$", "decision": "allow"}], + on_disallowed_action, + default_action="deny", + ) + weather = {"toolSpec": {"name": "get_weather", "inputSchema": {"json": {}}}} + tools = [weather, {"cachePoint": {"type": "default"}}] + data = {"model": "bedrock/claude", "messages": [{"role": "user", "content": "hi"}], "tools": tools} + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == tools + + @pytest.mark.asyncio + async def test_pre_call_rewrite_drops_a_denied_bedrock_tool_block_by_its_name(self) -> None: + guardrail = self._pre_call([{"id": "deny_mcp", "tool_name": r"^mcp__.*$", "decision": "deny"}], "rewrite") + kept = [{"toolSpec": {"name": "get_weather"}}, {"cachePoint": {"type": "default"}}] + data = { + "model": "bedrock/claude", + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"toolSpec": {"name": "mcp__weather__lookup"}}, *kept], + } + + result = await self._run_pre_call(guardrail, data, "acompletion") + + assert result["tools"] == kept + @pytest.mark.asyncio async def test_denied_anthropic_tool_use_is_blocked(self): response = self._response({"type": "text", "text": "reading"}, self._tool_use("Read")) diff --git a/tests/unit/proxy/guardrails/test_usage_tracking.py b/tests/unit/proxy/guardrails/test_usage_tracking.py index 69ec098b840..7e92c9be4e8 100644 --- a/tests/unit/proxy/guardrails/test_usage_tracking.py +++ b/tests/unit/proxy/guardrails/test_usage_tracking.py @@ -654,3 +654,15 @@ async def test_one_failing_index_statement_does_not_drop_the_others_or_the_rollu assert prisma.db.litellm_spendlogguardrailindex.create_many.await_count == 3 assert prisma.db.litellm_dailyguardrailmetrics.upsert.await_count == len(guardrail_ids) + + +@pytest.mark.asyncio +async def test_disabled_log_deliveries_do_not_enable_guardrail_analytics(): + prisma = _prisma() + row = _payload("lean", usage={"topicPolicyUnits": 2}) + row["accounting_payload"] = {"request_id": "lean"} + row["accounting_log_disabled"] = True + await process_spend_logs_guardrail_usage(prisma, [row], pending=PendingRollups()) + prisma.db.litellm_dailyguardrailmetrics.upsert.assert_not_awaited() + prisma.db.litellm_spendlogguardrailindex.create_many.assert_not_awaited() + prisma.db.litellm_dailyguardrailusageunits.upsert.assert_not_awaited() diff --git a/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py b/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py index 45cc7eb8922..966e94fad80 100644 --- a/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py +++ b/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py @@ -8,6 +8,9 @@ Tests that session-scoped budget tracking works correctly: - Requests without agent_id pass through """ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final from unittest.mock import patch import logging @@ -22,6 +25,7 @@ from litellm.proxy.hooks.max_budget_per_session_limiter import ( ) from litellm.proxy.utils import InternalUsageCache from litellm.types.agents import AgentResponse +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG def _make_mock_agent(max_budget_per_session: float) -> AgentResponse: @@ -145,6 +149,255 @@ async def test_budget_per_session_independent_sessions(): assert result is None +@pytest.mark.asyncio +async def test_repeated_polls_of_a_finished_job_charge_the_session_once(): + from openai.types import Batch + + from litellm.types.videos.main import VideoObject + + handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(DualCache())) + video = VideoObject(id="job-1", object="video", status="completed") + batch = Batch( + id="job-1", completion_window="24h", created_at=1, endpoint="/v1/chat/completions", + input_file_id="file-1", object="batch", status="completed", output_file_id="file-out", + ) + polls = (("avideo_status", video, 0.4), ("aretrieve_batch", batch, 0.1), ("acompletion", None, 0.01)) + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + for call_type, response_obj, cost in polls * 3: + await handler.async_log_success_event( + { + "call_type": call_type, + "response_cost": cost, + POLLED_VIDEO_STATUS_KWARG: "completed", + "litellm_params": {"metadata": {"session_id": "session-jobs", "agent_id": "agent-budget-123"}}, + }, + response_obj, + None, + None, + ) + + spend = await handler._get_current_spend(handler._make_cache_key("session-jobs")) + assert spend == pytest.approx(0.4 + 0.1 + 3 * 0.01) + + +@pytest.mark.asyncio +async def test_a_batch_retrieve_before_its_cost_is_final_charges_the_session_nothing(): + from openai.types import Batch + + handler: Final = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(DualCache())) + + def polled(status: str, output_file_id: str | None = None) -> Batch: + return Batch( + id="batch-1", completion_window="24h", created_at=1, endpoint="/v1/chat/completions", + input_file_id="file-1", object="batch", status=status, output_file_id=output_file_id, + ) + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + for response_obj, cost in ( + (polled("in_progress"), 0.01), + (polled("completed"), 0.01), + (polled("completed", "file-out"), 0.25), + (polled("completed", "file-out"), 0.25), + ): + await handler.async_log_success_event( + { + "call_type": "aretrieve_batch", + "response_cost": cost, + "litellm_params": {"metadata": {"session_id": "session-batch", "agent_id": "agent-budget-123"}}, + }, + response_obj, + None, + None, + ) + + spend: Final = await handler._get_current_spend(handler._make_cache_key("session-batch")) + assert spend == pytest.approx(0.25) + + +def _video_poll( + video_cost: float, guardrail_cost: float = 0.0, session_id: str = "session-jobs", status: str = "completed" +) -> dict[str, object]: + return { + "call_type": "avideo_status", + "response_cost": video_cost, + POLLED_VIDEO_STATUS_KWARG: status, + "standard_logging_object": { + "response_cost": video_cost + guardrail_cost, + "guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": guardrail_cost}], + }, + "litellm_params": {"metadata": {"session_id": session_id, "agent_id": "agent-budget-123"}}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("video_cost", "guardrail_cost"), + [(0.40, 0.01), (0.05, 0.10)], + ids=["video_costs_more_than_its_guardrail", "video_costs_no_more_than_its_guardrail"], +) +async def test_every_video_poll_pays_its_guardrail_and_only_the_finished_poll_pays_the_session_for_the_video( + video_cost: float, guardrail_cost: float +): + from litellm.types.videos.main import VideoObject + + handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(DualCache())) + completed_polls: Final = 3 + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + await handler.async_log_success_event( + _video_poll(0.0, guardrail_cost=guardrail_cost, status="in_progress"), + VideoObject(id="job-1", object="video", status="in_progress"), + None, + None, + ) + for _ in range(completed_polls): + await handler.async_log_success_event( + _video_poll(video_cost, guardrail_cost=guardrail_cost), + VideoObject(id="job-1", object="video", status="completed"), + None, + None, + ) + + spend = await handler._get_current_spend(handler._make_cache_key("session-jobs")) + assert spend == pytest.approx(video_cost + (1 + completed_polls) * guardrail_cost) + + +class _ClockedRedis: + """The Redis calls the session limiter makes, on a clock the test moves.""" + + def __init__(self) -> None: + self.now = 0.0 + self.entries: Mapping[str, tuple[float, float | None]] = MappingProxyType({}) + self.scripts_fail = False + + def _live(self, key: str) -> tuple[float, float | None] | None: + entry = self.entries.get(key) + if entry is None or (entry[1] is not None and entry[1] <= self.now): + return None + return entry + + def _store(self, key: str, value: float, expires_at: float | None) -> float: + self.entries = MappingProxyType({**self.entries, key: (value, expires_at)}) + return value + + async def async_get_cache(self, key: str, **kwargs: object) -> float | None: + entry = self._live(key) + return None if entry is None else entry[0] + + async def async_increment( + self, key: str, value: float, ttl: int | None = None, parent_otel_span: object = None, refresh_ttl: bool = False + ) -> float: + live = self._live(key) + kept_expiry = None if live is None else live[1] + expires_at = kept_expiry if ttl is None or (live is not None and not refresh_ttl) else self.now + ttl + return self._store(key, value + (0.0 if live is None else live[0]), expires_at) + + def _increment_expiring_on_first_write(self, key: str, amount: float, ttl: float) -> float: + live = self._live(key) + expires_at = self.now + ttl if live is None else live[1] + return self._store(key, amount + (0.0 if live is None else live[0]), expires_at) + + def async_register_script(self, script: str): + async def run(keys: list[str], args: list[object]) -> float: + if self.scripts_fail: + raise ConnectionError("Redis dropped the script call") + if len(keys) == 1: + return self._increment_expiring_on_first_write(keys[0], float(str(args[0])), float(str(args[1]))) + marker = self._live(keys[1]) + polls = self._store(keys[1], 1.0 + (0.0 if marker is None else marker[0]), self.now + float(str(args[2]))) + amount = float(str(args[0] if polls == 1 else args[1])) + if amount > 0: + self._increment_expiring_on_first_write(keys[0], amount, float(str(args[2]))) + return amount + + return run + + +@pytest.mark.asyncio +async def test_a_finished_video_polled_without_pause_is_not_charged_again_once_the_session_window_rolls_over(): + from litellm.types.videos.main import VideoObject + + redis = _ClockedRedis() + handler = PROXY_MaxBudgetPerSessionHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis)) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) + video = VideoObject(id="job-1", object="video", status="completed") + chat = { + "call_type": "acompletion", + "response_cost": 0.01, + "litellm_params": {"metadata": {"session_id": "session-jobs", "agent_id": "agent-budget-123"}}, + } + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + for _ in range(3): + await handler.async_log_success_event(_video_poll(0.4), video, None, None) + redis.now += 0.6 * handler.ttl + await handler.async_log_success_event(chat, None, None, None) + + assert await handler._get_current_spend(handler._make_cache_key("session-jobs")) == pytest.approx(0.01) + + +@pytest.mark.asyncio +async def test_a_finished_video_whose_charge_missed_redis_is_charged_on_the_next_poll_once_redis_answers(): + from litellm.types.videos.main import VideoObject + + redis: Final = _ClockedRedis() + handler: Final = PROXY_MaxBudgetPerSessionHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis)) # pyright: ignore[reportArgumentType] # duck-typed Redis double + ) + video: Final = VideoObject(id="job-1", object="video", status="completed") + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + redis.scripts_fail = True + await handler.async_log_success_event(_video_poll(0.4), video, None, None) + redis.scripts_fail = False + for _ in range(2): + await handler.async_log_success_event(_video_poll(0.4), video, None, None) + + assert await handler._get_current_spend(handler._make_cache_key("session-jobs")) == pytest.approx(0.4) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), True]) +async def test_a_cost_that_is_not_a_finite_number_leaves_the_session_spend_alone(response_cost): + handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(DualCache())) + cache_key = handler._make_cache_key("session-unpriced") + await handler._increment_spend(cache_key, 0.25) + + with patch( + "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" + ) as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=5.0) + await handler.async_log_success_event( + { + "call_type": "acompletion", + "response_cost": response_cost, + "litellm_params": {"metadata": {"session_id": "session-unpriced", "agent_id": "agent-budget-123"}}, + }, + None, + None, + None, + ) + + assert await handler._get_current_spend(cache_key) == 0.25 + + @pytest.mark.asyncio async def test_no_agent_id_passes(): """ diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index f5a2ccf7b76..f1fd02981d6 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -27,6 +27,8 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( from litellm.router import Router from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import LiteLLMBatch, Usage +from litellm.types.videos.main import VideoObject +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG KEY_HASH: Final = "key-hash-batch" USER_ID: Final = "user-batch" @@ -47,6 +49,7 @@ def _batch(batch_id: str, status: str) -> LiteLLMBatch: input_file_id="file-batch", object="batch", status=status, + output_file_id="file-output", usage=Usage(prompt_tokens=20, completion_tokens=18, total_tokens=38), ) @@ -57,12 +60,14 @@ def _event( team_member_model_max_budget: Mapping[str, object] | None = None, user_id: str = USER_ID, team_id: str = TEAM_ID, + guardrail_information: tuple[Mapping[str, object], ...] = (), ) -> dict[str, object]: return { "call_type": call_type, "standard_logging_object": { "call_type": call_type, "response_cost": response_cost, + "guardrail_information": guardrail_information, "model": "openai/gpt-5.4-mini", "model_group": MODEL_GROUP, "metadata": { @@ -326,6 +331,57 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again( assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST) + +async def _poll_video( + limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, video_id: str, status: str, response_cost: float +) -> None: + await _log_success( + limiter, + {**_event("avideo_status", response_cost), POLLED_VIDEO_STATUS_KWARG: status}, + response_obj=VideoObject(id=video_id, object="video", status=status), + ) + + +@pytest.mark.asyncio +async def test_polls_of_a_finished_video_charge_each_per_model_budget_once(): + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + + await _poll_video(limiter, "video_first", "in_progress", response_cost=0) + for _ in range(3): + await _poll_video(limiter, "video_first", "completed", response_cost=BATCH_COST) + await _poll_video(limiter, "video_second", "completed", response_cost=BATCH_COST) + await _poll(limiter, _batch("video_first", "completed"), response_cost=BATCH_COST) + + assert await _spend(limiter, KEY_SPEND_KEY) == pytest.approx(3 * BATCH_COST) + assert await _spend(limiter, USER_SPEND_KEY) == pytest.approx(3 * BATCH_COST) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "polled", "guardrail_charges"), + ( + ("avideo_status", lambda status: VideoObject(id="job_first", object="video", status=status), 4), + ("aretrieve_batch", lambda status: _batch("job_first", status), 1), + ), +) +async def test_only_the_finished_poll_pays_the_job_and_only_a_video_repoll_pays_its_guardrail( + call_type, polled, guardrail_charges +): + """A running video poll pays its guardrail like any request, while a batch retrieve before its cost is final + charges nothing, as key spend defers it, so the retrieve that settles the batch pays the only guardrail.""" + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + guardrail_cost: Final = 0.01 + priced: Final = ({"guardrail_name": "priced", "guardrail_cost": guardrail_cost},) + + def poll(status: str, response_cost: float) -> dict[str, object]: + return {**_event(call_type, response_cost, guardrail_information=priced), POLLED_VIDEO_STATUS_KWARG: status} + + await _log_success(limiter, poll("in_progress", guardrail_cost), polled("in_progress")) + for _ in range(3): + await _log_success(limiter, poll("completed", BATCH_COST + guardrail_cost), polled("completed")) + + assert await _spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST + guardrail_charges * guardrail_cost) + OPUS_GROUP: Final = "claude-opus" OPUS_GROUP_MEMBERS: Final = MappingProxyType( {"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})} diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index bed185ba8f5..f9c234be55a 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3711,6 +3711,255 @@ async def test_failure_event_refunds_reservation_for_input_only_estimate_v3(): assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +@pytest.mark.parametrize( + ("response_obj", "used_tokens"), + [ + ([], 0), + (ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), 30), + ], +) +async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation_once_v3( + failure_first: bool, response_obj: object, used_tokens: int +) -> None: + api_key: Final = hash_token("sk-failure-and-success") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "dropped-call") + kwargs: Final = { + "litellm_call_id": "dropped-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, kwargs=dict(kwargs), response_obj=response_obj, start_time=now, end_time=now + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens + + +class _YieldingIncrementInMemoryCache(InMemoryCache): + async def async_increment(self, key: str, value: float, **kwargs: object) -> float: + await asyncio.sleep(0) + return await super().async_increment(key, value, **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("success_starts_first", [True, False]) +@pytest.mark.parametrize("failure_path", ["failure_event", "post_call_failure_hook"]) +async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_once_v3( + success_starts_first: bool, failure_path: str +) -> None: + api_key: Final = hash_token("sk-concurrent-failure-and-success") + model: Final = "gpt-4o-mini" + project: Final = "proj-concurrent" + local_cache: Final = DualCache(in_memory_cache=_YieldingIncrementInMemoryCache()) + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=10_000, + project_id=project, + project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}}, + ) + counter_keys: Final = ( + handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens"), + handler.create_rate_limit_keys( + key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens" + ), + handler.create_rate_limit_keys( + key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens" + ), + ) + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=local_cache, + data={ + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "concurrent-call", + }, + call_type="completion", + ) + kwargs: Final = { + "litellm_call_id": "concurrent-call", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = ( + partial(handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now) + if failure_path == "failure_event" + else partial( + handler.async_post_call_failure_hook, + request_data=dict(kwargs), + original_exception=Exception("stream ended"), + user_api_key_dict=caller, + ) + ) + log_success: Final = partial( + handler.async_log_success_event, + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), + start_time=now, + end_time=now, + ) + + await asyncio.gather( + *((log_success(), log_failure()) if success_starts_first else (log_failure(), log_success())) + ) + + assert tuple([int(await local_cache.async_get_cache(key=key) or 0) for key in counter_keys]) == (30, 20, 10) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("second_call_type", "second_model"), + [("call_mcp_tool", "mcp-tool-call"), ("completion", "gpt-4o-mini")], +) +async def test_a_pre_call_after_a_failure_leaves_the_success_charging_project_io_once_v3( + second_call_type: str, second_model: str +) -> None: + api_key: Final = hash_token("sk-retried-request-with-mcp-tool") + model: Final = "gpt-4o-mini" + project: Final = "proj-retried-mcp" + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=100_000, + project_id=project, + project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}}, + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "retried-call-with-mcp-tool", + } + kwargs: Final = { + "litellm_call_id": "retried-call-with-mcp-tool", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + itpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens" + ) + otpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens" + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, cache=local_cache, data=dict(request), call_type="completion" + ) + await handler.async_log_failure_event(kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now) + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=local_cache, + data={**request, "model": second_model}, + call_type=second_call_type, + ) + await handler.async_log_success_event( + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + assert ( + int(await local_cache.async_get_cache(key=itpm_key) or 0), + int(await local_cache.async_get_cache(key=otpm_key) or 0), + ) == (20, 7) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("steps", "key_tpm_limit", "expected_counters"), + [ + (("failure", "mcp-tool-call", "failure"), 100_000, (0, 0, 0, 0)), + (("failure", "tool-model-with-io-limit", "post_call_failure"), None, (0, 0, 0, 0)), + (("failure", "mcp-tool-call", "post_call_failure", "success"), 100_000, (20, 7, 0, 0)), + (("tool-model-over-io-limit", "success"), 100_000, (20, 7, 0, 0)), + ], +) +async def test_a_nested_mcp_pre_call_settles_each_project_io_hold_once_v3( + steps: tuple[str, ...], key_tpm_limit: Optional[int], expected_counters: tuple[int, int, int, int] +) -> None: + api_key: Final = hash_token("sk-nested-mcp-hold-flags") + model: Final = "gpt-4o-mini" + tool_model: Final = "tool-model-with-io-limit" + project: Final = "proj-nested-mcp-hold-flags" + io_limits: Final = {model: 10_000, tool_model: 10_000, "tool-model-over-io-limit": 1} + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=key_tpm_limit, + project_id=project, + project_metadata={"model_itpm_limit": io_limits, "model_otpm_limit": io_limits}, + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "nested-mcp-hold-flags", + } + kwargs: Final = { + "litellm_call_id": "nested-mcp-hold-flags", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + counter_keys: Final = tuple( + handler.create_rate_limit_keys( + key=f"model_per_project_{kind}", value=f"{project}:{scope}", rate_limit_type="tokens" + ) + for scope, kind in ((model, "itpm"), (model, "otpm"), (tool_model, "itpm"), (tool_model, "otpm")) + ) + mcp_pre_call: Final = partial( + handler.async_pre_call_hook, user_api_key_dict=caller, cache=local_cache, call_type="call_mcp_tool" + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, cache=local_cache, data=dict(request), call_type="completion" + ) + for step in steps: + match step: + case "failure": + await handler.async_log_failure_event( + kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + case "post_call_failure": + await handler.async_post_call_failure_hook( + request_data=dict(request), + original_exception=HTTPException(status_code=429, detail="upstream rate limited"), + user_api_key_dict=caller, + ) + case "success": + await handler.async_log_success_event( + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + case "tool-model-over-io-limit": + with pytest.raises(ProxyRateLimitError): + await mcp_pre_call(data={**request, "model": step}) + case _: + await mcp_pre_call(data={**request, "model": step}) + + assert tuple([int(await local_cache.async_get_cache(key=key) or 0) for key in counter_keys]) == expected_counters + + @pytest.mark.asyncio async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3(): """ @@ -4198,6 +4447,125 @@ async def test_failure_event_settles_project_itpm_otpm_at_recovered_partial_usag assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +async def test_a_request_logging_a_failure_and_a_success_settles_its_project_itpm_otpm_once_v3( + failure_first: bool, +) -> None: + api_key: Final = hash_token("sk-failure-and-success-project-io") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + api_key=api_key, + project_id="proj-dropped", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ), + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "dropped-project-call", + }, + call_type="completion", + ) + itpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_itpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + otpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_otpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + kwargs: Final = { + "litellm_call_id": "dropped-project-call", + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=itpm_key) or 0) == 20 + assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + + +@pytest.mark.asyncio +async def test_a_reservation_made_after_a_released_one_in_the_same_context_settles_at_its_usage_v3() -> None: + api_key: Final = hash_token("sk-second-reservation") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + user_api_key_dict: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=10_000, + project_id="proj-second", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ) + now: Final = datetime.now() + + async def reserve(call_id: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": call_id, + }, + call_type="completion", + ) + + def logged(call_id: str) -> dict[str, object]: + return { + "litellm_call_id": call_id, + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + + await reserve("first-call") + await handler.async_log_failure_event(kwargs=logged("first-call"), response_obj=None, start_time=now, end_time=now) + await reserve("second-call") + await handler.async_log_success_event( + kwargs=logged("second-call"), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + counters: Final = { + "tokens": ("api_key", api_key), + "itpm": ("model_per_project_itpm", "proj-second:gpt-4o-mini"), + "otpm": ("model_per_project_otpm", "proj-second:gpt-4o-mini"), + } + settled: Final = { + name: int( + await local_cache.async_get_cache( + key=handler.create_rate_limit_keys(key=key, value=value, rate_limit_type="tokens") + ) + or 0 + ) + for name, (key, value) in counters.items() + } + assert settled == {"tokens": 27, "itpm": 20, "otpm": 7} + # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- diff --git a/tests/unit/proxy/hooks/test_proxy_hooks_init.py b/tests/unit/proxy/hooks/test_proxy_hooks_init.py index a6edd3db944..417ea5a9650 100644 --- a/tests/unit/proxy/hooks/test_proxy_hooks_init.py +++ b/tests/unit/proxy/hooks/test_proxy_hooks_init.py @@ -33,7 +33,7 @@ def test_managed_vector_stores_hook_registered(): assert hook_cls.__name__ == "_PROXY_LiteLLMManagedVectorStores" -def test_isolation_module_does_not_pull_in_proxy_utils(): +def test_isolation_module_does_not_pull_in_proxy_utils(monkeypatch): """Layering guard: litellm.llms.* must not transitively import litellm.proxy.utils, which would reintroduce the import cycle.""" import importlib @@ -44,7 +44,7 @@ def test_isolation_module_does_not_pull_in_proxy_utils(): "litellm.proxy.management_endpoints.common_utils", "litellm.llms.base_llm.managed_resources.isolation", ]: - sys.modules.pop(mod, None) + monkeypatch.delitem(sys.modules, mod, raising=False) importlib.import_module("litellm.llms.base_llm.managed_resources.isolation") assert "litellm.proxy.utils" not in sys.modules diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index a4e78a48a1a..c6366a40504 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -4,7 +4,7 @@ import logging from collections.abc import Mapping from datetime import datetime, timedelta, timezone from pathlib import Path -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, create_autospec, patch @@ -24,6 +24,7 @@ from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( ProxyDBLogger, + _charge_request_cost, _get_budget_reservation_from_metadata, _should_track_cost_callback, _update_database_and_spend_counters, @@ -42,6 +43,7 @@ from litellm.proxy.utils import ( ) from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage +from litellm.types.videos.main import VideoObject @pytest.mark.asyncio @@ -1595,14 +1597,16 @@ def _retrieved_batch(status: str, output_file_id: str | None): @pytest.mark.asyncio @pytest.mark.parametrize( - ("call_type", "status", "output_file_id", "row_claimed", "spend_written", "charged"), + ("call_type", "status", "output_file_id", "row_claimed", "spend_written", "charged", "redacted"), [ - ("aretrieve_batch", "in_progress", None, True, False, False), - ("aretrieve_batch", "completed", None, True, False, False), - ("aretrieve_batch", "completed", "file-out", False, True, False), - ("aretrieve_batch", "completed", "file-out", True, True, True), - ("aretrieve_batch", "failed", None, True, True, True), - ("acreate_batch", "validating", None, True, True, True), + ("aretrieve_batch", "in_progress", None, True, False, False, False), + ("aretrieve_batch", "completed", None, True, False, False, False), + ("aretrieve_batch", "completed", "file-out", False, True, False, False), + ("aretrieve_batch", "completed", "file-out", True, True, True, False), + ("aretrieve_batch", "failed", None, True, True, True, False), + ("acreate_batch", "validating", None, True, True, True, False), + ("aretrieve_batch", "in_progress", None, True, False, False, True), + ("aretrieve_batch", "completed", "file-out", True, True, True, True), ], ids=[ "retrieve_before_final", @@ -1611,20 +1615,31 @@ def _retrieved_batch(status: str, output_file_id: str | None): "retrieve_first_final", "retrieve_failed_batch", "create_before_final", + "redacted_retrieve_before_final", + "redacted_retrieve_first_final", ], ) async def test_track_cost_callback_charges_a_batch_once_and_only_when_final( # test-quality-ok: whether the spend writer runs, whether the counters move, and whether the poll's reservation is handed back is the whole observable contract of the gate - call_type, status, output_file_id, row_claimed, spend_written, charged + call_type, status, output_file_id, row_claimed, spend_written, charged, redacted ): """ A poll before the batch is final used to pin its shared spend row at $0, and every completed retrieve after the first charged the key again (LIT-7048). Only retrieves are gated, since creating a batch is its own billable request, and a retrieve that - charges nothing hands its budget reservation back instead. + charges nothing hands its budget reservation back instead. Message redaction replaces + the logged batch, so the gate reads the batch logging stamped before redacting it. """ + from litellm.litellm_core_utils.batch_utils import POLLED_BATCH_COST_IS_FINAL_KWARG, POLLED_BATCH_ID_KWARG + logger = ProxyDBLogger() budget_reservation = None if charged else {"reserved_cost": 0.5, "entries": []} - kwargs = _batch_retrieve_kwargs(call_type, reservation=budget_reservation) + retrieved: Final = _retrieved_batch(status, output_file_id) + stamped: Final = ( + {POLLED_BATCH_ID_KWARG: retrieved.id, POLLED_BATCH_COST_IS_FINAL_KWARG: status != "in_progress"} + if redacted + else {} + ) + kwargs = {**_batch_retrieve_kwargs(call_type, reservation=budget_reservation), **stamped} with ( patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam @@ -1646,7 +1661,7 @@ async def test_track_cost_callback_charges_a_batch_once_and_only_when_final( # await logger._PROXY_track_cost_callback( kwargs=kwargs, - completion_response=_retrieved_batch(status, output_file_id), + completion_response={"text": "redacted-by-litellm"} if redacted else retrieved, start_time=datetime.now(), end_time=datetime.now(), ) @@ -3376,18 +3391,8 @@ async def test_async_log_success_event_hands_the_sidecar_a_compact_event_and_ski assert event.response_cost == 0.0125 -@pytest.mark.asyncio -async def test_async_log_success_event_keeps_batch_retrieves_in_process(): - producer = SpendEventProducer( - address=UnixAddress(path="/nonexistent/spend.sock"), - on_unavailable="drop", - buffer_size=10, - connect_timeout=1.0, - fallback=_no_fallback, - ) - logger = ProxyDBLogger(producer) - kwargs = {**_offload_kwargs(), "call_type": CallTypes.aretrieve_batch.value} - completed_batch = LiteLLMBatch( +def _completed_batch() -> LiteLLMBatch: + return LiteLLMBatch( id="batch_abc", completion_window="24h", created_at=1, @@ -3398,6 +3403,29 @@ async def test_async_log_success_event_keeps_batch_retrieves_in_process(): status="completed", ) + +def _completed_video() -> VideoObject: + return VideoObject(id="video_abc", object="video", status="completed") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "polled_job"), + [(CallTypes.aretrieve_batch.value, _completed_batch), ("avideo_status", _completed_video)], +) +async def test_async_log_success_event_keeps_job_polls_in_process(call_type, polled_job): + """The spend event leaves out the polled video's status and id, which decide whether the poll charges the job.""" + producer = SpendEventProducer( + address=UnixAddress(path="/nonexistent/spend.sock"), + on_unavailable="drop", + buffer_size=10, + connect_timeout=1.0, + fallback=_no_fallback, + ) + logger = ProxyDBLogger(producer) + kwargs = {**_offload_kwargs(), "call_type": call_type} + completed_job = polled_job() + with ( patch( # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam "litellm.proxy.proxy_server.proxy_logging_obj" @@ -3411,7 +3439,7 @@ async def test_async_log_success_event_keeps_batch_retrieves_in_process(): ): mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True) mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() - await logger.async_log_success_event(kwargs, completed_batch, datetime.now(), datetime.now()) + await logger.async_log_success_event(kwargs, completed_job, datetime.now(), datetime.now()) mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once() assert producer.stats().queued == 0 @@ -3491,6 +3519,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() +def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]: + row: Final = get_logging_payload( + kwargs=cast(dict, update_database_kwargs["kwargs"]), + response_obj=update_database_kwargs["completion_response"], + start_time=cast(datetime, update_database_kwargs["start_time"]), + end_time=cast(datetime, update_database_kwargs["end_time"]), + ) + return json.loads(row["metadata"])["error_information"] + + +@pytest.mark.asyncio +async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100") + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + provider_error: Final = "upstream failure " + "x" * 1000 + + await logger.async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"] + assert isinstance(stored_message, str) + assert stored_message.count("skipped") == 1 + kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1]) + skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0]) + assert kept_chars + skipped_chars > len(provider_error) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -3504,9 +3566,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), ) - error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ - "error_information" - ] + error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs) assert "medical records" not in json.dumps(error_information) assert ( error_information["error_message"] @@ -3951,138 +4011,48 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( assert len(writer.rows) == rows_written -@pytest.mark.asyncio -async def test_failure_capture_overload_does_not_skip_budget_release( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier, _write +class _RecordedCharges: + def __init__(self, outcomes: tuple[bool, ...]) -> None: + self.outcomes = outcomes + self.calls: tuple[tuple[object, float, dict[str, object] | None], ...] = () - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "prisma_client", None) - meter: Final = runtime.meter - assert meter is not None - receipt: Final = meter.submit((_write("occupied-one"), _write("occupied-two"))) - auth: Final = UserAPIKeyAuth(budget_reservation={"reserved_cost": 0.5, "entries": [], "finalized": False}) - request: Final = { - "call_type": "acompletion", - "litellm_call_id": "failed", - "model": "capture-model", - "start_time": _CALL_START, - "litellm_params": {}, + async def __call__( + self, *, kwargs: dict, response_cost: float, budget_reservation: dict[str, object] | None + ) -> bool: + self.calls = (*self.calls, (kwargs.get("response_cost"), response_cost, budget_reservation)) + return self.outcomes[len(self.calls) - 1] + + +_GUARDED_POLL: Final = MappingProxyType( + { + "call_type": "avideo_status", + "response_cost": 0.43, + "standard_logging_object": {"guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}]}, } - try: - await database.entered.wait() - await ProxyDBLogger().async_post_call_failure_hook(request, RuntimeError("provider failed"), auth) - assert auth.budget_reservation is not None and auth.budget_reservation["finalized"] is True - assert "Usage capture failed on the request failure path" in caplog.text - assert "_accounting_capture" not in request - finally: - database.release.set() - await runtime.stop(1) - await receipt - assert database.committed == ("occupied-one", "occupied-two") +) @pytest.mark.asyncio -@pytest.mark.parametrize("failure", [False, True]) -@pytest.mark.parametrize("complete", [False, True]) -@pytest.mark.parametrize("deferred", [False, True]) -async def test_capture_and_legacy_share_resolved_key_attribution( - failure: bool, complete: bool, deferred: bool, monkeypatch: pytest.MonkeyPatch -) -> None: - from pydantic import TypeAdapter +@pytest.mark.parametrize( + ("outcomes", "poll", "charged", "second_call"), + [ + ((True,), _GUARDED_POLL, 0.43, None), + ((False, True), _GUARDED_POLL, 0.01, (0.01, 0.01, None)), + ((False, False), _GUARDED_POLL, None, (0.01, 0.01, None)), + ((False,), {"response_cost": 0.43}, None, None), + ((False,), {**_GUARDED_POLL, "call_type": "aretrieve_batch"}, None, None), + ], + ids=[ + "first_poll_pays_the_job", + "repeat_poll_pays_its_guardrail", + "guardrail_charge_not_written", + "repeat_poll_free", + "repeat_batch_poll_free_whatever_its_guardrails_cost", + ], +) +async def test_a_poll_of_an_already_charged_job_still_charges_its_own_guardrails(outcomes, poll, charged, second_call): + charge_spend: Final = _RecordedCharges(outcomes) + reservation: Final = {"reserved": 0.43} - from litellm.accounting import DeliveryTarget, DimensionName - from litellm.proxy import proxy_server - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache - from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage - - key: Final = UserAPIKeyAuth( - api_key="a" * 64, - user_id="owner", - team_id="team", - org_id="org", - project_id="project", - key_alias="key", - team_alias="team-alias", - request_route="/v1/chat/completions", - ) - cache: Final = UserApiKeyCache() - await cache.async_set_cache(key="a" * 64, value=key) - monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) - monkeypatch.setattr(proxy_server, "prisma_client", object()) - writer: Final = _RecordingSpendWriter() - monkeypatch.setattr(proxy_server, "proxy_logging_obj", SimpleNamespace(db_spend_update_writer=writer)) - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - rejected: Final = failure and deferred and not complete - owner: Final = "original-owner" if deferred and not rejected else "owner" - metadata: Final = { - "user_api_key": "b" * 64 if rejected else "a" * 64, - **( - { - "user_api_key_user_id": "forged-owner" if rejected else owner, - "user_api_key_team_id": "forged-team" if rejected else "team", - "user_api_key_org_id": "forged-org" if rejected else "org", - "user_api_key_project_id": None if deferred and not complete else "project", - "user_api_key_alias": "key", - "user_api_key_team_alias": "team-alias", - } - if complete or deferred - else {} - ), - } - kwargs: Final = { - "call_type": "aretrieve_batch" if deferred else "call_mcp_tool", - "litellm_call_id": "attributed", - "model": "MCP: echo", - "response_cost": 0.25, - "start_time": _CALL_START, - "litellm_params": {"metadata": metadata}, - } - if rejected: - from litellm.proxy.utils import REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY - - kwargs[REJECTED_BEFORE_REQUEST_SETUP_KEY] = REJECTED_BEFORE_REQUEST_SETUP - logger: Final = ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer)) - try: - with track_batch_accounting("attributed" if deferred and not rejected else "other"): - if failure: - await logger.async_post_call_failure_hook( - kwargs, - HTTPException(401, "model not allowed"), - key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"), - ) - else: - await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START) - finally: - await runtime.stop(1) - assert len(writer.rows) == 1 - assert len(database.rows) == 2 - dimensions: Final = {item.name: item.value for item in database.rows[-1].event.dimensions} - if rejected: - assert dimensions.get(DimensionName.KEY) == key.api_key - assert { - name: dimensions.get(name) - for name in (DimensionName.USER, DimensionName.TEAM, DimensionName.ORGANIZATION, DimensionName.PROJECT) - } == { - DimensionName.USER: owner, - DimensionName.TEAM: "team", - DimensionName.ORGANIZATION: "org", - DimensionName.PROJECT: None if deferred and not complete and not rejected else "project", - } - assert set(database.rows[-1].targets) >= {DeliveryTarget.DAILY_TEAM_SPEND, DeliveryTarget.DAILY_ORGANIZATION_SPEND} - legacy: Final = TypeAdapter(dict[str, object]).validate_python(writer.rows[0]["kwargs"]) - payload: Final = get_logging_payload(legacy, {"id": "tool"}, _CALL_START, _CALL_START) - assert (payload["user"], payload["team_id"], payload["organization_id"]) == (owner, "team", "org") + assert await _charge_request_cost(charge_spend, dict(poll), 0.43, reservation) == charged + assert charge_spend.calls == ((0.43, 0.43, reservation), *(() if second_call is None else (second_call,))) diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index e43a72541c7..0d517e87ef6 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -20,6 +20,7 @@ import copy import io import json from datetime import datetime, timedelta +from types import MappingProxyType from typing import Any, Dict, Final import pytest @@ -27,7 +28,7 @@ import soundfile as sf from fastapi import HTTPException from PIL import Image -from litellm import token_counter +from litellm import get_model_info, token_counter from litellm.caching.caching import DualCache from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, estimate_image_tokens @@ -3060,7 +3061,7 @@ def test_retry_success_charges_released_project_io_reservations( stash.itpm_reserved_scopes = frozenset({itpm_scope}) stash.otpm_reserved_tokens = 60 stash.otpm_reserved_scopes = frozenset({otpm_scope}) - stash.reservation_released = True + stash.mark_reservation_released() operations = handler._build_io_token_reservation_ops( {}, @@ -3971,6 +3972,334 @@ async def test_media_pre_call_rejects_tight_quota_and_reserves_sufficient_quota( assert expected[0] > 1000 and expected[1] >= expected[0] +_EMBEDDING_MODEL: Final = "gemini-embedding-2" +_EMBEDDING_CAPTION: Final = "a red bicycle leaning on a wall" + + +def _embedding_media_uri(kind: str, compressed: bool) -> str: + mime: Final = "image/png" if kind == "image" else "audio/wav" + return f"data:{mime};base64,{_quota_media_payload(kind, compressed)}" + + +@pytest.mark.parametrize("kind", ["image", "audio"]) +@pytest.mark.parametrize( + "shape", + ["string", "list", "file_block", "with_caption", "combined"], +) +def test_embedding_media_input_is_estimated_independently_of_encoding( + rate_limiter: tuple[RateLimitHandler, DualCache], kind: str, shape: str +) -> None: + handler, _ = rate_limiter + + def embedding_request(uri: str) -> dict[str, object]: + shapes: Final[dict[str, object]] = { + "string": uri, + "list": [uri], + "file_block": [{"type": "file", "file": {"file_data": uri}}], + "with_caption": [_EMBEDDING_CAPTION, uri], + "combined": [[_EMBEDDING_CAPTION, uri]], + } + return {"model": _EMBEDDING_MODEL, "input": shapes[shape]} + + uncompressed: Final = embedding_request(_embedding_media_uri(kind, False)) + original: Final = copy.deepcopy(uncompressed) + estimate: Final = handler._estimate_input_and_output_tokens(uncompressed, call_type="aembedding") + assert estimate == handler._estimate_input_and_output_tokens( + embedding_request(_embedding_media_uri(kind, True)), call_type="aembedding" + ) + assert 0 < estimate[0] < 2_000 and estimate[1] == 0 + assert uncompressed == original + + +@pytest.mark.parametrize("kind", ["image", "audio"]) +def test_embedding_media_input_costs_the_same_as_chat_media( + rate_limiter: tuple[RateLimitHandler, DualCache], kind: str +) -> None: + handler, _ = rate_limiter + uri: Final = _embedding_media_uri(kind, False) + chat_block: Final = ( + {"type": "image_url", "image_url": {"url": uri}} + if kind == "image" + else {"type": "input_audio", "input_audio": {"data": _quota_media_payload(kind, False), "format": "wav"}} + ) + chat_input_tokens, _ = handler._estimate_input_and_output_tokens( + {"model": _EMBEDDING_MODEL, "messages": [{"role": "user", "content": [chat_block]}]}, + call_type="acompletion", + ) + caption_tokens, _ = handler._estimate_input_and_output_tokens( + {"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION]}, call_type="aembedding" + ) + + assert handler._estimate_input_and_output_tokens( + {"model": _EMBEDDING_MODEL, "input": uri}, call_type="aembedding" + ) == (chat_input_tokens, 0) + assert handler._estimate_input_and_output_tokens( + {"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, uri]}, call_type="aembedding" + ) == (caption_tokens + chat_input_tokens, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"]) +async def test_project_token_limit_admits_image_embedding_and_still_rejects_long_text( + rate_limiter: tuple[RateLimitHandler, DualCache], limit_field: str +) -> None: + handler, cache = rate_limiter + auth: Final = UserAPIKeyAuth( + api_key=hash_token(f"sk-embedding-image-{limit_field}"), + project_id=f"proj-embedding-image-{limit_field}", + project_metadata={limit_field: {_EMBEDDING_MODEL: 75_000}}, + ) + image_uri: Final = _embedding_media_uri("image", False) + long_text: Final = "word " * 80_000 + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": _EMBEDDING_MODEL, "input": image_uri}, + call_type="aembedding", + ) + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": _EMBEDDING_MODEL, "input": long_text}, + call_type="aembedding", + ) + assert error.value.status_code == 429 + assert len(image_uri) > len(long_text) + + +_EMBEDDING_FILE_MIMES: Final = MappingProxyType({"pdf": "application/pdf", "video": "video/mp4"}) + + +def _embedding_file_uri(kind: str) -> str: + return f"data:{_EMBEDDING_FILE_MIMES[kind]};base64,{base64.b64encode(bytes(range(256)) * 1200).decode()}" + + +def _embedding_uri(kind: str) -> str: + return _embedding_file_uri(kind) if kind in _EMBEDDING_FILE_MIMES else _embedding_media_uri(kind, False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"]) +@pytest.mark.parametrize("kind", ["pdf", "video"]) +@pytest.mark.parametrize("shape", ["data_uri", "file_block"]) +async def test_project_token_limit_admits_document_embedding_up_to_the_model_input_limit( + rate_limiter: tuple[RateLimitHandler, DualCache], limit_field: str, kind: str, shape: str +) -> None: + handler, cache = rate_limiter + uri: Final = _embedding_file_uri(kind) + model_input_limit: Final = get_model_info(_EMBEDDING_MODEL)["max_input_tokens"] + assert isinstance(model_input_limit, int) and len(uri) // 4 > 75_000 > model_input_limit + + def project_auth(limit: int) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key=hash_token(f"sk-embedding-{kind}-{shape}-{limit_field}-{limit}"), + project_id=f"proj-embedding-{kind}-{shape}-{limit_field}-{limit}", + project_metadata={limit_field: {_EMBEDDING_MODEL: limit}}, + ) + + item: Final = uri if shape == "data_uri" else {"type": "file", "file": {"file_data": uri}} + request: Final = {"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, item]} + await handler.async_pre_call_hook( + user_api_key_dict=project_auth(75_000), cache=cache, data=dict(request), call_type="aembedding" + ) + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=project_auth(model_input_limit // 2), + cache=cache, + data=dict(request), + call_type="aembedding", + ) + assert error.value.status_code == 429 + + +_TEXT_ONLY_EMBEDDING_MODEL: Final = "text-embedding-3-small" + + +@pytest.mark.parametrize("kind", ["image", "audio", "pdf", "video"]) +def test_text_only_embedding_model_counts_data_uri_as_text( + rate_limiter: tuple[RateLimitHandler, DualCache], kind: str +) -> None: + handler, _ = rate_limiter + uri: Final = _embedding_uri(kind) + + assert handler._estimate_input_and_output_tokens( + {"model": _TEXT_ONLY_EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, uri]}, call_type="aembedding" + ) == handler._estimate_input_and_output_tokens( + {"model": _TEXT_ONLY_EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, "x" * len(uri)]}, + call_type="aembedding", + ) + + +def _project_limited_embedding_call( + limit_field: str, backends: tuple[str, ...], kind: str = "image" +) -> tuple[RateLimitHandler, DualCache, UserAPIKeyAuth, dict[str, object]]: + cache: Final = DualCache() + handler: Final = RateLimitHandler( + internal_usage_cache=InternalUsageCache(cache), media_model_resolver=lambda _model, _team_id: backends + ) + scope: Final = f"{limit_field}-{kind}-{'-'.join(backends)}" + auth: Final = UserAPIKeyAuth( + api_key=hash_token(f"sk-embedding-backends-{scope}"), + project_id=f"proj-embedding-backends-{scope}", + project_metadata={limit_field: {"team-embedder": 75_000}}, + ) + return handler, cache, auth, {"model": "team-embedder", "input": _embedding_uri(kind)} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"]) +@pytest.mark.parametrize( + ("backends", "kind"), + [ + ((_TEXT_ONLY_EMBEDDING_MODEL,), "image"), + ((_EMBEDDING_MODEL, _TEXT_ONLY_EMBEDDING_MODEL), "image"), + (("voyage/voyage-multimodal-3.5",), "image"), + (("oci/cohere.embed-v4.0",), "image"), + (("bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",), "image"), + (("bedrock/twelvelabs.marengo-embed-3-0-v1:0",), "image"), + (("bedrock/amazon.nova-2-multimodal-embeddings-v1:0",), "pdf"), + ], + ids=["text", "mixed", "voyage", "oci", "marengo_2_7", "marengo_3", "nova_pdf"], +) +async def test_project_token_limit_rejects_embedding_media_a_backend_reads_as_text( + limit_field: str, backends: tuple[str, ...], kind: str +) -> None: + handler, cache, auth, request = _project_limited_embedding_call(limit_field, backends, kind) + + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data=request, call_type="aembedding") + assert error.value.status_code == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"]) +@pytest.mark.parametrize( + "backend", + [ + _EMBEDDING_MODEL, + "bedrock/amazon.titan-embed-image-v1", + "bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", + ], + ids=["gemini", "titan", "nova_async_invoke"], +) +async def test_project_token_limit_admits_image_embedding_through_an_alias_of_an_image_backend( + limit_field: str, backend: str +) -> None: + handler, cache, auth, request = _project_limited_embedding_call(limit_field, (backend,)) + + await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data=request, call_type="aembedding") + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={**request, "input": "word " * 100_000}, + call_type="aembedding", + ) + assert error.value.status_code == 429 + + +def _malformed_embedding_uri(shape: str) -> str: + payload: Final = _quota_media_payload("image", False) + whole_quanta: Final = payload.rstrip("=")[: len(payload.rstrip("=")) // 4 * 4] + words: Final = "word " * 100_000 + return { + "no_comma": f"data:audio/wav;base64{words}", + "text_payload": f"data:image/png;base64,{words}", + "no_base64_marker": f"data:image/png,{payload}", + "header_past_256_chars": f"data:image/png;{'x' * 300};base64,{payload}", + "noncanonical_padding": f"data:image/png;base64,{whole_quanta}AB==", + "padding_mid_payload": f"data:image/png;base64,{whole_quanta}AA==AAAA", + "length_not_multiple_of_4": f"data:image/png;base64,{whole_quanta}A", + }[shape] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "shape", + [ + "no_comma", + "text_payload", + "no_base64_marker", + "header_past_256_chars", + "noncanonical_padding", + "padding_mid_payload", + "length_not_multiple_of_4", + ], +) +async def test_project_token_limit_counts_a_malformed_data_uri_as_text( + rate_limiter: tuple[RateLimitHandler, DualCache], shape: str +) -> None: + handler, cache = rate_limiter + auth: Final = UserAPIKeyAuth( + api_key=hash_token(f"sk-embedding-malformed-{shape}"), + project_id=f"proj-embedding-malformed-{shape}", + project_metadata={"model_tpm_limit": {_EMBEDDING_MODEL: 75_000}}, + ) + + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": _EMBEDDING_MODEL, "input": _malformed_embedding_uri(shape)}, + call_type="aembedding", + ) + assert error.value.status_code == 429 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("backend", ["cohere/embed-v4.0", "cohere.embed-v4:0"], ids=["cohere", "bedrock_cohere"]) +async def test_project_token_limit_counts_a_mixed_batch_as_text_when_the_backend_types_the_whole_batch( + backend: str, +) -> None: + handler, cache, auth, request = _project_limited_embedding_call("model_tpm_limit", (backend,)) + image_uri: Final = request["input"] + + await handler.async_pre_call_hook( + user_api_key_dict=auth, cache=cache, data={**request, "input": [image_uri, image_uri]}, call_type="aembedding" + ) + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={**request, "input": [image_uri, _EMBEDDING_CAPTION]}, + call_type="aembedding", + ) + assert error.value.status_code == 429 + + +@pytest.mark.asyncio +async def test_project_itpm_limit_reserves_media_tokens_for_an_image_embedding( + rate_limiter: tuple[RateLimitHandler, DualCache], +) -> None: + handler, cache = rate_limiter + image_uri: Final = _embedding_media_uri("image", False) + image_tokens, _ = handler._estimate_input_and_output_tokens( + {"model": _EMBEDDING_MODEL, "input": image_uri}, call_type="aembedding" + ) + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-embedding-image-itpm-media"), + project_id="proj-embedding-image-itpm-media", + project_metadata={"model_itpm_limit": {_EMBEDDING_MODEL: image_tokens // 2}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": _EMBEDDING_MODEL, "input": _EMBEDDING_CAPTION}, + call_type="aembedding", + ) + with pytest.raises(HTTPException) as error: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": _EMBEDDING_MODEL, "input": image_uri}, + call_type="aembedding", + ) + assert error.value.status_code == 429 + + @pytest.mark.asyncio async def test_ptu_pre_call_weights_media_input_and_output() -> None: cache: Final = DualCache() diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py index c1505d3c8ec..59e12a97899 100644 --- a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py @@ -75,25 +75,25 @@ def _router() -> Router: return Router(model_list=ROUTER_MODEL_LIST) -class RecordingRouter(Router): +def _recording_router(classified_tier: str) -> tuple[Router, list[dict]]: """A real router that records the classifier calls the endpoint makes instead of sending them. Injected at the same `proxy_server.llm_router` boundary the endpoint reads, so model resolution and the key's model-access checks still run against a genuine Router. """ + recorded_calls: Final[list[dict]] = [] - def __init__(self, classified_tier: str) -> None: - super().__init__(model_list=ROUTER_MODEL_LIST) - self.classified_tier = classified_tier - self.recorded_calls: list[dict] = [] - - async def acompletion(self, model, messages, stream=False, **kwargs): - self.recorded_calls.append({"model": model, "messages": messages, **kwargs}) + async def record(model, messages, stream=False, **kwargs): + recorded_calls.append({"model": model, "messages": messages, **kwargs}) return ModelResponse( - choices=[Choices(message=Message(content=f'{{"tier": "{self.classified_tier}"}}'))], + choices=[Choices(message=Message(content=f'{{"tier": "{classified_tier}"}}'))], model=model, ) + router: Final = Router(model_list=ROUTER_MODEL_LIST) + router.acompletion = AsyncMock(side_effect=record) + return router, recorded_calls + def _request_from(body: Mapping[str, object], **config_overrides: object) -> AutoRouterRoutingTestRequest: return AutoRouterRoutingTestRequest.model_validate( @@ -137,7 +137,7 @@ async def _classifier_user_payload(body: Mapping[str, object], monkeypatch: pyte """The variable half of the classifier call this body produces.""" from litellm.proxy import proxy_server - router = RecordingRouter("SIMPLE") + router, recorded_calls = _recording_router("SIMPLE") monkeypatch.setattr(proxy_server, "llm_router", router) await preview_auto_router_routing( @@ -145,7 +145,7 @@ async def _classifier_user_payload(body: Mapping[str, object], monkeypatch: pyte data=_request_from(body, classifier_type="llm", classifier_llm_config={"model": "classifier-model"}), user_api_key_dict=ADMIN, ) - return router.recorded_calls[0]["messages"][1]["content"] + return recorded_calls[0]["messages"][1]["content"] @pytest.mark.parametrize( @@ -2710,7 +2710,7 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke async def test_jev_test_routing_authorizes_paid_evaluation_before_contacting_typesafe( monkeypatch: pytest.MonkeyPatch, denial: str | None ) -> None: - router: Final = RecordingRouter("SIMPLE") + router, recorded_calls = _recording_router("SIMPLE") monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setenv("TYPESAFE_API_KEY", "test") monkeypatch.setenv("TYPESAFE_API_BASE", "https://typesafe.test") @@ -2765,7 +2765,7 @@ async def test_jev_test_routing_authorizes_paid_evaluation_before_contacting_typ assert response.routing_decision["cause"] == "jev_classifier" assert response.routed_model == "cheap-model" assert evaluation.call_count == 1 - assert router.recorded_calls == [] + assert recorded_calls == [] await handler.client.aclose() @@ -2775,7 +2775,7 @@ async def test_jev_test_routing_authorizes_paid_evaluation_before_contacting_typ ["allowed", "credential-free", "team-admin-environment", "missing", "blocked", "key", "budget", "team", "not-router"], ) async def test_saved_jev_probe_uses_authorized_server_configuration(monkeypatch: pytest.MonkeyPatch, case: str) -> None: - router: Final = RecordingRouter("SIMPLE") + router, recorded_calls = _recording_router("SIMPLE") stored_key: Final = "synthetic-server-jev-key" monkeypatch.setenv("SAVED_JEV_KEY", stored_key) monkeypatch.setenv("SAVED_JEV_BASE", "https://saved-jev.test") @@ -2876,7 +2876,7 @@ async def test_saved_jev_probe_uses_authorized_server_configuration(monkeypatch: assert evaluation.calls.last.request.headers["authorization"] == f"Bearer {stored_key}" assert stored_key not in result.model_dump_json() assert evaluation.call_count == (1 if case in ("allowed", "credential-free", "team-admin-environment") else 0) - assert router.recorded_calls == [] + assert recorded_calls == [] await handler.client.aclose() @@ -3513,7 +3513,7 @@ async def test_member_billable_preview_checks_and_charges_destination_team( auth_module: Final = importlib.import_module("litellm.proxy.auth.user_api_key_auth") actor: Final = _configure_member_preview(monkeypatch).model_copy(update={"metadata": {"tags": ["key-tag"]}}) - router: Final = RecordingRouter("SIMPLE") + router, recorded_calls = _recording_router("SIMPLE") monkeypatch.setattr(proxy_server, "llm_router", router) async def check_and_tag( @@ -3547,13 +3547,13 @@ async def test_member_billable_preview_checks_and_charges_destination_team( if over_budget: with pytest.raises(litellm.BudgetExceededError): await preview_auto_router_routing(data, actor, http_request) - assert router.recorded_calls == [] + assert recorded_calls == [] else: await preview_auto_router_routing(data, actor, http_request) - assert len(router.recorded_calls) == 1 - assert router.recorded_calls[0]["metadata"]["user_api_key_team_id"] == "member-preview-team" - assert router.recorded_calls[0]["metadata"]["user_api_key_user_id"] == "preview-member" - assert set(router.recorded_calls[0]["metadata"]["tags"]) == {"key-tag", "header-tag"} + assert len(recorded_calls) == 1 + assert recorded_calls[0]["metadata"]["user_api_key_team_id"] == "member-preview-team" + assert recorded_calls[0]["metadata"]["user_api_key_user_id"] == "preview-member" + assert set(recorded_calls[0]["metadata"]["tags"]) == {"key-tag", "header-tag"} checks.assert_awaited_once() assert checks.await_args.kwargs["user_api_key_auth_obj"].team_id == "member-preview-team" assert checks.await_args.kwargs["route"] == "/auto_router/test_routing" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 39586a1a204..d8aa0239e2d 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -3352,6 +3352,8 @@ async def test_user_info_v2_internal_user_cannot_query_other(mocker): mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) + mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(return_value=[]) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) @@ -3456,6 +3458,40 @@ async def test_user_info_v2_nonexistent_user_returns_404(mocker): assert "nonexistent-user-id" in str(exc_info.value.message) +@pytest.mark.parametrize( + ("find_unique", "expected_code", "logs_traceback"), + [ + (AsyncMock(return_value=None), "404", False), + (AsyncMock(side_effect=RuntimeError("db down")), "500", True), + ], +) +@pytest.mark.asyncio +async def test_user_info_v2_logs_traceback_only_for_server_errors( + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + find_unique: AsyncMock, + expected_code: str, + logs_traceback: bool, +) -> None: + from fastapi import Request + + from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2 + + prisma: Final = mocker.MagicMock() + prisma.db.litellm_usertable.find_unique = find_unique + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma) + + with caplog.at_level(logging.ERROR), pytest.raises(ProxyException) as exc_info: + await user_info_v2( + request=mocker.MagicMock(spec=Request), + user_id="some-user", + user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert exc_info.value.code == expected_code + assert any(record.exc_info for record in caplog.records) is logs_traceback + + @pytest.mark.asyncio async def test_user_info_v2_response_shape(mocker): """ @@ -3674,6 +3710,8 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams) + mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(return_value=[]) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) @@ -3690,6 +3728,80 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): assert exc_info.value.code == "404" +_ORG_MEMBERSHIPS: Final = ( + SimpleNamespace(user_id="org-admin-user", organization_id="org-a", user_role="org_admin"), + SimpleNamespace(user_id="org-admin-user", organization_id="org-c", user_role="internal_user"), + SimpleNamespace(user_id="member-of-a", organization_id="org-a", user_role="internal_user"), + SimpleNamespace(user_id="member-of-b", organization_id="org-b", user_role="internal_user"), + SimpleNamespace(user_id="member-of-c", organization_id="org-c", user_role="internal_user"), +) + + +def _membership_matches(membership: SimpleNamespace, where: Mapping[str, object]) -> bool: + return membership.user_id == where["user_id"] and ( + "user_role" not in where or membership.user_role == where["user_role"] + ) + + +def _org_admin_prisma(mocker: MockerFixture) -> MagicMock: + def user_row(user_id: str) -> MagicMock: + row: Final = mocker.MagicMock() + row.teams = [] + row.model_dump.return_value = {"user_id": user_id, "user_email": f"{user_id}@example.com", "teams": []} + return row + + async def find_unique(**kwargs: dict[str, str]) -> MagicMock | None: + user_id: Final = kwargs["where"]["user_id"] + return user_row(user_id) if user_id in {m.user_id for m in _ORG_MEMBERSHIPS} else None + + async def find_many(**kwargs: Mapping[str, object]) -> list[SimpleNamespace]: + return [m for m in _ORG_MEMBERSHIPS if _membership_matches(m, kwargs["where"])] + + prisma: Final = mocker.MagicMock() + prisma.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=find_unique) + prisma.db.litellm_organizationmembership.find_many = mocker.AsyncMock(side_effect=find_many) + return prisma + + +@pytest.mark.asyncio +async def test_user_info_v2_org_admin_can_query_member_of_their_org(mocker: MockerFixture) -> None: + from fastapi import Request + + from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2 + + mocker.patch("litellm.proxy.proxy_server.prisma_client", _org_admin_prisma(mocker)) + + response: Final = await user_info_v2( + request=mocker.MagicMock(spec=Request), + user_id="member-of-a", + user_api_key_dict=UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + assert response.user_id == "member-of-a" + assert response.user_email == "member-of-a@example.com" + + +@pytest.mark.parametrize("target_user_id", ["member-of-b", "member-of-c"]) +@pytest.mark.asyncio +async def test_user_info_v2_org_admin_cannot_query_user_outside_their_admin_orgs( + mocker: MockerFixture, target_user_id: str +) -> None: + from fastapi import Request + + from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2 + + mocker.patch("litellm.proxy.proxy_server.prisma_client", _org_admin_prisma(mocker)) + + with pytest.raises(ProxyException) as exc_info: + await user_info_v2( + request=mocker.MagicMock(spec=Request), + user_id=target_user_id, + user_api_key_dict=UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + assert exc_info.value.code == "404" + + @pytest.mark.asyncio async def test_user_info_v2_url_encoding_plus_character(mocker): """ diff --git a/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py b/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py new file mode 100644 index 00000000000..434dcc7a83c --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py @@ -0,0 +1,180 @@ +from collections.abc import Iterator + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_state_repository, router +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.repositories.config_repository import ConfigParam, ConfigRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +class MemoryConfig(ConfigRepository): + def __init__(self) -> None: + self.rows: dict[str, object] = {} + + async def get_param(self, param_name: str) -> ConfigParam | None: + value = self.rows.get(param_name) + return None if value is None else ConfigParam(param_name, value) + + async def set_param(self, param_name: str, param_value: object) -> ConfigParam: + self.rows[param_name] = param_value + return ConfigParam(param_name, param_value) + + +@pytest.fixture() +def client() -> Iterator[tuple[TestClient, UserAPIKeyAuth]]: + app = FastAPI() + app.include_router(router) + user = UserAPIKeyAuth(user_id="admin-one", user_role=LitellmUserRoles.PROXY_ADMIN) + repository = MemoryConfig() + app.dependency_overrides[user_api_key_auth] = lambda: user + app.dependency_overrides[alert_state_repository] = lambda: repository + with TestClient(app) as http: + yield http, user + + +IDENTITY = {"model": "anthropic/claude-sonnet-4-5", "deprecation_date": "2026-11-30"} +UPDATE = {**IDENTITY, "status": "resolved", "last_seen_date": "2026-11-04", "last_seen_requests": 123} +PATH = "/alerting/model-deprecations/state" + + +def test_new_alert_is_open(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, _ = client + response = http.get(PATH, params=IDENTITY) + assert response.status_code == 200 + assert response.json()["status"] == "open" + + +def test_resolution_is_shared_and_keeps_the_usage_snapshot(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, user = client + response = http.put(PATH, json=UPDATE) + assert response.status_code == 200 + assert response.json()["updated_by"] == "admin-one" + user.user_id = "admin-two" + observed = http.get(PATH, params=IDENTITY).json() + assert observed["status"] == "resolved" + assert observed["last_seen_date"] == "2026-11-04" + assert observed["last_seen_requests"] == 123 + assert observed["updated_by"] == "admin-one" + + +def test_reopen_does_not_change_another_models_resolution(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, _ = client + http.put(PATH, json=UPDATE) + other = {**UPDATE, "model": "openai/gpt-4-turbo"} + http.put(PATH, json=other) + assert http.put(PATH, json={**UPDATE, "status": "open"}).json()["status"] == "open" + assert http.get(PATH, params={**IDENTITY, "model": other["model"]}).json()["status"] == "resolved" + + +@pytest.mark.parametrize("method", ["get", "put"]) +def test_internal_users_cannot_read_or_change_global_alerts( + client: tuple[TestClient, UserAPIKeyAuth], method: str +) -> None: + http, user = client + user.user_role = LitellmUserRoles.INTERNAL_USER + response = http.get(PATH, params=IDENTITY) if method == "get" else http.put(PATH, json=UPDATE) + assert response.status_code == 403 + + +def test_viewer_can_read_but_cannot_resolve_at_the_same_path(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, user = client + user.user_role = LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + assert http.get(PATH, params=IDENTITY).status_code == 200 + assert http.put(PATH, json=UPDATE).status_code == 403 + assert http.get(PATH, params=IDENTITY).json()["status"] == "open" + + +@pytest.mark.parametrize( + "field,value", [("status", "unknown"), ("last_seen_requests", -1), ("deprecation_date", "invalid"), ("model", "")] +) +def test_invalid_updates_are_rejected(client: tuple[TestClient, UserAPIKeyAuth], field: str, value: object) -> None: + http, _ = client + assert http.put(PATH, json={**UPDATE, field: value}).status_code == 422 + assert http.get(PATH, params=IDENTITY).json()["status"] == "open" + + +def test_tags_are_scoped_to_model_dates_and_the_visible_keys(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository + + http, _ = client + grouped = AsyncMock(return_value=[{"api_key": "reporting-one", "tag": "support"}]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={ + "model": "openai/gpt-4-turbo", + "api_keys": ["reporting-one", "reporting-two"], + "start_date": "2026-10-06", + "end_date": "2026-11-04", + }, + ) + assert response.status_code == 200 + assert response.json() == [{"api_key": "reporting-one", "tag": "support"}] + assert grouped.call_args.kwargs["where"] == { + "api_key": {"in": ["reporting-one", "reporting-two"]}, + "model": "openai/gpt-4-turbo", + "date": {"gte": "2026-10-06", "lte": "2026-11-04"}, + "api_requests": {"gt": 0}, + "tag": {"not": None}, + } + + +@pytest.mark.parametrize( + "role,expected", [(LitellmUserRoles.INTERNAL_USER, 403), (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, 200)] +) +def test_tag_access_matches_alert_visibility( + client: tuple[TestClient, UserAPIKeyAuth], role: LitellmUserRoles, expected: int +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository + + http, user = client + user.user_role = role + grouped = AsyncMock(return_value=[]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={"model": "test-model", "api_keys": ["one"], "start_date": "2026-10-06", "end_date": "2026-11-04"}, + ) + assert response.status_code == expected + if expected == 403: + grouped.assert_not_called() + + +def test_tag_query_rejects_more_than_100_keys_before_database_access(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( + alert_tag_repository, + ) + + http, _ = client + grouped = AsyncMock(return_value=[]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={ + "model": "test-model", + "api_keys": [f"reporting-{index}" for index in range(101)], + "start_date": "2026-10-07", + "end_date": "2026-11-05", + }, + ) + assert response.status_code == 422 + grouped.assert_not_called() diff --git a/tests/unit/proxy/management_endpoints/test_prompt_caching_requests.py b/tests/unit/proxy/management_endpoints/test_prompt_caching_requests.py index 39dcc9630df..ea61177f773 100644 --- a/tests/unit/proxy/management_endpoints/test_prompt_caching_requests.py +++ b/tests/unit/proxy/management_endpoints/test_prompt_caching_requests.py @@ -1,5 +1,6 @@ from collections.abc import Mapping from typing import Final +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -8,6 +9,7 @@ from fastapi import FastAPI from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.prompt_caching_requests import router +from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_error, statement_timeout_reads pytestmark = pytest.mark.usefixtures("local_model_cost_map") @@ -67,3 +69,37 @@ async def test_incomplete_cursor_is_rejected( ) as client: response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, **params}) assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_the_requests_read_is_cancelled_by_postgres_after_30s(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + prisma: Final = MagicMock() + transaction: Final = statement_timeout_reads(prisma, AsyncMock(return_value=[])) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" + ) as client: + response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END}) + + assert response.status_code == 200, response.text + assert transaction.statements == ("SET LOCAL statement_timeout = 30000",) + + +@pytest.mark.asyncio +async def test_a_cancelled_requests_read_is_a_503_with_a_date_range_hint(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + prisma: Final = MagicMock() + statement_timeout_reads(prisma, AsyncMock(side_effect=statement_timeout_error())) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" + ) as client: + response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END}) + + assert response.status_code == 503, response.text + assert response.json() == { + "detail": {"error": "Prompt caching requests took longer than 30s. Pick a shorter date range"} + } diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 2f4dad0ccd7..697d9810284 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch import httpx import litellm +from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_error, statement_timeout_reads import pytest from fastapi import HTTPException, Request from fastapi.testclient import TestClient @@ -17286,12 +17287,15 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) beta = _team_spend_by_user_team("team-beta", "Team Beta", Member(user_id="alice", role="user"), []) mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha, beta]) - mock_db_client.db.query_raw = AsyncMock( - return_value=[ - _team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3), - _team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2), - _team_spend_by_user_db_row("team-beta", "alice", 0.1, 1), - ] + transaction = statement_timeout_reads( + mock_db_client, + AsyncMock( + return_value=[ + _team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3), + _team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2), + _team_spend_by_user_db_row("team-beta", "alice", 0.1, 1), + ] + ), ) response = await get_team_spend_by_user( @@ -17301,7 +17305,9 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m end_date="2026-09-04", ) - sql, *params = mock_db_client.db.query_raw.call_args.args + assert any("statement_timeout = 30000" in statement for statement in transaction.statements) + mock_db_client.db.tx.assert_not_called() + sql, *params = transaction.query_raw.call_args.args assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta"] assert 'FROM "LiteLLM_SpendLogs" sl' in sql assert 'sl."startTime" >= $1::timestamp' in sql @@ -17328,7 +17334,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien caller = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER) alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) - mock_db_client.db.query_raw = AsyncMock(return_value=[]) + transaction = statement_timeout_reads(mock_db_client, AsyncMock(return_value=[])) mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=_team_spend_by_user_caller("alice", ["team-alpha"]) ) @@ -17337,7 +17343,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04" ) - sql, *params = mock_db_client.db.query_raw.call_args.args + sql, *params = transaction.query_raw.call_args.args assert params == ["2026-09-01", "2026-09-04", "team-alpha"] assert 'sl."user" = $' not in sql @@ -17350,7 +17356,9 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="bob", role="user"), ["/key/info"]) mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - mock_db_client.db.query_raw = AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)]) + transaction = statement_timeout_reads( + mock_db_client, AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)]) + ) mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=_team_spend_by_user_caller("bob", ["team-alpha"]) ) @@ -17359,7 +17367,7 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04" ) - sql, *params = mock_db_client.db.query_raw.call_args.args + sql, *params = transaction.query_raw.call_args.args assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob"] assert "sl.team_id IN ($3)" in sql assert 'AND sl."user" = $4' in sql @@ -17371,7 +17379,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER) - mock_db_client.db.query_raw = AsyncMock(return_value=[]) + transaction = statement_timeout_reads(mock_db_client, AsyncMock(return_value=[])) mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=_team_spend_by_user_caller("bob", ["team-alpha"]) ) @@ -17382,7 +17390,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie ) assert exc_info.value.status_code == 404 - mock_db_client.db.query_raw.assert_not_called() + transaction.query_raw.assert_not_called() @pytest.mark.asyncio @@ -17400,7 +17408,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids, start_date, end_date, expected_error): from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user - mock_db_client.db.query_raw = AsyncMock(return_value=[]) + transaction = statement_timeout_reads(mock_db_client, AsyncMock(return_value=[])) admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(HTTPException) as exc_info: @@ -17410,7 +17418,28 @@ async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids assert exc_info.value.status_code == 400 assert expected_error in str(exc_info.value.detail) - mock_db_client.db.query_raw.assert_not_called() + transaction.query_raw.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_team_spend_by_user_statement_timeout_returns_503(mock_db_client): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) + mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) + statement_timeout_reads(mock_db_client, AsyncMock(side_effect=statement_timeout_error())) + + with pytest.raises(HTTPException) as exc_info: + await get_team_spend_by_user( + user_api_key_dict=admin, + team_ids="team-alpha", + start_date="2026-09-01", + end_date="2026-09-04", + ) + + assert exc_info.value.status_code == 503 + assert "longer than 30s" in str(exc_info.value.detail) class _TeamRowWithOrganization(LiteLLM_TeamTable): diff --git a/tests/unit/proxy/mcp_server/test_upstream.py b/tests/unit/proxy/mcp_server/test_upstream.py new file mode 100644 index 00000000000..7c88c679972 --- /dev/null +++ b/tests/unit/proxy/mcp_server/test_upstream.py @@ -0,0 +1,192 @@ +import queue +from collections.abc import Iterator, Mapping +from typing import Final +from unittest.mock import MagicMock + +import httpx2 +import pytest +from mcp.server.context import ServerRequestContext +from mcp.types import CallToolResult + +from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.outbound_credentials.resolver import UpstreamCredentialProvider +from litellm.proxy.mcp_server.upstream import prepare_upstream_client +from litellm.types.mcp import MCPAuth, MCPTransport, MCPUpstreamProtocol +from litellm.types.mcp_server.mcp_server_manager import MCPServer +from tests._support.mcp_count_tool_upstream import count_tool_upstream, methods_received + +_HTTP_URL: Final = "https://upstream.example/mcp" +_SERVER_SHAPES: Final = pytest.mark.parametrize( + ("server_fields", "per_request_token", "forwarded_token"), + [ + ({"transport": MCPTransport.http, "url": _HTTP_URL}, None, None), + ( + { + "transport": MCPTransport.http, + "url": _HTTP_URL, + "auth_type": MCPAuth.bearer_token, + "authentication_token": "static-token", + }, + None, + None, + ), + ({"transport": MCPTransport.http, "url": _HTTP_URL, "auth_type": MCPAuth.bearer_token}, "caller-key", None), + ({"transport": MCPTransport.http, "url": _HTTP_URL, "auth_type": MCPAuth.oauth_delegate}, None, "caller-token"), + ], + ids=["no-auth", "static-bearer-token", "per-request-token", "forwarded-caller-token"], +) + + +@pytest.fixture +def _active_mcp_request_context() -> Iterator[None]: + context: Final = ServerRequestContext( + session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=None + ) + token: Final = active_mcp_request_ctx_var.set(context) + try: + yield + finally: + active_mcp_request_ctx_var.reset(token) + + +def _server(server_fields: Mapping[str, object], skip_setting: bool | None) -> MCPServer: + return MCPServer.model_validate( + { + **server_fields, + "server_id": "skip-validation", + "name": "skip_validation", + "mcp_info": {} if skip_setting is None else {"skip_tool_output_validation": skip_setting}, + } + ) + + +async def _call_count_tool( + manager: MCPServerManager, + server: MCPServer, + *, + per_request_token: str | None = None, + forwarded_token: str | None = None, + gateway_session_id: str | None = None, +) -> CallToolResult: + result: Final = await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="count", + arguments={}, + tasks=[], + mcp_auth_header=per_request_token, + mcp_server_auth_headers=None, + oauth2_headers=None if forwarded_token is None else {"Authorization": f"Bearer {forwarded_token}"}, + raw_headers=None if gateway_session_id is None else {"mcp-session-id": gateway_session_id}, + proxy_logging_obj=None, + ) + assert isinstance(result, CallToolResult) + return result + + +@pytest.mark.asyncio +@_SERVER_SHAPES +@pytest.mark.parametrize( + ("skip_setting", "rejected"), [(None, True), (False, True), (True, False)], ids=["unset", "off", "on"] +) +async def test_the_setting_alone_decides_whether_a_tool_result_is_checked( + server_fields: Mapping[str, object], + per_request_token: str | None, + forwarded_token: str | None, + skip_setting: bool | None, + rejected: bool, +) -> None: + upstream: Final = count_tool_upstream({"n": "not-an-integer"}, queue.SimpleQueue()) + manager: Final = MCPServerManager(upstream_transport=httpx2.MockTransport(upstream)) + + result: Final = await _call_count_tool( + manager, + _server(server_fields, skip_setting), + per_request_token=per_request_token, + forwarded_token=forwarded_token, + ) + + assert result.is_error is rejected + + +@pytest.mark.asyncio +@_SERVER_SHAPES +async def test_a_skipping_server_is_never_asked_for_its_tools( + server_fields: Mapping[str, object], per_request_token: str | None, forwarded_token: str | None +) -> None: + methods_seen: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + manager: Final = MCPServerManager( + upstream_transport=httpx2.MockTransport(count_tool_upstream({"n": 1}, methods_seen)) + ) + + result: Final = await _call_count_tool( + manager, + _server(server_fields, True), + per_request_token=per_request_token, + forwarded_token=forwarded_token, + ) + + assert result.is_error is False + assert "tools/list" not in methods_received(methods_seen) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first_setting", "second_setting", "rejected_in_order"), + [(True, False, (False, True)), (False, True, (True, False))], + ids=["turned-off", "turned-on"], +) +async def test_a_changed_setting_applies_to_the_next_call_in_a_live_gateway_session( + _active_mcp_request_context: None, + first_setting: bool, + second_setting: bool, + rejected_in_order: tuple[bool, bool], +) -> None: + upstream: Final = count_tool_upstream({"n": "not-an-integer"}, queue.SimpleQueue()) + manager: Final = MCPServerManager(upstream_transport=httpx2.MockTransport(upstream)) + manager.track_gateway_session("gateway-1") + fields: Final = {"transport": MCPTransport.http, "url": _HTTP_URL} + try: + first: Final = await _call_count_tool(manager, _server(fields, first_setting), gateway_session_id="gateway-1") + second: Final = await _call_count_tool(manager, _server(fields, second_setting), gateway_session_id="gateway-1") + finally: + await manager.close_all_upstream_sessions() + + assert (first.is_error, second.is_error) == rejected_in_order + + +@pytest.mark.asyncio +async def test_a_stdio_upstream_client_carries_the_setting(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") + server: Final = _server({"transport": MCPTransport.stdio, "command": "python", "args": ["server.py"]}, True) + + client: Final = await prepare_upstream_client( + server, provider=UpstreamCredentialProvider(), root_path="/mcp", protocol_version="auto" + ) + + assert client.skip_tool_output_validation is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("protocol_version", "expected"), [("auto", True), ("2025-11-25", True), ("2026-07-28", False)] +) +async def test_upstream_client_ignores_the_setting_where_the_call_already_lists_tools( + protocol_version: MCPUpstreamProtocol, expected: bool +) -> None: + server: Final = MCPServer.model_validate( + { + "server_id": "skip-validation", + "name": "skip_validation", + "url": _HTTP_URL, + "transport": MCPTransport.http, + "protocol_version": protocol_version, + "mcp_info": {"skip_tool_output_validation": True}, + } + ) + + client: Final = await prepare_upstream_client( + server, provider=UpstreamCredentialProvider(), root_path="/mcp", protocol_version=protocol_version + ) + + assert client.skip_tool_output_validation is expected diff --git a/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py index 2727857c352..48981288d00 100644 --- a/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py +++ b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py @@ -54,10 +54,11 @@ async def _echo_request(request: Request) -> Response: ) -def _client(max_decompressed_body_size_mb: float = CAP_MB, **options: object) -> TestClient: +def _client(max_decompressed_body_size_mb: float = CAP_MB, root_path: str = "", **options: object) -> TestClient: echo_app: Final = Starlette( routes=[ Route("/v1/systemone", _echo_request, methods=["POST"]), + Route("/claude-code/plugins/upload", _echo_request, methods=["POST"]), Route("/plugin-proxy/{path:path}", _echo_request, methods=["POST"]), Route("/claude_code_gateway/v1/{signal}", _echo_request, methods=["POST"]), ] @@ -67,7 +68,8 @@ def _client(max_decompressed_body_size_mb: float = CAP_MB, **options: object) -> echo_app, max_decompressed_body_size_mb=max_decompressed_body_size_mb, **options, # pyright: ignore[reportArgumentType] # each test passes one typed keyword - ) + ), + root_path=root_path, ) @@ -220,19 +222,51 @@ def test_body_decoding_past_the_cap_is_rejected(): @pytest.mark.parametrize( ("operator_max_mb", "effective_max_mb"), - [(0.25, 0.25), (CAP_MB * 4, CAP_MB), (None, CAP_MB), (0, CAP_MB)], + [(0.25, 0.25), (CAP_MB * 4, CAP_MB), (None, CAP_MB), (0, CAP_MB), (1 / 64, 1 / 64)], ) +@pytest.mark.parametrize(("route_max_mb", "root_path"), [(None, ""), (1 / 32, ""), (1 / 32, "/gateway")]) +@pytest.mark.parametrize("coding", ("gzip", "zstd")) def test_operator_request_size_limit_caps_the_decoded_body_when_it_is_lower( - operator_max_mb: float | None, effective_max_mb: float + operator_max_mb: float | None, effective_max_mb: float, route_max_mb: float | None, root_path: str, coding: str, ): - client: Final = _client(get_max_request_size_mb=lambda: operator_max_mb) - within: Final = zstd.compress(_low_entropy_text(int(effective_max_mb * MIB))) - over: Final = zstd.compress(_low_entropy_text(int(effective_max_mb * MIB) + 1)) - - assert _post(client, within, ("zstd",)).status_code == 200 - over_response: Final = _post(client, over, ("zstd",)) + path: Final = "/claude-code/plugins/upload" if route_max_mb else "/v1/systemone" + limit: Final = min(effective_max_mb, route_max_mb or effective_max_mb) + client: Final = _client( + root_path=root_path, + get_max_request_size_mb=lambda: operator_max_mb, + route_max_request_size_mb={("POST", path): route_max_mb} if route_max_mb else None, + ) + compress: Final = gzip.compress if coding == "gzip" else zstd.compress + within: Final = compress(_low_entropy_text(int(limit * MIB))) + over: Final = compress(_low_entropy_text(int(limit * MIB) + 1)) + assert _post(client, within, (coding,), f"{root_path}{path}").status_code == 200 + over_response: Final = _post(client, over, (coding,), f"{root_path}{path}") assert over_response.status_code == 413 - assert over_response.json() == _decoded_too_large(effective_max_mb) + assert over_response.json() == _decoded_too_large(limit) + + +@pytest.mark.parametrize(("coding", "compress"), [ + ("identity", bytes), ("gzip", gzip.compress), ("zstd", zstd.compress), +]) +def test_proxy_bounds_skill_uploads_before_authentication_or_body_parsing( + coding: str, compress: Callable[[bytes], bytes], +) -> None: + from litellm.proxy.proxy_server import app + + body: Final = random.Random(0).randbytes(6 * MIB) + bytes(6 * MIB) + encoded: Final = compress(body) + + response: Final = TestClient(app).post( + "/claude-code/plugins/upload", + content=encoded, + headers={"content-encoding": coding, "content-type": "multipart/form-data; boundary=upload"}, + ) + + assert (len(encoded) < 11 * MIB) == (coding != "identity") + assert response.status_code == 413 + assert response.json() == ( + {"error": "Request size is too large. Max size is 11 MB"} if coding == "identity" else _decoded_too_large(11) + ) @pytest.mark.parametrize(("signature", "content_encoding"), [(GZIP_MAGIC, "gzip"), (ZSTD_FRAME_MAGIC, "zstd")]) diff --git a/tests/unit/proxy/middleware/test_request_size_limit_middleware.py b/tests/unit/proxy/middleware/test_request_size_limit_middleware.py index 0a30b4cf9ef..7bb73b77000 100644 --- a/tests/unit/proxy/middleware/test_request_size_limit_middleware.py +++ b/tests/unit/proxy/middleware/test_request_size_limit_middleware.py @@ -1,5 +1,7 @@ +from collections.abc import AsyncIterator from typing import Final +import httpx import pytest from starlette.responses import JSONResponse from starlette.testclient import TestClient @@ -10,7 +12,16 @@ from litellm.proxy.middleware.request_size_limit_middleware import ( ) -def test_request_size_limit_middleware_rejects_content_length_before_body_read(): +@pytest.mark.parametrize(("path", "root_path", "operator_mb", "route_mb"), [ + ("/chat/completions", "", 1, None), + ("/claude-code/plugins/upload", "", None, 1), + ("/gateway/claude-code/plugins/upload", "/gateway", None, 1), + ("/claude-code/plugins/upload", "", 2, 1), + ("/claude-code/plugins/upload", "", 1, 2), +]) +def test_request_size_limit_middleware_rejects_content_length_before_body_read( + path: str, root_path: str, operator_mb: int | None, route_mb: int | None, +) -> None: downstream_called = False async def app(scope, receive, send): @@ -22,13 +33,15 @@ def test_request_size_limit_middleware_rejects_content_length_before_body_read() client = TestClient( RequestSizeLimitMiddleware( app, - get_max_request_size_mb=lambda: 1, - is_request_size_limit_enabled=lambda: True, - ) + get_max_request_size_mb=lambda: operator_mb, + is_request_size_limit_enabled=lambda: operator_mb is not None, + route_max_request_size_mb={("POST", "/claude-code/plugins/upload"): route_mb} if route_mb else None, + ), + root_path=root_path, ) response = client.post( - "/chat/completions", + path, content=b"x" * (1024 * 1024 + 1), headers={"content-type": "application/json"}, ) @@ -39,7 +52,16 @@ def test_request_size_limit_middleware_rejects_content_length_before_body_read() assert downstream_called is False -def test_request_size_limit_middleware_zero_limit_disables_guard(): +@pytest.mark.parametrize(("enabled", "operator_mb"), [(True, 0), (False, 1)]) +@pytest.mark.parametrize(("method", "path"), [ + ("POST", "/chat/completions"), + ("PUT", "/claude-code/plugins/upload"), + ("POST", "/claude-code/plugins/upload-other"), + ("POST", "/claude-code/plugins"), +]) +def test_request_size_limit_middleware_zero_or_disabled_global_limit_preserves_other_routes( + enabled: bool, operator_mb: int, method: str, path: str, +) -> None: downstream_called = False async def app(scope, receive, send): @@ -51,14 +73,15 @@ def test_request_size_limit_middleware_zero_limit_disables_guard(): client = TestClient( RequestSizeLimitMiddleware( app, - get_max_request_size_mb=lambda: 0, - is_request_size_limit_enabled=lambda: True, + get_max_request_size_mb=lambda: operator_mb, + is_request_size_limit_enabled=lambda: enabled, + route_max_request_size_mb={("POST", "/claude-code/plugins/upload"): 1}, ) ) - response = client.post( - "/chat/completions", - content=b"x", + response = client.request( + method, path, + content=b"x" * (1024 * 1024 + 1), headers={"content-type": "application/json"}, ) @@ -154,3 +177,33 @@ def test_an_oversized_messages_request_gets_the_anthropic_request_too_large_erro "type": "error", "error": {"type": "request_too_large", "message": "Request size is too large. Max size is 1 MB"}, } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/claude-code/plugins/upload", "/chat/completions")) +async def test_chunked_upload_overflow_stops_fastapi_and_auth_multipart_parsing( + path: str, +) -> None: + from litellm.proxy.proxy_server import app + + limited: Final = RequestSizeLimitMiddleware( + app, + get_max_request_size_mb=lambda: None, + is_request_size_limit_enabled=lambda: False, + route_max_request_size_mb={("POST", path): 1 / 1024}, + ) + + async def chunks() -> AsyncIterator[bytes]: + yield b'--skill\r\nContent-Disposition: form-data; name="file"; filename="skill.zip"\r\n\r\n' + yield b"x" * 1024 + pytest.fail("The server consumed more of the upload after its byte limit") + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=limited), base_url="http://test") as client: + response: Final = await client.post( + path, + content=chunks(), + headers={"content-type": "multipart/form-data; boundary=skill"}, + ) + + assert response.status_code == 413 + assert response.json() == {"detail": "Request size is too large. Max size is 0.0009765625 MB"} diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 4a76aa77f14..11b07472eaa 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7241,6 +7241,178 @@ class TestRouterModelRelayUpstreamContract: assert json.loads(result.body) == upstream_body assert result.headers["x-ms-request-id"] == "req-1" + @pytest.mark.asyncio + async def test_azure_relay_answers_401_when_entra_rejects_the_deployment_credential(self, monkeypatch): + from litellm.llms.azure.common_utils import EntraRejectionError + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + reason: Final = "Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided." + + class EntraRejectingRouter: + async def allm_passthrough_route(self, **kwargs): + try: + raise EntraRejectionError(status_code=401, message=reason) + except EntraRejectionError as rejection: + raise BaseLLMException(status_code=401, message=str(rejection)) from rejection + + self._install_router(monkeypatch, EntraRejectingRouter(), {"model": "gpt-5", "stream": False}) + + with pytest.raises(ProxyException) as excinfo: + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert (excinfo.value.code, excinfo.value.message) == ("401", reason) + + @pytest.mark.asyncio + async def test_azure_relay_entra_rejection_hides_an_internal_issuer_host(self, monkeypatch): + from litellm.llms.azure.common_utils import EntraRejectionError + + reason: Final = ( + "Failed to get Azure AD token: AADSTS700211: No matching federated identity record found for " + "presented assertion issuer 'https://oidc.build.internal:8443/realms/ci'." + ) + + class EntraRejectingRouter: + async def allm_passthrough_route(self, **kwargs): + raise EntraRejectionError(status_code=401, message=reason) + + self._install_router(monkeypatch, EntraRejectingRouter(), {"model": "gpt-5", "stream": False}) + + with pytest.raises(ProxyException) as excinfo: + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert excinfo.value.code == "401" + assert "AADSTS700211" in excinfo.value.message + assert "oidc.build.internal" not in excinfo.value.message + + @pytest.mark.asyncio + async def test_azure_relay_keeps_a_failure_entra_did_not_cause(self, monkeypatch): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + upstream_failure: Final = BaseLLMException(status_code=500, message="Connection reset by Azure") + + class FailingRouter: + async def allm_passthrough_route(self, **kwargs): + raise upstream_failure + + self._install_router(monkeypatch, FailingRouter(), {"model": "gpt-5", "stream": False}) + + with pytest.raises(BaseLLMException) as excinfo: + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert excinfo.value is upstream_failure + + @pytest.mark.asyncio + async def test_azure_relay_keeps_entra_throttling_the_oidc_token_exchange_as_a_429(self, monkeypatch): + monkeypatch.setenv("LITELLM_QA_FEDERATED_TOKEN", "federated-token") + monkeypatch.delenv("AZURE_AUTHORITY_HOST", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5", + "litellm_params": { + "model": "azure/gpt-5", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "throttled-tenant", + "client_id": "qa-client", + "azure_ad_token": "oidc/env/LITELLM_QA_FEDERATED_TOKEN", + }, + } + ], + num_retries=0, + ) + self._install_router(monkeypatch, router, {"model": "gpt-5", "stream": False}) + + with respx.mock: + respx.post("https://login.microsoftonline.com/throttled-tenant/oauth2/v2.0/token").mock( + return_value=httpx.Response( + 429, + headers={"Retry-After": "7"}, + json={"error": "temporarily_unavailable", "error_codes": [90055]}, + ) + ) + deployment_route: Final = respx.post(url__startswith="https://litellm-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert (response.status_code, response.headers.get("retry-after")) == (429, "7") + assert not deployment_route.called + + + @pytest.mark.asyncio + async def test_azure_relay_keeps_a_later_deployment_failure_after_an_entra_rejection( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from azure.core.exceptions import ClientAuthenticationError + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + def rejected_token() -> str: + raise ClientAuthenticationError(message="AADSTS7000215: Invalid client secret provided.") + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5", + "litellm_params": { + "model": "azure/gpt-5", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "azure_ad_token_provider": rejected_token, + "order": 1, + }, + }, + { + "model_name": "gpt-5", + "litellm_params": { + "model": "azure/gpt-5", + "api_base": "https://litellm-qa-unreachable.openai.azure.com", + "api_version": "2025-04-01-preview", + "api_key": "configured-key", + "order": 2, + }, + }, + ], + num_retries=1, + ) + self._install_router(monkeypatch, router, {"model": "gpt-5", "stream": False}) + + with respx.mock: + unreachable_route: Final = respx.post(url__startswith="https://litellm-qa-unreachable.openai.azure.com").mock( + side_effect=httpx.ConnectError("Connection refused") + ) + with pytest.raises(BaseLLMException) as excinfo: + await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=self._request("application/json"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert unreachable_route.called + assert excinfo.value.status_code == 500 + NIM_INFER_BODY = { "input": [ diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 55226fc629b..712b4134c05 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -9,6 +9,7 @@ Pins covered: from __future__ import annotations import asyncio +import base64 import json import logging import os @@ -25,11 +26,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding, rsa +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey from prisma.errors import DataError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import turn_on_json +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.config_resolvers.settings_store import SettingsStore @@ -2671,6 +2676,213 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) +@pytest.mark.asyncio +async def test_load_config_refuses_allowed_ips_without_a_license(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config_file: Final = tmp_path / "allowed_ips.yaml" + config_file.write_text('model_list: []\nlitellm_settings: {}\ngeneral_settings:\n allowed_ips: ["127.0.0.1"]\n') + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + expected_message: Final = ( + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + +def _signed_test_license(private_key: RSAPrivateKey, max_users: int) -> str: + message: Final = json.dumps({"expiration_date": "9999-12-31", "user_id": "test", "max_users": max_users}).encode() + signature: Final = private_key.sign( + message, + padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH), + hashes.SHA256(), + ) + return base64.b64encode(message + b"." + signature).decode() + + +def _write_config(path: Path, general_settings: dict[str, object], litellm_settings: dict[str, object]) -> str: + path.write_text( + json.dumps({"model_list": [], "litellm_settings": litellm_settings, "general_settings": general_settings}) + ) + return str(path) + + +def _boot_proxy( + monkeypatch: pytest.MonkeyPatch, + public_key: RSAPublicKey, + env_license: str | None, + license_server_urls: list[str], + server_accepted_license: str | None = None, +) -> None: + from litellm.proxy import proxy_server + + def _license_server(request: httpx.Request) -> httpx.Response: + license_server_urls.append(str(request.url)) + return httpx.Response( + 200, json={"verify": server_accepted_license is not None and server_accepted_license in str(request.url)} + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.public_key", public_key) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.airgapped_license_data", None) + monkeypatch.setattr( + "litellm.proxy.proxy_server._license_check.http_handler", + HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_license_server))), + ) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + if env_license is None: + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + else: + monkeypatch.setenv("LITELLM_LICENSE", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.license_str", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", proxy_server._license_check.is_premium()) + monkeypatch.setattr( + "litellm.proxy.proxy_server.premium_user_data", proxy_server._license_check.airgapped_license_data + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "gated_config", + [ + {"general_settings": {"allowed_ips": ["127.0.0.1"]}}, + {"general_settings": {"enforced_params": ["user"]}}, + {"litellm_settings": {"callbacks": ["openai_moderations"]}}, + ], +) +async def test_load_config_lets_a_config_license_unlock_its_enterprise_gates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, gated_config: dict[str, dict[str, list[str]]] +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + license_key: Final = _signed_test_license(private_key, max_users=10) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", + {"litellm_license": license_key, **gated_config.get("general_settings", {})}, + dict(gated_config.get("litellm_settings", {})), + ) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + + +@pytest.mark.asyncio +async def test_load_config_keeps_a_valid_env_license_when_the_config_license_is_invalid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.user_seat_limit() == 5 + + +@pytest.mark.asyncio +async def test_load_config_prefers_the_config_license_when_both_licenses_are_valid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", {"litellm_license": _signed_test_license(private_key, max_users=100)}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server._license_check.user_seat_limit() == 100 + assert (await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()))["limits"]["max_users"] == 100 + + +@pytest.mark.asyncio +async def test_load_config_still_reports_an_invalid_config_license_as_configured( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + license_status: Final = await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()) + assert (license_status["has_license"], license_status["license_type"]) == (True, "community") + + +@pytest.mark.asyncio +async def test_load_config_does_not_recheck_an_env_license_that_failed_at_boot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + license_server_urls: Final[list[str]] = [] + _boot_proxy(monkeypatch, private_key.public_key(), "also-not-a-valid-license", license_server_urls) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is False + assert [url for url in license_server_urls[asked_at_boot:] if "also-not-a-valid-license" in url] == [] + assert any("not-a-valid-license" in url for url in license_server_urls[asked_at_boot:]) + + +@pytest.mark.asyncio +async def test_load_config_keeps_the_boot_verdict_of_a_server_checked_env_license( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + license_server_urls: Final[list[str]] = [] + _boot_proxy( + monkeypatch, + private_key.public_key(), + "server-checked-license", + license_server_urls, + server_accepted_license="server-checked-license", + ) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.license_str == "server-checked-license" + assert [url for url in license_server_urls[asked_at_boot:] if "server-checked-license" in url] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "setting", ["vector_store_deny_by_default", "search_tool_deny_by_default", "claude_code_marketplace_auth_required"] diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index 9b39678f203..9fa617c0dc3 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -11,21 +11,35 @@ from __future__ import annotations import contextlib import io -from typing import TYPE_CHECKING, Callable, Final +import json +import math +import wave +from collections.abc import AsyncIterator, Callable, Mapping +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi import HTTPException +from fastapi.testclient import TestClient +from openai import AsyncOpenAI +from starlette.datastructures import FormData, UploadFile +import litellm from litellm import Router +from litellm.caching.caching import Cache +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.route_llm_request import route_request +from litellm.proxy.utils import _failure_fields_to_lift +from litellm.types.guardrails import GuardrailEventHooks, GuardrailStreamScope from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep from litellm.types.router import RouterErrors - -if TYPE_CHECKING: - from fastapi.testclient import TestClient +from litellm.types.utils import TranscriptionResponse _TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']" @@ -130,9 +144,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -164,6 +176,47 @@ def patched_transcription_error(monkeypatch, patched_transcription): yield +@pytest.fixture +def patched_transcription_stream(monkeypatch, patched_transcription): + class _FakeEvent: + def model_dump_json(self, exclude_unset: bool = False) -> str: + return '{"type":"transcript.text.done","text":"hello world"}' + + class _FakeAsyncStream: + def __init__(self): + self.closed = False + + def __aiter__(self): + async def _events(): + yield _FakeEvent() + + return _events() + + async def aclose(self): + self.closed = True + + async def _form_data(request): + from starlette.datastructures import FormData, UploadFile + + upload = UploadFile( + filename="audio.mp3", + file=io.BytesIO(b"\x00\x01\x02"), + ) + return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")]) + + stream = _FakeAsyncStream() + + async def _llm_call(): + return stream + + async def _fake_route_request(*args, **kwargs): + return _llm_call() + + monkeypatch.setattr(proxy_server, "get_form_data", _form_data) + monkeypatch.setattr(proxy_server, "route_request", _fake_route_request) + yield stream + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_happy_path(client, auth_as, patched_speech, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy).""" @@ -366,6 +419,637 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, assert len(response.content) > 0 +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path): + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + data = {"model": "gpt-transcribe", "stream": "true"} + with auth_as(): + response = client.post(path, files=files, data=data) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\ndata: [DONE]\n\n' + assert patched_transcription_stream.closed is True + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize("configuration", ["default", "policy"]) +def test_streaming_transcription_keeps_streaming_with_output_guardrails( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + configuration: str, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="transcription-output", + event_hook=GuardrailEventHooks.post_call, + default_on=configuration == "default", + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + if configuration == "policy": + pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) + proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { + **kwargs["data"], + "metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]}, + } + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"hello world"' in response.text + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_returns_the_litellm_call_and_model_headers( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"hello world"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", + file=data["file"], + stream=True, + client=upstream, + litellm_call_id=data["litellm_call_id"], + model_info={"id": "transcription-deployment"}, + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + headers={"x-litellm-call-id": "streamed-transcription-call"}, + ) + + assert response.status_code == 200 + assert '"text":"hello world"' in response.text + assert response.headers["x-litellm-call-id"] == "streamed-transcription-call" + assert response.headers["x-litellm-model-id"] == "transcription-deployment" + assert [name for name in response.headers if name.startswith("x-litellm-response-cost")] == [] + proxy_server.proxy_logging_obj.update_request_status.assert_called_once_with( + litellm_call_id="streamed-transcription-call", status="success" + ) + + +class _TranscriptRedactor(CustomGuardrail): + def __init__(self, stream_scope: GuardrailStreamScope = "both") -> None: + super().__init__( + guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call, stream_scope=stream_scope + ) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +def _streams_when_asked(request: httpx.Request) -> httpx.Response: + if b'name="stream"' in request.content: + return httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"my pin is 1234"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + return httpx.Response(200, json={"text": "my pin is 1234", "usage": {"type": "duration", "seconds": 2}}) + + +def _sse_events(body: str) -> tuple[object, ...]: + assert body.endswith("data: [DONE]\n\n") + return tuple(json.loads(frame.removeprefix("data: ")) for frame in body.removesuffix("data: [DONE]\n\n").split("\n\n") if frame) + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None)) +def test_streamed_transcription_honors_a_deployment_post_call_guardrail( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_guardrails: list[str] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked) + deployment_params: Final = { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + } + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": ( + {**deployment_params, "guardrails": deployment_guardrails} + if deployment_guardrails + else deployment_params + ), + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + if deployment_guardrails: + assert response.headers["content-type"].startswith("text/event-stream") + assert _sse_events(response.text) == ( + {"type": "transcript.text.delta", "delta": "[REDACTED]"}, + {"type": "transcript.text.done", "text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}, + ) + else: + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"my pin is 1234"' in response.text + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize( + "stream_scope,upstream_streamed,expected_text", + [("streaming", False, "[REDACTED]"), ("non_streaming", True, "my pin is 1234")], +) +def test_streamed_transcription_applies_a_deployment_guardrail_only_in_its_stream_scope( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + stream_scope: GuardrailStreamScope, + upstream_streamed: bool, + expected_text: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor(stream_scope=stream_scope)]) + upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock( + side_effect=_streams_when_asked + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "guardrails": ["transcript-redactor"], + }, + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream") + assert (b'name="stream"' in upstream_route.calls.last.request.content) is upstream_streamed + assert _sse_events(response.text)[-1]["text"] == expected_text + + +async def _form_without_stream_field(request: object) -> FormData: + return FormData( + [("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))), ("model", "gpt-transcribe")] + ) + + +@pytest.mark.usefixtures("patched_transcription") +@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None)) +def test_a_deployment_that_streams_transcription_answers_sse_to_a_client_that_sent_no_stream_field( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_guardrails: list[str] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + monkeypatch.setattr(proxy_server, "get_form_data", _form_without_stream_field) + respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked) + deployment_params: Final = { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "stream": True, + } + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": ( + {**deployment_params, "guardrails": deployment_guardrails} + if deployment_guardrails + else deployment_params + ), + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe"}, + ) + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream") + assert _sse_events(response.text)[-1]["text"] == ("[REDACTED]" if deployment_guardrails else "my pin is 1234") + header_hook_response: Final = proxy_server.proxy_logging_obj.post_call_response_headers_hook.await_args.kwargs[ + "response" + ] + if deployment_guardrails: + assert isinstance(header_hook_response, TranscriptionResponse) + assert header_hook_response.text == "[REDACTED]" + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_cached_guarded_transcription_still_streams_to_a_stream_client( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock( + side_effect=_streams_when_asked + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "guardrails": ["transcript-redactor"], + }, + } + ], + cache_responses=True, + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + responses: Final = [ + client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + for _ in range(2) + ] + + assert upstream_route.call_count == 1 + assert [response.headers["content-type"].split(";")[0] for response in responses] == ["text/event-stream"] * 2 + assert _sse_events(responses[1].text)[-1] == { + "type": "transcript.text.done", + "text": "[REDACTED]", + "usage": {"type": "duration", "seconds": 2}, + } + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_frames_keep_the_provider_event_shape( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider_events: Final = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 2}}, + ) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in provider_events).encode(), + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", file=data["file"], stream=data["stream"], client=upstream + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert _sse_events(response.text) == provider_events + + +class _FirstDeltaThenReset(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise httpx.ReadError("connection reset") + + +def _silent_wav(seconds: int) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00\x00" * 16_000 * seconds) + return buffer.getvalue() + + +@pytest.mark.usefixtures("patched_transcription") +def test_a_streamed_transcription_that_fails_mid_transcript_bills_the_uploaded_seconds( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenReset() + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream) + + four_second_upload: Final = _silent_wav(4) + + async def stream_form(request: object) -> FormData: + return FormData( + [ + ("file", UploadFile(filename="sample.wav", file=io.BytesIO(four_second_upload))), + ("model", "gpt-transcribe"), + ("stream", "true"), + ] + ) + + monkeypatch.setattr(proxy_server, "get_form_data", stream_form) + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", four_second_upload, "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert '"error"' in response.text + failure_request: Final = proxy_server.proxy_logging_obj.post_call_failure_hook.await_args.kwargs["request_data"] + billed_cost: Final = _failure_fields_to_lift(failure_request)["response_cost"] + assert isinstance(billed_cost, float) + assert math.isclose(billed_cost, 4 * litellm.model_cost["gpt-transcribe"]["input_cost_per_second"]) + + +async def _scribe_stream_form(request: object) -> FormData: + return FormData( + [ + ("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))), + ("model", "scribe"), + ("stream", "true"), + ] + ) + + +@pytest.mark.usefixtures("patched_transcription") +def test_transcription_stream_form_field_reaches_other_providers_as_sent( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = respx_mock.post("https://elevenlabs.example/v1/speech-to-text").mock( + return_value=httpx.Response(200, json={"text": "hello world", "language_code": "en"}) + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "scribe", + "litellm_params": { + "model": "elevenlabs/scribe_v1", + "api_key": "sk-test", + "api_base": "https://elevenlabs.example", + }, + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + monkeypatch.setattr(proxy_server, "get_form_data", _scribe_stream_form) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "scribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert response.json()["text"] == "hello world" + assert b'name="stream"\r\n\r\ntrue\r\n' in upstream.calls.last.request.content + + +def _streams_chat_and_transcription(request: httpx.Request) -> httpx.Response: + sse_body: Final = ( + b'data: {"id":"chat-1","object":"chat.completion.chunk","created":0,"model":"gpt-4o",' + b'"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' + if request.url.path.endswith("/chat/completions") + else b'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + ) + return httpx.Response( + 200, + content=sse_body, + headers={"content-type": "text/event-stream", "x-ratelimit-remaining-requests": "99"}, + ) + + +def _litellm_header_names(response: httpx.Response) -> frozenset[str]: + return frozenset( + name + for name in response.headers + if name.startswith("x-litellm-") and not name.startswith("x-litellm-response-cost") + ) + + +def test_streamed_transcription_carries_the_headers_a_streamed_chat_completion_carries( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.route(host="stream-upstream.example").mock(side_effect=_streams_chat_and_transcription) + upstream: Final = {"api_key": "sk-test", "api_base": "https://stream-upstream.example/v1"} + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", **upstream}}, + {"model_name": "gpt-transcribe", "litellm_params": {"model": "openai/gpt-transcribe", **upstream}}, + ] + ), + ) + + with auth_as(): + chat: Final = client.post( + "/v1/chat/completions", + json={"model": "gpt-4o", "stream": True, "messages": [{"role": "user", "content": "hi"}]}, + ) + transcription: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert chat.status_code == 200 + assert transcription.status_code == 200 + assert transcription.headers["content-type"].startswith("text/event-stream") + assert _litellm_header_names(chat) - _litellm_header_names(transcription) <= {"x-litellm-overhead-duration-ms"} + assert transcription.headers["x-litellm-model-name"] == "openai/gpt-transcribe" + assert transcription.headers["llm_provider-x-ratelimit-remaining-requests"] == "99" + assert [name for name in transcription.headers if name.startswith("x-litellm-response-cost")] == [] + + +class _FailsAfterTwoDeltas(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + yield b'data: {"type":"transcript.text.delta","delta":"world"}\n\n' + raise httpx.ReadError("upstream connection dropped at /srv/app/stt_stream.py via stt.internal.corp") + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_error_mid_stream_ends_with_an_error_event_and_runs_the_failure_hook( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, stream=_FailsAfterTwoDeltas(), headers={"content-type": "text/event-stream"} + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream) + + failure_hook: Final = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "post_call_failure_hook", failure_hook) + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + data_lines: Final = tuple( + line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ") + ) + frames: Final = tuple(json.loads(line) for line in data_lines[:-1]) + assert response.status_code == 200 + assert tuple(frame.get("delta") for frame in frames[:2]) == ("hello ", "world") + assert len(frames) == 3 + assert data_lines[-1] == "[DONE]" + assert "upstream connection dropped" in frames[2]["error"]["message"] + assert "/srv/app/stt_stream.py" not in frames[2]["error"]["message"] + assert "stt.internal.corp" not in frames[2]["error"]["message"] + assert set(frames[2]["error"]) == {"message", "type", "param", "code"} + assert (frames[2]["error"]["type"], frames[2]["error"]["param"], frames[2]["error"]["code"]) == ( + "internal_server_error", + None, + "500", + ) + failure_hook.assert_awaited_once() + assert isinstance(failure_hook.await_args.kwargs["original_exception"], litellm.APIConnectionError) + + def _post_speech(client: TestClient) -> httpx.Response: return client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"}) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index bffa4612964..6ad5ef1a4eb 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi.testclient import TestClient import litellm @@ -576,6 +577,55 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"}) +@pytest.mark.parametrize( + ("key_models", "expected_names", "expected_listings"), + [(["qwen-open"], ["qwen-open"], 0), (["hosted_vllm/*"], ["hosted_vllm/qwen2.5"], 1)], + ids=["key-without-the-wildcard", "key-granted-the-wildcard"], +) +def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it( + client, auth_as, monkeypatch, request, key_models, expected_names, expected_listings +): + from litellm.proxy._types import LitellmUserRoles + + wildcard_server: Final = f"http://vllm-{request.node.callspec.id}.test:8000/v1" + other_alias_server: Final = f"http://vllm-secure-{request.node.callspec.id}.test:8000/v1" + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-open", + "litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"}, + }, + {"model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": wildcard_server}}, + { + "model_name": "vllm-secure/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": other_alias_server}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with respx.mock(assert_all_called=False) as upstream: + listing: Final = upstream.get(f"{wildcard_server}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + other_alias_listing: Final = upstream.get(f"{other_alias_server}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + with auth_as(role=LitellmUserRoles.INTERNAL_USER, models=key_models): + response: Final = client.get("/v1/model/info") + + assert response.status_code == 200, response.text + assert ( + [model["model_name"] for model in response.json()["data"]], + listing.call_count, + other_alias_listing.call_count, + ) == (expected_names, expected_listings, 0) + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py b/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py index a771b1ccdb0..5d460617dda 100644 --- a/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py +++ b/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py @@ -69,7 +69,7 @@ def test_get_blog_posts_returns_every_fetched_post(client): def test_get_blog_posts_returns_local_backup_on_failure(client): - """Endpoint returns the local backup, with empty tags, when the fetcher raises.""" + """Endpoint returns the local backup when the fetcher raises.""" with patch( "litellm.proxy.public_endpoints.public_endpoints.get_blog_posts", side_effect=Exception("fetch failed"), @@ -79,4 +79,3 @@ def test_get_blog_posts_returns_local_backup_on_failure(client): assert response.status_code == 200 backup = [post.model_dump(mode="json") for post in GetBlogPosts.load_local_blog_posts()] assert response.json()["posts"] == backup - assert all(post["tags"] == [] for post in response.json()["posts"]) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 72755633376..c17a9ba9371 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -147,29 +147,20 @@ def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_ent ) -def test_get_litellm_model_cost_map_returns_cost_map(): - app = FastAPI() +def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + model_cost: Final = { + "audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"}, + "token-model": {"input_cost_per_token": 0.01, "mode": "chat"}, + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + app: Final = FastAPI() app.include_router(router) - client = TestClient(app) + client: Final = TestClient(app) - response = client.get("/public/litellm_model_cost_map") + response: Final = client.get("/public/litellm_model_cost_map") assert response.status_code == 200 - payload = response.json() - assert isinstance(payload, dict) - assert len(payload) > 0, "Expected model cost map to contain at least one model" - - # Verify the structure contains expected keys for at least one model - # Check for a common model like gpt-4 or gpt-3.5-turbo - model_keys = list(payload.keys()) - assert len(model_keys) > 0 - - # Verify at least one model has expected cost fields - sample_model = model_keys[0] - sample_model_data = payload[sample_model] - assert isinstance(sample_model_data, dict) - # Check for common cost fields that should be present - assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data + assert response.json() == model_cost def test_public_ai_hub_info_is_public_by_default(monkeypatch): @@ -536,7 +527,9 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: ( ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), ("Nace", "Nace", LlmProviders.NACE), + ("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI), ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), + ("EMPIRIOLABS", "EmpirioLabs AI", LlmProviders.EMPIRIOLABS), ), ) def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f5c97142dde..ab8dd8567b6 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -4,18 +4,23 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: - POST /v1/realtime/calls """ +import functools import json import time -from collections.abc import Awaitable -from typing import Protocol +from collections.abc import Awaitable, Mapping +from typing import Final, Protocol from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from starlette.routing import Match - +import litellm +from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE +from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -23,8 +28,18 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.realtime_endpoints.endpoints import ( + _ALLOWED_SESSION_TYPES, + _coerce_realtime_session_type, _decode_realtime_token_payload, _encode_realtime_token_payload, + _prepare_client_secret_session, +) +from litellm.types.realtime import ( + RealtimeAudioInputConfig, + RealtimeAudioTranscriptionConfig, + RealtimeClientSecretRequest, + RealtimeSessionAudioConfig, + RealtimeSessionConfig, ) # --- Unit tests: token encode/decode helpers --- @@ -126,9 +141,7 @@ def mock_route_request_client_secrets(): future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { @@ -215,9 +228,7 @@ async def test_client_secrets_success_with_mock( mock_pre_call_hook, ): """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -275,13 +286,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -312,12 +317,8 @@ async def test_client_secrets_transcription_routes_on_nested_model( async def _inner(): resp = MagicMock(spec=httpx.Response) resp.status_code = 200 - resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) - resp.content = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ).encode() + resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode() resp.headers = {} resp.json.return_value = { "value": "upstream_ephemeral_key", @@ -351,13 +352,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -367,10 +362,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( session = captured["data"]["session"] assert session["type"] == "transcription" assert "model" not in session - assert ( - session["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" encrypted_value = response.json()["value"] decoded = _decode_realtime_token_payload( decrypt_value_helper( @@ -530,10 +522,7 @@ async def test_realtime_calls_replays_transcription_session_type( ) assert captured["session"]["type"] == "transcription" - assert ( - captured["session"]["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" # --- transcription_sessions endpoint --- @@ -587,8 +576,12 @@ def test_transcription_sessions_requires_auth(proxy_app): @pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + {"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, +]) async def test_transcription_sessions_rejects_disallowed_resolved_model( - proxy_app, + proxy_app, body, ): proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="test-user", @@ -605,9 +598,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json=body, ) assert response.status_code == 403 @@ -651,9 +642,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -696,9 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -751,9 +738,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -947,13 +932,12 @@ async def test_transcription_sessions_encrypts_client_secret( POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key under client_secret.value must be encrypted (never the raw upstream key). """ - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") captured_route_type = {} async def _capturing_route(*args, **kwargs): captured_route_type["route_type"] = kwargs.get("route_type") + captured_route_type["session"] = kwargs["data"]["transcription_session"] return await mock_route_request_transcription_sessions(*args, **kwargs) try: @@ -976,6 +960,8 @@ async def test_transcription_sessions_encrypts_client_secret( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, json={ + "model": "gpt-realtime-whisper", + "audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}}, "input_audio_format": "pcm16", "input_audio_transcription": {"model": "gpt-realtime-whisper"}, }, @@ -993,16 +979,61 @@ async def test_transcription_sessions_encrypts_client_secret( assert decrypted is not None assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. - assert ( - captured_route_type["route_type"] - == "acreate_realtime_transcription_session" - ) + assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" + assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper" + assert captured_route_type["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", "language": "en", + } + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}}, +]) +async def test_transcription_sessions_route_the_allowed_model_over_a_nested_model_the_key_cannot_call( + proxy_app, + mock_route_request_transcription_sessions, + mock_add_litellm_data, + mock_pre_call_hook, + body, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", models=["gpt-4o-realtime-preview"] + ) + route: Final = AsyncMock(side_effect=mock_route_request_transcription_sessions) + + try: + client = TestClient(proxy_app) + with ( + patch("litellm.proxy.proxy_server.route_request", route), + patch("litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data), + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging, + ): + mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/transcription_sessions", + headers={"Authorization": "Bearer sk-test-master-key"}, + json=body, + ) + + assert response.status_code == 200, response.text + route.assert_awaited_once() + routed_session: Final = route.await_args.kwargs["data"]["transcription_session"] + assert { + routed_session.get("input_audio_transcription", {}).get("model"), + routed_session.get("audio", {}).get("input", {}).get("transcription", {}).get("model"), + } - {None} == {"gpt-4o-realtime-preview"} finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) def test_session_type_coerced_for_unknown_value(): - """An unrecognized session_type in the token falls back to 'realtime'.""" payload = _encode_realtime_token_payload( ephemeral_key="epk", model_id="gpt-4o", @@ -1011,12 +1042,327 @@ def test_session_type_coerced_for_unknown_value(): expires_at=None, session_type="INJECTED_TYPE", ) - # Force-deserialize and check the coercion that happens in proxy_realtime_calls. decoded = json.loads(payload) - session_type = decoded.get("session_type") or "realtime" - if session_type not in ("realtime", "transcription"): - session_type = "realtime" - assert session_type == "realtime" + assert decoded["session_type"] == "INJECTED_TYPE" + assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime" + assert _coerce_realtime_session_type(None) == "realtime" + for allowed_session_type in _ALLOWED_SESSION_TYPES: + assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type + + +@pytest.mark.asyncio +async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"), + ) + ), + ), + ) + + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"): + await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"), + ) + ), + ), + ) + + model, session_data, session_type = await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + assert model == "gpt-realtime-translate" + assert session_type == "translation" + assert session_data is not None + assert session_data["model"] == "gpt-realtime-translate" + assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + + +@pytest.mark.parametrize( + "path", + ( + "/openai/v1/realtime/client_secrets", + "/openai/v1/realtime/calls", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ), +) +def test_openai_prefixed_realtime_http_paths_stay_with_the_openai_pass_through( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import openai_proxy_route + + monkeypatch.setenv("LITELLM_DISABLE_LAZY_ROUTES", "true") + features = tuple(feature for feature in LAZY_FEATURES if feature.name in ("llm_passthrough", "realtime")) + app = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + scope = {"type": "http", "method": "POST", "path": path, "root_path": "", "headers": [], "query_string": b""} + + with TestClient(app): + resolved = next(route for route in app.router.routes if route.matches(scope)[0] is Match.FULL) + + assert getattr(resolved, "endpoint", None) is openai_proxy_route + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/client_secrets", + "/realtime/translations/client_secrets", + ], +) +def test_translation_client_secret_aliases_bind_token_family( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_client_secrets(*args, **kwargs) + + try: + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "acreate_realtime_translation_client_secret" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + decrypted = decrypt_value_helper( + response.json()["value"], + key="client_secret.value", + exception_type="debug", + ) + decoded = _decode_realtime_token_payload(decrypted or "") + assert decoded is not None + assert decoded["session_type"] == "translation" + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/calls", + "/realtime/translations/calls", + ], +) +def test_translation_calls_aliases_route_translation_session( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id="gpt-realtime-translate", + user_id="test-user", + team_id=None, + expires_at=int(time.time()) + 3600, + session_type="translation", + ) + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_realtime_calls(*args, **kwargs) + + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 201 + assert captured["route_type"] == "arealtime_translation_calls" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + + +@pytest.mark.parametrize( + ("disable_global_guardrails", "expected_status", "expected_body"), + [(True, 201, "v=0"), (False, 400, TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE)], +) +def test_translation_calls_honor_the_minting_keys_guardrail_opt_out( + proxy_app: FastAPI, + mock_route_request_client_secrets: object, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + monkeypatch: pytest.MonkeyPatch, + disable_global_guardrails: bool, + expected_status: int, + expected_body: str, +) -> None: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy import proxy_server + from litellm.types.guardrails import GuardrailEventHooks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-realtime-translate", + "litellm_params": {"model": "openai/gpt-realtime-translate", "api_key": "sk-upstream"}, + } + ] + ), + ) + monkeypatch.setattr( + litellm, + "callbacks", + [CustomGuardrail(guardrail_name="global-pre-call", event_hook=GuardrailEventHooks.pre_call, default_on=True)], + ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + metadata={"disable_global_guardrails": disable_global_guardrails}, + ) + client: Final = TestClient(proxy_app, raise_server_exceptions=False) + try: + with ( + patch( # test-quality-ok: endpoint test stands in for the upstream client_secrets call + "litellm.proxy.proxy_server.route_request", side_effect=mock_route_request_client_secrets + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment while minting + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger while minting + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + minted: Final = client.post( + "/v1/realtime/translations/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://api.openai.com/v1/realtime/translations/calls").mock( + return_value=httpx.Response(201, content=b"v=0\r\n", headers={"content-type": "application/sdp"}) + ) + answered: Final = client.post( + "/v1/realtime/translations/calls", + headers={"Authorization": f"Bearer {minted.json()['value']}"}, + content=b"v=0\r\n", + ) + + assert minted.status_code == 200 + assert answered.status_code == expected_status + assert expected_body in answered.text + + +@pytest.mark.parametrize( + "session_type,path", + [ + ("realtime", "/v1/realtime/translations/calls"), + ("translation", "/v1/realtime/calls"), + ], +) +def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path): + model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2" + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id=model, + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + session_type=session_type, + ) + ) + response = TestClient(proxy_app).post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert response.json()["error"] == "Token is not valid for this Realtime endpoint" @pytest.mark.asyncio @@ -1142,9 +1488,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim( return _inner() - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -1184,9 +1528,7 @@ async def test_transcription_sessions_wraps_route_exception( async def _raise_http(*args, **kwargs): raise HTTPException(status_code=403, detail="Model not allowed") - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") try: client = TestClient(proxy_app, raise_server_exceptions=False) with ( @@ -1287,3 +1629,260 @@ def test_transcription_sessions_rejection_answers_an_openai_typed_error( assert response.status_code == 400 assert (response.json()["error"]["type"], response.json()["error"]["param"]) == ("invalid_request_error", None) + + +_AZURE_REALTIME_DEPLOYMENTS: Final = ( + ("azure-realtime-alias", "azure/qa-realtime-deployment"), + ("azure-transcribe-alias", "azure/qa-whisper-deployment"), + ("azure-translate-alias", "azure/qa-translate-deployment"), +) + + +def _post_through_the_azure_deployments( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: object, + deployments: tuple[tuple[str, str], ...] = _AZURE_REALTIME_DEPLOYMENTS, +) -> tuple[httpx.Response, respx.Route]: + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_list: Final = [ # mutable-ok: Router takes a list of deployments + { + "model_name": alias, + "litellm_params": { + "model": deployment, + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + for alias, deployment in deployments + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600}) + ) + monkeypatch.setitem( + proxy_app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in deployments]), + ) + with ( + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response: Final = TestClient(proxy_app).post( + path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body + ) + return response, upstream + + +def _upstream_session_value(upstream: respx.Route, session_path: tuple[str, ...]) -> object: + return functools.reduce( + lambda node, key: node[key], session_path, json.loads(upstream.calls.last.request.content)["session"] + ) + + +@pytest.mark.parametrize( + ("path", "body", "upstream_model_path", "expected_model"), + ( + ( + "/v1/realtime/client_secrets", + {"session": {"type": "realtime", "model": "azure-realtime-alias"}}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + {"model": "azure-realtime-alias"}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}, + } + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), + ( + "/v1/realtime/translations/client_secrets", + {"model": "azure-translate-alias"}, + ("model",), + "qa-translate-deployment", + ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "azure-translate-alias", + "session": {"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}}, + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), + ), +) +def test_client_secrets_mint_a_secret_for_the_azure_deployment_model( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: dict[str, object], + upstream_model_path: tuple[str, ...], + expected_model: str, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body + ) + + assert response.status_code == 200, response.text + assert _upstream_session_value(upstream, upstream_model_path) == expected_model + + +@pytest.mark.parametrize( + ("body", "session_path", "expected"), + ( + ( + { + "session": { + "type": "realtime", + "model": "azure-realtime-alias", + "audio": {"input": {"turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}, "turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ({"session": {"type": "conversation", "model": "azure-realtime-alias"}}, ("type",), "realtime"), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias", "delay": "auto"}}}, + } + }, + ("audio", "input", "transcription", "delay"), + "auto", + ), + ), +) +def test_client_secrets_forward_every_session_value_the_caller_set( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, object], + session_path: tuple[str, ...], + expected: object, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + body, + ) + + assert response.status_code == 200, response.text + assert _upstream_session_value(upstream, session_path) == expected + + +def test_client_secrets_without_a_model_let_the_upstream_pick_its_default_through_a_wildcard_deployment( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "realtime"}}, + deployments=(("*", "openai/*"),), + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == {"type": "realtime"} + + +def test_client_secrets_send_the_deployment_model_in_the_flat_transcription_field( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "transcription", "input_audio_transcription": {"model": "azure-transcribe-alias"}}}, + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == { + "type": "transcription", + "input_audio_transcription": {"model": "qa-whisper-deployment"}, + } + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ("/v1/realtime/client_secrets", {"session": "not-a-session"}), + ("/v1/realtime/translations/client_secrets", {"session": {"audio": {"input": "not-an-input"}}}), + ("/v1/realtime/transcription_sessions", {"audio": {"input": {"transcription": "not-a-transcription"}}}), + ), +) +def test_realtime_session_routes_answer_400_for_a_malformed_body( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: Mapping[str, object], +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body + ) + + assert (response.status_code, response.json()["error"]["type"]) == (400, "invalid_request_error") + assert upstream.called is False diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index ef43fe56146..b7d0d9e62cd 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -2556,6 +2556,66 @@ class TestResponsesWSFirstFrameModelAuth: "response": {"input": "hi [guarded]", "tools": tools[:1]}, } + @pytest.mark.asyncio + @pytest.mark.parametrize("nested", [True, False]) + async def test_frame_guard_forwards_the_tool_choice_a_guardrail_rewrote(self, nested: bool): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): + return {**data, "tools": [], "tool_choice": "none"} + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() + ) + body = { + "input": "hi", + "tools": [{"type": "function", "name": "a"}], + "tool_choice": {"type": "function", "name": "a"}, + } + frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body} + + guarded = json.loads((await guard(json.dumps(frame))).frame) + + sent = guarded["response"] if nested else guarded + assert (sent["tools"], sent["tool_choice"]) == ([], "none") + + @pytest.mark.asyncio + @pytest.mark.parametrize("nested", [True, False]) + async def test_frame_guard_forwards_the_extra_body_tools_a_rewrite_guardrail_stripped(self, nested: bool): + from litellm.proxy.guardrails.guardrail_hooks.tool_permission import ToolPermissionGuardrail + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.types.guardrails import GuardrailEventHooks + + guardrail = ToolPermissionGuardrail( + guardrail_name="ws-rewrite", + rules=[{"id": "deny_shell", "tool_name": r"^shell$", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + + async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): + return await guardrail.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, cache=MagicMock(), data=data, call_type=call_type + ) + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() + ) + weather = {"type": "function", "name": "get_weather"} + body = {"input": "hi", "extra_body": {"tools": [weather, {"type": "function", "name": "shell"}]}} + frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body} + + guarded = json.loads((await guard(json.dumps(frame))).frame) + + sent = guarded["response"] if nested else guarded + assert sent["extra_body"] == {"tools": [weather]} + @pytest.mark.asyncio async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self): from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard @@ -2991,7 +3051,7 @@ class TestResponsesWSFirstFrameModelAuth: assert dict(guarded.pii_tokens) == {"": "alice@example.com"} @pytest.mark.asyncio - @pytest.mark.parametrize("field", ["input", "instructions", "tools"]) + @pytest.mark.parametrize("field", ["input", "instructions", "tools", "tool_choice", "extra_body"]) async def test_frame_guard_rejects_and_bills_a_guarded_field_beside_a_nested_response(self, field: str): from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index a87ac7fa7c0..9267a951459 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -588,3 +588,10 @@ async def test_group_reserves_on_its_current_deployments_after_another_vendors_d by_own_name: Final = litellm.token_counter(model="prod", messages=_MULTILINGUAL_MESSAGES) assert openai_only != by_own_name assert (await _input_tokens(router, "prod"))["prod"] == by_own_name + + +@pytest.mark.parametrize("window", [{}, {"budget_duration": "1d"}, {"reset_at": "2026-11-03T00:00:00Z"}]) +def test_fixed_budget_window_requires_duration_and_reset_boundary(window: object) -> None: + from litellm.proxy.spend_tracking.budget_reservation import fixed_budget_window + + assert fixed_budget_window(window) is None diff --git a/tests/unit/proxy/spend_tracking/test_cost_finalization.py b/tests/unit/proxy/spend_tracking/test_cost_finalization.py index c9515c1232e..ef70a193280 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_finalization.py +++ b/tests/unit/proxy/spend_tracking/test_cost_finalization.py @@ -1,8 +1,9 @@ import asyncio from datetime import datetime, timezone -from typing import Final +from typing import Final, Literal import pytest +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging @@ -91,12 +92,6 @@ def response(cost: float = 0.5) -> ModelResponse: async def test_header_and_logging_share_one_allowance_allocation( logging_obj: Logging, used: int, expected: float ) -> None: - import json - - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings - counter: Final = Counter(used) result: Final = response() costs: Final = await asyncio.gather(*(finalize_response_cost(logging_obj, result, counter, NOW) for _ in range(3))) @@ -115,24 +110,6 @@ async def test_header_and_logging_share_one_allowance_allocation( assert payload["cost_finalized"] is True assert payload["cost_breakdown"] == costs[0].logging_breakdown() - committed: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - for write in writes: - committed.put_nowait(write) - - meter: Final = DurableUsageMeter(persist, UsageCaptureSettings()) - assert logging_obj.prepared_cost is not None - prepared_usage: Final = logging_obj.prepared_cost.usage.model_dump() - result.usage = Usage(prompt_tokens=999) - capture_logging_usage(meter, logging_obj, result, NOW, NOW) - await meter.close() - committed.get_nowait() - event: Final = json.loads(committed.get_nowait().envelope) - assert event["finalized_cost"]["response_cost"] == pytest.approx(costs[0].amount) - assert event["allocation"]["discount"] == pytest.approx(costs[0].allowance_discount) - assert event["usage"] == prepared_usage - assert counter.calls == 1 @pytest.mark.asyncio @@ -450,18 +427,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails( disable_spend_updates: bool, ) -> None: from unittest.mock import create_autospec + import anyio - from fastapi import Request, Response - from litellm.proxy._types import ProxyException - from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyException from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.proxy_server import ProxyConfig - from litellm.proxy import proxy_server from litellm.proxy.utils import ProxyLogging scope: Final = anyio.CancelScope() @@ -598,6 +574,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage( logging_obj: Logging, monkeypatch: pytest.MonkeyPatch ) -> None: import httpx + from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.utils import ProxyLogging @@ -633,3 +610,222 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage( assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5) assert payload["response_cost"] == 0.5 ProxyLogging._callback_capabilities_cache.clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"]) +@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"]) +async def test_finalized_header_distinguishes_missing_prices_from_free_models( + monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float +) -> None: + model: Final = "claude-header-pricing-test" + monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount}) + monkeypatch.setitem( + litellm.model_cost, + f"azure/{model}", + { + "litellm_provider": "azure", + "mode": "chat", + "regional_endpoint_uplift_multiplier": 1.0, + **({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}), + }, + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="pricing-header", + function_id="pricing-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="azure", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None + assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount)) + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "source", + ["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough"], +) +@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1]) +async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges( + source: Literal[ + "unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough" + ], + guardrail_cost: float, +) -> None: + model: Final = "unmapped-header-cost" + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="adelete_responses" if source == "non-inference" else "acompletion", + start_time=NOW, + litellm_call_id="zero-cost-header", + function_id="zero-cost-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"} + match source: + case "provider": + logging._provider_reported_cost = 0.0 + case "provider-header": + result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0} + case "per-token": + logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + case "per-second": + logging.custom_cost_per_second = 0.0 + case "passthrough": + logging.model_call_details["passthrough_logging_payload"] = {"cost_per_request": 0.0} + case _: + pass + finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost) + assert finalized is not None + assert finalized.amount == guardrail_cost + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers.get("x-litellm-response-cost") == ( + None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost) + ) + + +@pytest.mark.asyncio +async def test_free_model_priced_only_per_request_keeps_the_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "gpt-free-per-request-header" + monkeypatch.setitem( + litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", "input_cost_per_request": 0.0} + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="per-request-header", + function_id="per-request-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None and finalized.amount == 0.0 + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers["x-litellm-response-cost"] == "0.0" + + +@pytest.mark.asyncio +async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "gpt-free-off-peak-header" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + }, + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="off-peak-header", + function_id="off-peak-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + result._hidden_params["response_cost"] = 0.0 + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None and finalized.amount == 0.0 + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + assert headers["x-litellm-response-cost"] == "0.0" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"]) +@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"]) +async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices( + monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None +) -> None: + from litellm.litellm_core_utils.fallback_generalizations import ( + get_fallback_generalization_rules, + set_fallback_generalizations, + ) + + model: Final = "gpt-rule-header-model" + previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules()) + if mapped: + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"}) + try: + set_fallback_generalizations( + [ + { + "name": "header-pricing", + "pattern": model, + "fill_missing_for_providers": ["openai"], + "model_info": { + "mode": "chat", + **( + {"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {} + ), + }, + } + ] + ) + logging: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="acompletion", + start_time=NOW, + litellm_call_id="rule-header", + function_id="rule-header", + ) + logging.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + finalized: Final = await finalize_response_cost(logging, result, None, NOW) + assert finalized is not None + assert finalized.amount == pytest.approx((price or 0.0) * 15) + headers: Final = TypeAdapter(dict[str, str]).validate_python( + ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), + response_cost=finalized.amount, + litellm_logging_obj=logging, + ) + ) + assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount)) + finally: + set_fallback_generalizations(previous_rules) diff --git a/tests/unit/proxy/spend_tracking/test_session_page.py b/tests/unit/proxy/spend_tracking/test_session_page.py new file mode 100644 index 00000000000..a86990bddbb --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_page.py @@ -0,0 +1,327 @@ +import itertools +import random +import re +from collections.abc import Sequence +from typing import Final, TypeAlias + +import pytest + +from litellm.proxy.spend_tracking.session_page import ( + Session, + _parse_session_cursor, + _Read, + _ReadRow, + _Request, + _session_page, + read_session_page, +) + +_Row: TypeAlias = tuple[int, str, str] # (startTime, session key, api_key) +_Ranked: TypeAlias = tuple[int, str, str] # (last activity, session key, api_key) + + +def _session_of(row: _Row) -> tuple[str, str]: + return row[1], row[2] + + +def _ranked(rows: Sequence[_Row], request: _Request) -> tuple[_Ranked, ...]: + """The sessions in ``rows`` in page order, after the request's cursor or offset.""" + by_session: Final = itertools.groupby(sorted(rows, key=_session_of), key=_session_of) + ranked: Final = tuple( + sorted( + ((max(start for start, _, _ in group), key, api_key) for (key, api_key), group in by_session), + reverse=request.sort_desc, + ) + ) + if request.cursor is None: + return ranked[request.offset :] + cursor: Final = (int(request.cursor[0]), request.cursor[1], request.cursor[2]) + return tuple(session for session in ranked if (session < cursor if request.sort_desc else session > cursor)) + + +class _Reads: + """Reads like the page query, whose LIMIT cuts tied rows lowest key first, the opposite of page order. It fails a + read no larger than the one before, since only growing reads end the loop.""" + + def __init__(self, rows: Sequence[_Row], request: _Request, every_row: bool = False) -> None: + self.newest_first: Final = tuple(sorted(rows, key=lambda row: (-row[0], row[1], row[2]))) + self.request: Final = request + self.every_row: Final = every_row + self.budgets: tuple[int | None, ...] = () + + async def __call__(self, budget: int | None) -> _Read: + assert not self.budgets or (self.budgets[-1] is not None and (budget is None or budget > self.budgets[-1])), ( + self.budgets, + budget, + ) + self.budgets = (*self.budgets, budget) + taken: Final = self.newest_first if budget is None or self.every_row else self.newest_first[:budget] + page: Final = _ranked(taken, self.request)[: self.request.page_limit + 1] if self.request.page_limit > 0 else () + return _Read( + candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page), + last_settled=bool(page) and page[-1][0] > taken[-1][0], + rows_read=len(taken), + session_count=len(frozenset(_session_of(row) for row in taken)), + saw_every_row=budget is None or self.every_row or len(taken) < budget, + ) + + +def _session_rows(rng: random.Random, index: int, clock: int) -> tuple[_Row, ...]: + start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40)) + return tuple((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length)) + + +def _sessions(rng: random.Random) -> tuple[_Row, ...]: + """Sessions of one to forty rows over a short clock, so many rows share a time.""" + clock: Final = rng.choice((3, 40, 400)) + return tuple( + itertools.chain.from_iterable(_session_rows(rng, index, clock) for index in range(rng.randrange(1, 120))) + ) + + +def _random_cursor(rng: random.Random, ranked: Sequence[_Ranked]) -> tuple[str, str, str]: + at, key, api_key = ranked[rng.randrange(len(ranked))] + return f"{at:06d}", key, api_key + + +def _request(rng: random.Random, rows: Sequence[_Row]) -> _Request: + cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8 + mode: Final = rng.choice(("first", "offset", "cursor")) + full: Final = _ranked(rows, _Request(None, 0, 0, sort_desc, None, cap)) + cursor: Final = _random_cursor(rng, full) if mode == "cursor" and full else None + offset: Final = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0 + return _Request(cursor, offset, min(page_size, cap - offset), sort_desc, None, cap) + + +@pytest.mark.parametrize("seed", range(60)) +@pytest.mark.asyncio +async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed: int) -> None: + rng: Final = random.Random(seed) + rows: Final = _sessions(rng) + for request in tuple(_request(rng, rows) for _ in range(12)): + grouped = await _session_page(_Reads(rows, request, every_row=True), request) + assert await _session_page(_Reads(rows, request), request) == grouped, request + + +@pytest.mark.parametrize( + "rows_per_session, expected_budgets", + [ + pytest.param(1, (102,), id="one-read-holds-the-cap"), + pytest.param(2, (102, 253), id="sized-by-the-first-reads-density"), + pytest.param(50, (102, 4293, 8586), id="doubles-when-the-density-misleads"), + pytest.param(500, (102, 12878, 62533), id="re-sized-by-each-reads-density"), + ], +) +@pytest.mark.asyncio +async def test_reads_grow_by_the_last_reads_rows_per_session_until_one_settles_the_page( + rows_per_session: int, expected_budgets: tuple[int, ...] +) -> None: + """150 sessions a row per second apart. A cap of 100 starts at 100 + 2 rows.""" + rows: Final = tuple((index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == expected_budgets + + +@pytest.mark.asyncio +async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted() -> None: + """Thirty one-row sessions share one time and the first read takes 12 of them. The page's sixth session sits at the + read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on.""" + rows: Final = tuple((0, f"s{index:02d}", "k") for index in range(30)) + request: Final = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, status=None, cap=10) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert tuple(session.key for session in page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.budgets == (12, 24, 48) + + +@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))]) +@pytest.mark.asyncio +async def test_the_total_is_capped_only_past_the_cap(session_count: int, expected_total: tuple[int, bool]) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(session_count)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + page: Final = await _session_page(_Reads(rows, request), request) + assert (page.total, page.total_is_capped) == expected_total + + +@pytest.mark.asyncio +async def test_a_page_number_near_the_bigint_limit_reads_once() -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + offset: Final = (9_300_000_000_000_000 - 1) * 1000 + request: Final = _Request( + cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, status=None, cap=100 + ) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert reads.budgets == (102,) + assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True) + + +@pytest.mark.parametrize( + "sort_desc, status", + [pytest.param(False, None, id="oldest-first"), pytest.param(True, "failure", id="filtered-by-outcome")], +) +@pytest.mark.asyncio +async def test_a_page_that_needs_every_row_reads_them_once(sort_desc: bool, status: str | None) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=sort_desc, status=status, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == (None,) + + +def test_parse_session_cursor() -> None: + assert _parse_session_cursor(None) is None + assert _parse_session_cursor("no-delimiter") is None + assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None + assert _parse_session_cursor("|hashed-key|sess-1") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( + "2026-08-29 10:00:00", + "sess-1", + "hashed-key", + ) + assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( + "2026-08-29 10:00:00.123", + "sess|with|pipes", + "hashed-key", + ) + + +def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]: + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found: Final = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +class _TableDb: + """A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row.""" + + def __init__(self, rows: int, sessions: int) -> None: + self.rows: Final = rows + self.sessions: Final = sessions + self.calls: tuple[tuple[str, tuple[object, ...]], ...] = () + + async def query_raw(self, query: str, *args: object) -> tuple[_ReadRow, ...]: + self.calls = (*self.calls, (query, args)) + budget: Final = re.search(r"DESC LIMIT (\d+)", query) + rows_read: Final = min(int(budget.group(1)), self.rows) if budget else self.rows + return ( + _ReadRow( + session_key=None, + api_key=None, + last_activity=None, + settled=None, + rows_read=rows_read, + session_count=self.sessions, + ), + ) + + +def _page_bounds(sql: str) -> tuple[int, int]: + """The LIMIT and OFFSET of the read's sessions.""" + found: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s*\) AS page", sql) + assert found, sql + return int(found.group(1)), int(found.group(2) or 0) + + +def _read_size(sql: str) -> int | None: + found: Final = re.search(r"DESC LIMIT (\d+)", sql) + return int(found.group(1)) if found else None + + +@pytest.mark.asyncio +async def test_a_read_that_fills_its_size_is_not_taken_for_every_row() -> None: + """A read that took every row it asked for may have stopped short, so its 50 sessions are no total: the reads + grow until one takes fewer rows than it asked for.""" + db: Final = _TableDb(rows=9_000, sessions=50) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=1, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(_read_size(sql) for sql, _ in db.calls) == (102, 258, 652, 1647, 4159, 10502) + assert (page.total, page.total_is_capped) == (50, False) + + +@pytest.mark.parametrize("page_number", [pytest.param(5, id="at-the-cap"), pytest.param(6, id="past-the-cap")]) +@pytest.mark.asyncio +async def test_a_page_starting_at_or_past_the_cap_reads_once_and_shows_no_session(page_number: int) -> None: + db: Final = _TableDb(rows=1_000, sessions=150) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=page_number, + page_size=25, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert len(db.calls) == 1, "the first read already holds more sessions than the cap" + assert _page_bounds(db.calls[0][0])[0] == 0 + assert (page.total, page.total_is_capped) == (100, True) + + +@pytest.mark.asyncio +async def test_the_read_binds_the_filters_and_the_cursor_to_their_placeholders() -> None: + db: Final = _TableDb(rows=50, sessions=40) + where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a") + for page, page_size, session_cursor in ((1, 3, "2026-10-28 09:00:00|key-a|sess|9"), (3, 1, None)): + await read_session_page( + db, + where_clause=where, + params=params, + page=page, + page_size=page_size, + status=None, + sort_desc=True, + session_cursor=session_cursor, + cap=100, + ) + (cursor_sql, cursor_args), (offset_sql, _) = db.calls + for sql, args in db.calls: + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)), ( + "every argument needs a placeholder and every placeholder an argument" + ) + assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params + assert _bound( + cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)" + ) == ("2026-10-28 09:00:00", "sess|9", "key-a") + assert _page_bounds(cursor_sql) == (4, 0) + assert _page_bounds(offset_sql) == (2, 2) + + +@pytest.mark.asyncio +async def test_a_status_filtered_read_binds_the_status_between_the_filters_and_the_cursor() -> None: + db: Final = _TableDb(rows=50, sessions=40) + await read_session_page( + db, + where_clause="model = $1", + params=("gpt-4o",), + status="failure", + page=1, + page_size=3, + sort_desc=True, + session_cursor="2026-10-28 09:00:00|key-a|sess-9", + cap=100, + ) + ((sql, args),) = db.calls + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)) + assert _bound(sql, args, r"model = \$(\d+)") == ("gpt-4o",) + assert _bound(sql, args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",) + assert _bound(sql, args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)") == ( + "2026-10-28 09:00:00", + "sess-9", + "key-a", + ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_event.py b/tests/unit/proxy/spend_tracking/test_spend_event.py index ff449235582..e1912d36d91 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_event.py +++ b/tests/unit/proxy/spend_tracking/test_spend_event.py @@ -194,9 +194,11 @@ def test_undecodable_line_is_an_error_value(): assert isinstance(decode_spend_event(b"not json\n"), SpendEventDecodeError) -def test_batch_retrieves_stay_in_process(): - assert is_offloadable_success(_response()) is True - assert is_offloadable_success(None) is True +def test_batch_retrieves_and_video_status_polls_stay_in_process(): + assert is_offloadable_success(_response(), "acompletion") is True + assert is_offloadable_success(None, None) is True + assert is_offloadable_success(None, "avideo_generation") is True + assert is_offloadable_success(None, "avideo_status") is False assert ( is_offloadable_success( LiteLLMBatch( @@ -207,7 +209,8 @@ def test_batch_retrieves_stay_in_process(): input_file_id="f", object="batch", status="completed", - ) + ), + "aretrieve_batch", ) is False ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 6598b94e60f..e119acd5ffc 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5,7 +5,7 @@ import hashlib import json import re import sqlite3 -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import timezone from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -20,6 +20,11 @@ import litellm.proxy.proxy_server as ps from litellm.proxy.auth.authorization import OwnedRows from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup, load_permitted_log_team_ids from litellm.proxy.auth.route_checks import RouteChecks +from tests.unit.proxy.db.fake_prisma_engine import ( + ReadTargetOf, + statement_timeout_error, + statement_timeout_reads, +) def _default_date_range(): @@ -258,6 +263,7 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No return [row for row in ordered[skip : skip + page_size]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() self.db.litellm_spendlogs = self.db @@ -317,6 +323,7 @@ async def test_is_admin_view_safe_exception(): async def test_can_team_member_view_log_none_team_id(): # team_id=None should immediately return False class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -338,6 +345,7 @@ async def test_can_team_member_view_log_none_team_id(): async def test_can_team_member_view_log_team_not_found(monkeypatch): # Non-existent team should return False class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -377,6 +385,7 @@ async def test_can_team_member_view_log_not_admin(monkeypatch): } class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -415,6 +424,7 @@ async def test_can_team_member_view_log_admin(monkeypatch): } class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -480,6 +490,7 @@ def _make_owner_lookup_prisma(rows): return _emulate_spend_log_owner_lookup(rows, sql_query, params) class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -985,6 +996,7 @@ async def test_ui_view_spend_logs_sort_by_and_sort_order( return [row for row in sorted_logs[skip : skip + page_size]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -1046,6 +1058,7 @@ async def test_ui_view_spend_logs_sort_validation_errors(client, monkeypatch, so return 0 class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -1122,6 +1135,7 @@ async def test_ui_view_spend_logs_sort_by_request_duration_ms(client, monkeypatc return [row for row in sorted_logs[skip : skip + page_size]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -1218,6 +1232,7 @@ async def test_ui_view_spend_logs_sort_by_model(client, monkeypatch, sort_order, return [row for row in sorted_logs[skip : skip + page_size]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -1330,6 +1345,7 @@ async def test_ui_view_spend_logs_sort_by_ttft_ms(client, monkeypatch): return [{k: v for k, v in row.items() if k != "_ttft_ms"} for row in sorted_logs[skip : skip + page_size]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -2135,6 +2151,7 @@ async def test_ui_view_session_spend_logs_pagination(client, monkeypatch): return [mock_spend_logs[0]] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() self.db.litellm_spendlogs = self.db @@ -2187,6 +2204,7 @@ async def test_ui_view_session_spend_logs_rehydrates_metadata_jsonb_text(client, return [dict(raw_row)] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() self.db.litellm_spendlogs = self.db @@ -2235,6 +2253,7 @@ async def test_ui_view_session_spend_logs_includes_permitted_team_logs(client, m ] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() self.db.litellm_spendlogs = self.db @@ -2712,6 +2731,7 @@ async def test_ui_view_request_response_rejects_foreign_row_inserted_after_owner ] class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -2743,6 +2763,7 @@ async def test_ui_view_request_response_custom_logger_denies_foreign_payload_own return [] class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4056,6 +4077,7 @@ async def test_view_spend_logs_summarize_parameter(client, monkeypatch): ] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4151,6 +4173,7 @@ async def test_view_spend_logs_bounds_row_count(client, monkeypatch): return [{}] * min(kwargs.get("take", 0), self.available_rows) class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4274,6 +4297,7 @@ def _tag_query_prisma_client(assert_query_raw): return assert_query_raw(sql_query, params) class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4493,6 +4517,7 @@ async def test_view_spend_logs_with_date_range_summarized(client, monkeypatch): return mock_summarized_response class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4555,6 +4580,7 @@ async def test_view_spend_logs_summarize_groups_by_day_in_sql(client, monkeypatc return mock_rows class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4620,6 +4646,7 @@ async def test_view_spend_logs_summarize_empty_rows(client, monkeypatch): return [] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4659,6 +4686,7 @@ async def test_view_spend_logs_summarize_unhashed_api_key_without_padding(client return mock_rows class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MockDB() @@ -4973,6 +5001,7 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5038,6 +5067,7 @@ async def test_build_ui_spend_logs_response_caps_session_models(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5102,6 +5132,7 @@ async def test_build_ui_spend_logs_response_key_split_session_gets_per_key_aggre }, ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5155,6 +5186,7 @@ async def test_build_ui_spend_logs_response_empty_api_key_keeps_session_aggregat } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5210,6 +5242,7 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5292,6 +5325,7 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_tokens(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5372,6 +5406,7 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_duration(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5397,6 +5432,35 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_duration(): ) +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_keeps_the_page_when_postgres_cancels_the_session_totals_read(): + """The page already fetched is returned without session totals, like any other failed enrichment read.""" + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, + ) + + dict_rows = [ + {"request_id": "req-1", "session_id": "sess-slow", "call_type": "completion", "api_key": "k"}, + {"request_id": "req-2", "session_id": "sess-slow", "call_type": "completion", "api_key": "k"}, + ] + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(side_effect=statement_timeout_error()) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=2, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + assert [row["request_id"] for row in result["data"]] == ["req-1", "req-2"] + assert all(row["session_total_count"] == 1 for row in result["data"]) + + @pytest.mark.asyncio async def test_build_ui_spend_logs_response_session_cache_hit_count(): """ @@ -5430,6 +5494,7 @@ async def test_build_ui_spend_logs_response_session_cache_hit_count(): } ] ) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) result = await _build_ui_spend_logs_response( prisma_client=mock_prisma, @@ -5476,6 +5541,7 @@ async def test_can_team_member_view_log_with_spend_logs_permission(monkeypatch): } class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -5512,6 +5578,7 @@ async def test_can_team_member_view_log_without_spend_logs_permission(monkeypatc } class MockPrisma: + read_db = property(lambda self: ReadTargetOf(self.db)) class DB: class TeamTable: async def find_unique(self, where: dict): @@ -6201,6 +6268,7 @@ async def test_ui_view_spend_logs_rehydrates_metadata_jsonb_text(client, monkeyp return [{**raw_row, "total_count": 1}] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -6285,6 +6353,7 @@ async def test_ui_view_spend_logs_metadata_invalid_json_falls_back_to_empty_dict return [{**raw_row, "total_count": 1}] class MockPrismaClient: + read_db = property(lambda self: ReadTargetOf(self.db)) def __init__(self): self.db = MagicMock() self.db.litellm_spendlogs = MagicMock() @@ -6628,6 +6697,7 @@ _SCOPED_SPEND_REPORT_PATHS = [ def _spend_report_mock_prisma(query_raw_returns=None, team_rows=None, user_row=None): pc = MagicMock() pc.db.query_raw = AsyncMock(return_value=query_raw_returns if query_raw_returns is not None else []) + statement_timeout_reads(pc, pc.db.query_raw) pc.db.litellm_teamtable.find_many = AsyncMock(return_value=team_rows if team_rows is not None else []) pc.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) return pc @@ -7151,37 +7221,19 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) -def test_parse_session_cursor(): - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _parse_session_cursor, - ) - - assert _parse_session_cursor(None) is None - assert _parse_session_cursor("no-delimiter") is None - assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None - assert _parse_session_cursor("|hashed-key|sess-1") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( - "2026-08-29 10:00:00", - "sess-1", - "hashed-key", - ) - assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( - "2026-08-29 10:00:00.123", - "sess|with|pipes", - "hashed-key", - ) - - SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key" -def _session_grouped_mock_prisma(session_page_rows, session_total, representative_rows): +def _session_grouped_mock_prisma( + session_page_rows: Sequence[Mapping[str, object]], + session_total: int, + representative_rows: Sequence[Mapping[str, object]], +) -> MagicMock: """Mock prisma dispatching the raw queries the keyset grouped path emits.""" - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": session_total}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return _session_read_rows(session_page_rows, session_total) if "DISTINCT ON" in sql_query: return representative_rows if "COALESCE(SUM(spend)" in sql_query: @@ -7199,11 +7251,12 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ "session_models": ["gpt-4o"], } ] - return session_page_rows + raise AssertionError(f"unexpected query:\n{sql_query}") mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) return mock_prisma @@ -7228,27 +7281,38 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} -def _session_grouped_paginating_prisma(sessions, counted_total=None): - """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" +def _session_read_rows( + page_rows: Sequence[Mapping[str, object]], session_count: int +) -> tuple[Mapping[str, object], ...]: + """The read query's rows for ``page_rows``, as if the read held every matching row.""" + summary: Final = {"rows_read": 0, "session_count": session_count} + if not page_rows: + return ({"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary},) + return tuple({**row, "settled": True, **summary} for row in page_rows) - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] + +def _session_grouped_paginating_prisma(sessions: Sequence[tuple[str, str]]) -> MagicMock: + """Mock prisma serving the grouped page read out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" + + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: if "DISTINCT ON" in sql_query: return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] if "COALESCE(SUM(spend)" in sql_query: return [] - bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) - limit = params[int(bounds.group(1)) - 1] - offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 - return [ - _session_page_row(session_key, last_activity) - for session_key, last_activity in sessions[offset : offset + limit] - ] + bounds: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s+\) AS page", sql_query) + limit, offset = int(bounds.group(1)), int(bounds.group(2) or 0) + return _session_read_rows( + [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ], + len(sessions), + ) mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) return mock_prisma @@ -7296,20 +7360,15 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert data["data"][1]["session_total_count"] == 1 emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list] + assert len(emitted) == 3, "one read picks the page and counts it, then the representatives and their totals" page_query_sql = emitted[0][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql assert "OFFSET" not in page_query_sql - assert "HAVING" not in page_query_sql - assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 3 " in page_query_sql, "the read fetches page_size + 1 sessions to detect has_more" - count_sql = emitted[1][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - - rep_call = emitted[2] + rep_call: Final = emitted[1] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( - f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL ) in rep_call[0] assert rep_call[-2] == ["sess-1", "req-solo"] assert rep_call[-1] == ["hashed-key", "hashed-key"] @@ -7319,7 +7378,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch): - """A session_cursor becomes a HAVING keyset predicate instead of an OFFSET.""" + """A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET.""" page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")] reps = [_session_representative_row("req-2", "sess-2")] mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps) @@ -7352,7 +7411,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat page_query_call = mock_prisma.db.query_raw.await_args_list[0] page_query_sql = page_query_call.args[0] - assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql + assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql assert "OFFSET" not in page_query_sql cursor_index = page_query_call.args.index("2026-08-29 09:00:00") assert page_query_call.args[cursor_index : cursor_index + 3] == ( @@ -7403,9 +7462,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor( @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): - """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + """A page that runs out of sessions has read every matching row, so that one read gives the total.""" sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) - mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + mock_prisma: Final = _session_grouped_paginating_prisma(sessions) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7429,10 +7488,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie ) assert response.status_code == 200, response.text data = response.json() - assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total"] == 10 assert data["total_is_capped"] is False assert data["total_pages"] == 1 assert len(data["data"]) == 10 + reads: Final = tuple(call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]) + assert len(reads) == 1, "a page that ends the list is not read again for its total" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7517,7 +7578,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty( assert past_cap_data["data"] == [] assert past_cap_data["has_more"] is False assert past_cap_data["total"] == cap - assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7572,6 +7633,7 @@ async def test_ui_view_spend_logs_group_by_session_offset_for_non_starttime_sort return [] mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7622,6 +7684,7 @@ async def test_ui_view_spend_logs_search_returns_flat_rows_when_grouping_by_sess mock_prisma = MagicMock() mock_prisma.db = MagicMock() mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + statement_timeout_reads(mock_prisma, mock_prisma.db.query_raw) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" @@ -7820,14 +7883,14 @@ def test_session_representative_uses_completed_agent_outcome(parent_status, chil ) result = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert result == ("parent", expected) connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'") fallback = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert fallback == ("llm", "success") diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 93fae093340..aaf4bc5438c 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -6,7 +6,9 @@ GitHub Issue: #17487 """ import datetime +from collections.abc import Mapping, Sequence from datetime import timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -16,6 +18,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_spend_by_team, get_spend_by_team_and_customer, ) +from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_error, statement_timeout_reads @pytest.mark.asyncio @@ -30,10 +33,8 @@ async def test_spend_query_uses_timestamp_filtering(): """ # Mock prisma client mock_prisma = MagicMock() - mock_db = MagicMock() mock_query_raw = AsyncMock(return_value=[]) - mock_db.query_raw = mock_query_raw - mock_prisma.db = mock_db + statement_timeout_reads(mock_prisma, mock_query_raw) # Use timezone-aware datetime objects start_date = datetime.datetime(2024, 1, 1, tzinfo=timezone.utc) @@ -168,7 +169,7 @@ async def test_spend_logs_ui_wraps_params_in_at_time_zone_utc(monkeypatch): mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(return_value=[]) + statement_timeout_reads(mock_prisma, AsyncMock(return_value=[])) mock_prisma.db.litellm_spendlogs = MagicMock() mock_prisma.db.litellm_spendlogs.count = AsyncMock(return_value=0) @@ -195,22 +196,27 @@ async def test_spend_logs_ui_wraps_params_in_at_time_zone_utc(monkeypatch): user_api_key_dict=auth, ) - assert mock_prisma.db.query_raw.called, "query_raw should have been called" - sql = mock_prisma.db.query_raw.call_args[0][0] + assert _spend_log_reads(mock_prisma).called, "query_raw should have been called" + sql = _spend_log_reads(mock_prisma).call_args[0][0] assert sql.count("AT TIME ZONE 'UTC'") >= 2, ( f"/spend/logs/ui must wrap both `startTime` bounds with `AT TIME ZONE 'UTC'`. SQL was:\n{sql}" ) +def _spend_log_reads(mock_prisma: MagicMock) -> AsyncMock: + """The raw reads ``/spend/logs/ui`` ran under the statement timeout.""" + return mock_prisma.read_db.tx.return_value.query_raw + + def _make_ui_spend_logs_mock(count_total, page_rows): """ - Build a prisma mock whose first `query_raw` (the bounded count) returns - `count_total` and whose second `query_raw` (the page data) returns + Build a prisma mock whose first statement-timeout read (the bounded count) returns + `count_total` and whose second read (the page data) returns `page_rows`. """ mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=[[{"total_count": count_total}], page_rows]) + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=[[{"total_count": count_total}], page_rows])) mock_prisma.db.litellm_spendlogs = MagicMock() mock_prisma.db.litellm_spendlogs.count = AsyncMock(return_value=0) return mock_prisma @@ -260,7 +266,7 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): mock_prisma.db.litellm_spendlogs.count.assert_not_called() - count_call = mock_prisma.db.query_raw.call_args_list[0] + count_call = _spend_log_reads(mock_prisma).call_args_list[0] count_sql = count_call[0][0] assert "COUNT(*) OVER ()" not in count_sql assert "LIMIT" in count_sql and "FROM (" in count_sql, ( @@ -268,7 +274,7 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): ) assert count_call[0][-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1, "the bounded count must probe at most cap+1 rows" - page_sql = mock_prisma.db.query_raw.call_args_list[1][0][0] + page_sql = _spend_log_reads(mock_prisma).call_args_list[1][0][0] assert "COUNT(*) OVER ()" not in page_sql, ( f"the page query must not carry a window count that forces a full-window scan. SQL was:\n{page_sql}" ) @@ -342,7 +348,7 @@ async def test_spend_logs_ui_empty_page_reports_zero_total(monkeypatch): # page. mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=[[{"total_count": 0}], []]) + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=[[{"total_count": 0}], []])) mock_prisma.db.litellm_spendlogs = MagicMock() mock_prisma.db.litellm_spendlogs.count = AsyncMock(return_value=0) @@ -391,7 +397,7 @@ async def test_spend_logs_ui_out_of_range_page_keeps_total(monkeypatch): # out-of-range page (empty). mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=[[{"total_count": 7}], []]) + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=[[{"total_count": 7}], []])) mock_prisma.db.litellm_spendlogs = MagicMock() mock_prisma.db.litellm_spendlogs.count = AsyncMock(return_value=0) @@ -432,9 +438,8 @@ async def test_get_spend_by_team_binds_optional_team_filter(): guard for LIT-4125 (the team query previously carried no team_id predicate). """ mock_prisma = MagicMock() - mock_prisma.db = MagicMock() mock_query_raw = AsyncMock(return_value=[]) - mock_prisma.db.query_raw = mock_query_raw + statement_timeout_reads(mock_prisma, mock_query_raw) start_date = datetime.datetime(2024, 1, 1, tzinfo=timezone.utc) end_date = datetime.datetime(2024, 1, 31, tzinfo=timezone.utc) @@ -484,7 +489,7 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(return_value=[]) + statement_timeout_reads(mock_prisma, AsyncMock(return_value=[])) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) @@ -499,29 +504,19 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): customer_id=None, ) - assert mock_prisma.db.query_raw.called, "query_raw should have been called" - sql = mock_prisma.db.query_raw.call_args[0][0] - params = mock_prisma.db.query_raw.call_args[0][1:] + assert _spend_log_reads(mock_prisma).called, "query_raw should have been called" + sql = _spend_log_reads(mock_prisma).call_args[0][0] + params = _spend_log_reads(mock_prisma).call_args[0][1:] assert "team_x" in params, "team_id must be forwarded into the DB query params" assert "sl.team_id = $3" in sql, f"team query must filter on team_id. SQL was:\n{sql}" @pytest.mark.asyncio async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): - """ - With group_by_session=true, /spend/logs/ui must page and count SESSIONS, - not raw calls: the page is one representative row per session (DISTINCT - ON the session group key, preferring non-MCP calls, newest first), the - bounded count counts groups, and on the default startTime sort the page is - selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the - UI collapses a server page of N calls into fewer visible rows while the - footer still claims N (issue #38060). - """ + """With group_by_session=true, /spend/logs/ui pages and counts sessions, not calls, one representative row per + session, so the footer's total matches the rows shown (issue #38060).""" from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - SPEND_LOGS_PAGINATION_COUNT_CAP, - ui_view_spend_logs, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ @@ -533,16 +528,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): {"request_id": "req-2", "api_key": "k", "metadata": "{}", "session_id": None}, ] - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 60}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows] if "AS session_representatives" in sql_query: return representative_rows - return session_rows + return [] mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock(side_effect=mock_query_raw) + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=mock_query_raw)) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") @@ -565,22 +560,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): user_api_key_dict=auth, group_by_session=True, session_cursor=None, + status_filter=None, ) - emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list] + emitted = [call[0] for call in _spend_log_reads(mock_prisma).call_args_list] page_sql = emitted[0][0] - assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" + assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected" - assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 51 " in page_sql, "the page read fetches page_size + 1 sessions to detect has_more" - count_call = emitted[1] - count_sql = count_call[0] - assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" - assert "COUNT(*) OVER ()" not in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - - rep_sql = emitted[2][0] + rep_sql: Final = emitted[1][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" assert ( f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " @@ -635,10 +624,10 @@ async def test_spend_logs_ui_group_by_session_offset_pages_for_other_sorts(monke ) group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" - count_sql = mock_prisma.db.query_raw.call_args_list[0][0][0] + count_sql = _spend_log_reads(mock_prisma).call_args_list[0][0][0] assert f"GROUP BY {group_key}" in count_sql - page_call = mock_prisma.db.query_raw.call_args_list[1][0] + page_call = _spend_log_reads(mock_prisma).call_args_list[1][0] page_sql = page_call[0] assert f"DISTINCT ON ({group_key})" in page_sql assert "ORDER BY spend DESC" in page_sql @@ -684,8 +673,108 @@ async def test_spend_logs_ui_request_id_lookup_with_grouping_returns_exact_row(m group_by_session=True, ) - page_call = mock_prisma.db.query_raw.call_args_list[1] + page_call = _spend_log_reads(mock_prisma).call_args_list[1] assert "request_id = $" in page_call[0][0], "the request_id equality filter must survive grouping" assert "req-deep-link" in page_call[0] assert [row["request_id"] for row in response["data"]] == ["req-deep-link"] assert response["total"] == 1 + + +async def _spend_logs_ui_page(mock_prisma: MagicMock, monkeypatch: pytest.MonkeyPatch) -> Mapping[str, object]: + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + mock_request = MagicMock() + mock_request.url.path = "/spend/logs/ui" + return await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), + request=mock_request, + api_key=None, + user_id=None, + request_id=None, + search=None, + start_date="2026-02-16 00:00:00", + end_date="2026-02-16 23:59:59", + page=1, + page_size=50, + sort_by="startTime", + sort_order="desc", + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + +@pytest.mark.asyncio +async def test_spend_logs_ui_reads_are_cancelled_by_postgres_after_30s(monkeypatch: pytest.MonkeyPatch): + """Prisma's HTTP client gives up at 30s either way; without SET LOCAL statement_timeout the scan kept running.""" + mock_prisma = MagicMock() + transaction = statement_timeout_reads(mock_prisma, AsyncMock(side_effect=[[{"total_count": 0}], []])) + + await _spend_logs_ui_page(mock_prisma, monkeypatch) + + assert transaction.statements == ("SET LOCAL statement_timeout = 30000",) * 2 + assert mock_prisma.read_db.tx.call_args.kwargs["timeout"] > datetime.timedelta(seconds=30) + + +@pytest.mark.asyncio +async def test_spend_logs_ui_cancelled_read_is_reported_as_503_with_a_date_range_hint(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy._types import ProxyException + + mock_prisma = MagicMock() + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=statement_timeout_error())) + + with pytest.raises(ProxyException) as exc_info: + await _spend_logs_ui_page(mock_prisma, monkeypatch) + + assert str(exc_info.value.code) == "503" + assert "Spend logs took longer than 30s. Pick a shorter date range" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_global_spend_report_returns_the_503_hint_not_its_catch_all_400_when_postgres_cancels( + monkeypatch: pytest.MonkeyPatch, +): + from fastapi import HTTPException + + from litellm.proxy.spend_tracking.spend_management_endpoints import get_global_spend_report + + mock_prisma = MagicMock() + statement_timeout_reads(mock_prisma, AsyncMock(side_effect=statement_timeout_error())) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + with pytest.raises(HTTPException) as exc_info: + await get_global_spend_report( + start_date="2026-02-01", + end_date="2026-02-16", + group_by="api_key", + api_key=None, + internal_user_id=None, + team_id=None, + customer_id=None, + ) + + assert exc_info.value.status_code == 503 + assert "Spend logs took longer than 30s. Pick a shorter date range" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_global_spend_report_reads_are_cancelled_by_postgres_after_30s(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.spend_tracking.spend_management_endpoints import get_global_spend_report + + mock_prisma = MagicMock() + transaction = statement_timeout_reads(mock_prisma, AsyncMock(return_value=[])) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + await get_global_spend_report( + start_date="2026-02-01", + end_date="2026-02-16", + group_by="api_key", + api_key=None, + internal_user_id=None, + team_id=None, + customer_id=None, + ) + + assert transaction.statements == ("SET LOCAL statement_timeout = 30000",) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index f73d33cc93e..45ded91b61e 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -47,7 +47,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_spend_logs_metadata, _get_vector_store_request_for_spend_logs_payload, is_master_key, - _redact_logged_api_key, + redact_logged_api_key, _redact_prompt_leaks_in_error_string, sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, @@ -4038,7 +4038,7 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form( assert REDACTED_BY_LITELM_STRING in sanitized["error_message"] -# ── _redact_logged_api_key unit tests ────────────────────────────────────── +# ── redact_logged_api_key unit tests ────────────────────────────────────── @pytest.mark.parametrize( @@ -4067,16 +4067,16 @@ def test_sanitize_error_information_persists_no_raw_model_for_an_unknown_model_r def test_redact_logged_api_key_none_returns_none(): - assert _redact_logged_api_key(None) is None + assert redact_logged_api_key(None) is None def test_redact_logged_api_key_empty_string_returns_none(): - assert _redact_logged_api_key("") is None + assert redact_logged_api_key("") is None def test_redact_logged_api_key_sk_key_is_hashed(): raw = "sk-9876secret" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result == hash_token(raw) assert result is not None assert not result.startswith("sk-") @@ -4085,15 +4085,15 @@ def test_redact_logged_api_key_sk_key_is_hashed(): def test_redact_logged_api_key_bearer_sk_equals_sk_hash(): raw = "sk-9876secret" - result_plain = _redact_logged_api_key(raw) - result_bearer = _redact_logged_api_key(f"Bearer {raw}") + result_plain = redact_logged_api_key(raw) + result_bearer = redact_logged_api_key(f"Bearer {raw}") assert result_bearer == result_plain def test_redact_logged_api_key_bearer_case_insensitive(): raw = "sk-9876secret" - result_lower = _redact_logged_api_key(f"bearer {raw}") - result_upper = _redact_logged_api_key(f"BEARER {raw}") + result_lower = redact_logged_api_key(f"bearer {raw}") + result_upper = redact_logged_api_key(f"BEARER {raw}") expected = hash_token(raw) assert result_lower == expected assert result_upper == expected @@ -4101,7 +4101,7 @@ def test_redact_logged_api_key_bearer_case_insensitive(): def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): raw = "anthropic-raw-key-xyz" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result is not None assert result != raw assert len(result) == 64 @@ -4111,7 +4111,7 @@ def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(already_hashed, already_redacted=True) + result = redact_logged_api_key(already_hashed, already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result # no double-hash @@ -4119,7 +4119,7 @@ def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): def test_redact_logged_api_key_sha256_without_flag_is_hashed(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(already_hashed) + result = redact_logged_api_key(already_hashed) assert result is not None assert result != already_hashed assert len(result) == 64 @@ -4129,7 +4129,7 @@ def test_redact_logged_api_key_sha256_without_flag_is_hashed(): def test_redact_logged_api_key_long_opaque_token_is_hashed(): raw = "x1" * 450 assert len(raw) == 900 - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result is not None assert result != raw assert raw not in result @@ -4139,27 +4139,27 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed(): def test_redact_logged_api_key_hashed_jwt_passes_through(): jwt_hash = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(jwt_hash, already_redacted=True) + result = redact_logged_api_key(jwt_hash, already_redacted=True) assert result == jwt_hash def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): lookalike = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(lookalike) + result = redact_logged_api_key(lookalike) assert result == hash_token(lookalike) assert result != lookalike def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): trailing = "hashed-jwt-" + "a" * 64 + "\n" - result = _redact_logged_api_key(trailing, already_redacted=True) + result = redact_logged_api_key(trailing, already_redacted=True) assert result == hash_token(trailing) assert result != trailing def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): short_jwt = "hashed-jwt-tooshort" - result = _redact_logged_api_key(short_jwt) + result = redact_logged_api_key(short_jwt) assert result is not None assert result != short_jwt assert len(result) == 64 @@ -4169,14 +4169,14 @@ def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): def test_redact_logged_api_key_master_key_alias_passes_through(): from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS - result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True) + result = redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True) assert result == LITELLM_PROXY_MASTER_KEY_ALIAS def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed(): from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS - result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS) + result = redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS) assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS) assert result != LITELLM_PROXY_MASTER_KEY_ALIAS @@ -4282,9 +4282,9 @@ def test_get_logging_payload_reads_used_client_oauth_token_from_the_bucket_the_p def test_redact_logged_api_key_bearer_only_returns_none(): # "bearer " with nothing after stripping is equivalent to no key - assert _redact_logged_api_key("bearer ") is None - assert _redact_logged_api_key("Bearer ") is None - assert _redact_logged_api_key("BEARER ") is None + assert redact_logged_api_key("bearer ") is None + assert redact_logged_api_key("Bearer ") is None + assert redact_logged_api_key("BEARER ") is None # ── _get_spend_logs_metadata key-hash invariant tests ───────────────────── @@ -4418,6 +4418,81 @@ def test_get_logging_payload_uses_recovered_combined_usage_on_failure(): assert payload["total_tokens"] == 31 +def _fallback_after_a_failed_follow_up( + cache_hit: bool, request_params: Mapping[str, object] = MappingProxyType({}) +): + from litellm.litellm_core_utils.hidden_params import set_hidden_param + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import ModelResponse, Usage + + now: Final = datetime.datetime.now(timezone.utc) + logger: Final = Logging( + model="fallback-model", + messages=[], + stream=False, + call_type="acompletion", + start_time=now, + litellm_call_id="caller", + function_id="function", + ) + logger.update_environment_variables( + model="fallback-model", + custom_llm_provider="openai", + litellm_params={"metadata": {"user_api_key": "sk-test"}}, + optional_params=dict(request_params), + ) + fallback: Final = ModelResponse(id="fallback", usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4)) + set_hidden_param(fallback, "response_cost", 0.1) + logger.record_served_rounds_for_failure(((Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7), 0.4),)) + logger.carry_failed_attempt_usage() + logger.model_call_details["cache_hit"] = cache_hit + prepared: Final = logger.prepare_response_cost(fallback, now) + assert prepared is not None + return logger, fallback, prepared, now + + +@pytest.mark.parametrize(("cache_hit", "own_cost"), [(False, 0.1), (True, 0.0)], ids=["fresh", "cached"]) +def test_success_row_bills_a_failed_follow_ups_cost_and_tokens(cache_hit: bool, own_cost: float): + logger, fallback, _, now = _fallback_after_a_failed_follow_up(cache_hit) + + payload: Final = get_logging_payload( + kwargs=logger.model_call_details, response_obj=fallback, start_time=now, end_time=now + ) + + assert payload["spend"] == pytest.approx(own_cost + 0.4) + assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (3 + 5, 1 + 2, 4 + 7) + + +@pytest.mark.parametrize(("cache_hit", "own_cost"), [(False, 0.1), (True, 0.0)], ids=["fresh", "cached"]) +def test_request_params_cannot_change_what_a_failed_follow_up_bills(cache_hit: bool, own_cost: float): + forged_attempt: Final = [{"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000}, -100.0] + logger, fallback, _, now = _fallback_after_a_failed_follow_up( + cache_hit, + { + "stash_billed_by_continuation": True, + "failed_attempts_owed_on_success": [forged_attempt], + "failed_attempts_billed_on_success": [forged_attempt], + }, + ) + + payload: Final = get_logging_payload( + kwargs=logger.model_call_details, response_obj=fallback, start_time=now, end_time=now + ) + + assert payload["spend"] == pytest.approx(own_cost + 0.4) + assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (8, 3, 11) + + +def test_cached_fallback_cost_breakdown_adds_up_to_the_billed_failed_follow_up(): + _, _, prepared, _ = _fallback_after_a_failed_follow_up(cache_hit=True) + + breakdown: Final = prepared.payload["cost_breakdown"] + + assert breakdown is not None + assert breakdown["total_cost"] == pytest.approx(prepared.payload["response_cost"]) == pytest.approx(0.4) + assert breakdown["additional_costs"]["failed_attempts_cost"] == pytest.approx(0.4) + + def test_get_logging_payload_failure_without_recovered_usage_is_zero(): """A failure with no recovered usage keeps zero token counts, so the combined-usage override never invents tokens for ordinary failures. @@ -4570,7 +4645,7 @@ class TestSpendLogKeyRedaction: def test_bearer_prefixed_sk_key_is_hashed(self): raw = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result is not None assert not result.startswith("Bearer") assert not result.startswith("sk-") @@ -4578,14 +4653,14 @@ class TestSpendLogKeyRedaction: def test_bare_sk_key_is_hashed(self): raw = "sk-WLi4iRn4JmbVlTaYw12IOA" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result is not None assert not result.startswith("sk-") assert len(result) == 64 def test_bearer_lowercase_is_handled(self): raw = "bearer sk-WLi4iRn4JmbVlTaYw12IOA" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result is not None assert not result.startswith("bearer") assert not result.startswith("sk-") @@ -4593,11 +4668,11 @@ class TestSpendLogKeyRedaction: def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _redact_logged_api_key(hashed, already_redacted=True) == hashed + assert redact_logged_api_key(hashed, already_redacted=True) == hashed def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" - result = _redact_logged_api_key(raw) + result = redact_logged_api_key(raw) assert result == hash_token("some-other-token-format") assert result is not None assert not result.startswith("Bearer") @@ -4605,7 +4680,7 @@ class TestSpendLogKeyRedaction: def test_bearer_and_bare_produce_same_hash(self): bare = "sk-WLi4iRn4JmbVlTaYw12IOA" bearer = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA" - assert _redact_logged_api_key(bare) == _redact_logged_api_key(bearer) + assert redact_logged_api_key(bare) == redact_logged_api_key(bearer) @patch("litellm.proxy.proxy_server.master_key", None) @@ -4702,7 +4777,7 @@ def test_get_logging_payload_keeps_internal_service_account_key_readable(service def test_redact_logged_api_key_service_account_name_without_provenance_is_hashed(): - result = _redact_logged_api_key(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME) + result = redact_logged_api_key(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME) assert result == hash_token(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME) @@ -5055,6 +5130,105 @@ def test_get_spend_logs_id_prefers_the_response_id_for_batch_and_file_calls(call assert get_spend_logs_id(call_type, {"id": "batch_abc123"}, {"litellm_call_id": "call-id-1"}) == expected +@pytest.mark.parametrize("status_call_type", ["avideo_status", "video_status"]) +@pytest.mark.parametrize( + ("response_cost", "status_id"), + [ + (0.42, "video_abc_video_cost"), + (0.0, "video_abc"), + (None, "video_abc"), + (float("nan"), "video_abc"), + ], + ids=["charged", "free", "unpriced", "non_finite"], +) +def test_get_logging_payload_gives_only_a_charged_video_poll_a_row_of_its_own(status_call_type, response_cost, status_id): + """A custom provider charges the video on the poll that sees it completed, so that poll needs + a row apart from the create row, which answers with the same video id and would drop it on the + skip_duplicates insert. A poll that charges nothing keeps the bare id, so it still lands on the + create row and adds no row of its own.""" + from litellm.types.videos.main import VideoObject + + video = VideoObject(id="video_abc", object="video", status="completed") + create, status = ( + get_logging_payload( + kwargs={ + "call_type": call_type, + "litellm_call_id": call_id, + "response_cost": cost, + "polled_video_status": "completed", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=video, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + for call_type, call_id, cost in (("avideo_generation", "call-1", 0.42), (status_call_type, "call-2", response_cost)) + ) + + assert create["request_id"] == "video_abc" + assert status["request_id"] == status_id + + +@pytest.mark.parametrize(("response_cost", "pays_the_job"), [(0.01, False), (0.43, True)]) +def test_a_video_poll_that_pays_only_its_guardrail_gets_a_row_of_its_own(response_cost, pays_the_job): + """A guardrail charges every poll, so a poll paying only that must neither land on the create row + nor take the row the video's own charge goes on, either of which the skip_duplicates insert drops.""" + kwargs = { + "litellm_call_id": "poll-call-id", + "response_cost": response_cost, + "polled_video_status": "completed", + "standard_logging_object": { + "response_cost": response_cost, + "guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}], + }, + } + + assert get_spend_logs_id("avideo_status", {"id": "video_abc", "status": "completed"}, kwargs) == ( + "video_abc_video_cost" if pays_the_job else "poll-call-id_guardrail_cost" + ) + + +@pytest.mark.parametrize("status", ["failed", "completed"]) +def test_a_finished_batch_retrieve_paying_only_its_guardrail_charges_under_the_batch_row(status): + """The retrieve that settles a batch charges its whole cost once, guardrails included, so a batch whose own + cost is $0 must still land on the row that makes its repeat retrieves free.""" + from litellm.types.utils import LiteLLMBatch + + kwargs = { + "litellm_call_id": "poll-call-id", + "response_cost": 0.01, + "standard_logging_object": { + "response_cost": 0.01, + "guardrail_information": [{"guardrail_name": "priced", "guardrail_cost": 0.01}], + }, + } + polled_batch = LiteLLMBatch( + id="batch_abc", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + object="batch", + status=status, + output_file_id="file-out", + ) + + assert get_spend_logs_id("aretrieve_batch", polled_batch.model_dump(), kwargs) == "batch_abc_batch_cost" + + +def test_a_video_polled_with_or_without_its_base64_padding_charges_one_row(): + from litellm.types.videos.utils import encode_video_id_with_provider + + padded = encode_video_id_with_provider("job-1", "custom_video", "deployment-a") + assert padded.endswith("=") + + def row_id(polled_id: str) -> str | None: + kwargs = {"response_cost": 0.42, "polled_video_status": "completed", "litellm_params": {"video_id": polled_id}} + return get_spend_logs_id("avideo_status", {"id": "job-1", "status": "completed"}, kwargs) + + assert row_id(padded.rstrip("=")) == row_id(padded) == f"{padded}_video_cost" + + def test_get_logging_payload_gives_redacted_batch_and_file_rows_distinct_request_ids(): """End to end at the payload level: a batch retrieve and a file create whose bodies were both flattened to the same redaction placeholder must still produce two @@ -5715,7 +5889,7 @@ def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal assert payload["api_key"] == "", ( f"Expected empty string but got {payload['api_key']!r}; " - "dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here" + "dropping redact_logged_api_key's 'or \"\"' guard would yield 'None' here" ) @@ -5766,7 +5940,7 @@ def test_get_logging_payload_stores_user_email_in_spend_log_metadata(user_email, def test_redact_logged_api_key_partial_sha256_is_hashed(): partial_hex = "a" * 63 - result = _redact_logged_api_key(partial_hex) + result = redact_logged_api_key(partial_hex) assert result is not None assert result != partial_hex assert len(result) == 64 @@ -5776,7 +5950,7 @@ def test_redact_logged_api_key_partial_sha256_is_hashed(): def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True) + result = redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result @@ -5784,7 +5958,7 @@ def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(f"Bearer {already_hashed}") + result = redact_logged_api_key(f"Bearer {already_hashed}") assert result is not None assert result != already_hashed assert result == hash_token(already_hashed) @@ -6396,12 +6570,12 @@ def test_get_logging_payload_never_lands_a_raw_cli_session_token(call_type: str) def test_redact_logged_api_key_cli_session_alias_needs_alias_provenance(): - assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True, key_alias=_CLI_SESSION_ALIAS) == ( + assert redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True, key_alias=_CLI_SESSION_ALIAS) == ( _CLI_SESSION_ALIAS ) - assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True) == hash_token(_CLI_SESSION_ALIAS) - assert _redact_logged_api_key(_CLI_SESSION_ALIAS, key_alias=_CLI_SESSION_ALIAS) == hash_token(_CLI_SESSION_ALIAS) - assert _redact_logged_api_key("alice", already_redacted=True, key_alias="alice") == hash_token("alice") + assert redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True) == hash_token(_CLI_SESSION_ALIAS) + assert redact_logged_api_key(_CLI_SESSION_ALIAS, key_alias=_CLI_SESSION_ALIAS) == hash_token(_CLI_SESSION_ALIAS) + assert redact_logged_api_key("alice", already_redacted=True, key_alias="alice") == hash_token("alice") def test_azure_spillover_stamped_from_response_headers(): @@ -7486,6 +7660,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr assert sanitized["error_code"] == "500" +def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False) + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200") + provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000 + now: Final = datetime.datetime.now(timezone.utc) + + row: Final = get_logging_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "failure", + "error_information": { + "error_code": "503", + "error_class": "BedrockError", + "error_message": provider_error, + "llm_provider": "bedrock", + }, + } + }, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + stored: Final = json.loads(row["metadata"])["error_information"] + assert row["status"] == "failure" + assert "private-prompt" not in stored["error_message"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"] + assert len(stored["error_message"]) < len(provider_error) + assert stored["error_class"] == "BedrockError" + + @pytest.mark.parametrize( "address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"] ) @@ -7580,13 +7794,11 @@ def test_optional_mcp_metadata_does_not_change_legacy_spend_payload(tool_metadat assert actual == expected -@pytest.mark.parametrize("capture_enabled", [False, True]) @pytest.mark.parametrize("completion_start", [None, "absent", "reported"]) -def test_spend_payload_preserves_unknown_completion_start(capture_enabled: bool, completion_start: str | None) -> None: +def test_spend_payload_preserves_unknown_completion_start(completion_start: str | None) -> None: start: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) end: Final = start + datetime.timedelta(seconds=2) reported: Final = start + datetime.timedelta(seconds=1) - observations: Final[list[object]] = [] kwargs: Final = { "call_type": "acompletion", "litellm_call_id": "timestamp-call", @@ -7598,9 +7810,49 @@ def test_spend_payload_preserves_unknown_completion_start(capture_enabled: bool, {"id": "timestamp-response"}, start, end, - capture_usage=(lambda row, usage: observations.append(row)) if capture_enabled else None, ) assert payload["completionStartTime"] == ( end if completion_start == "absent" else reported if completion_start else None ) - assert observations == ([payload] if capture_enabled else []) + + +def test_spend_log_metadata_retention_drops_accounting_inputs_it_was_not_told_to_keep() -> None: + metadata: Final = { + "status": "success", + "accounting_savings": { + "compression": 0.4, + "prompt_caching": 0.1, + "gateway_injected_caching": 0.2, + "autorouter": 0.3, + }, + "accounting_windows": [ + {"entity_type": "key", "entity_id": "key", "window_duration": "1d", "window_start": "2026-11-02T00:00:00"} + ], + "accounting_model_access_groups": ["group"], + "user_api_key_project_id": "project", + "usage_object": {"prompt_tokens_details": {"cached_tokens": 2}}, + "compression_savings": {"tokens_saved": 3}, + "guardrail_information": [{"guardrail_id": "guard", "guardrail_usage": {"text": 2}}], + "internal_call_origin": "classifier", + "billed_retry_attempt": True, + "user_api_key_alias": "discard", + } + row: Final = { + "request_id": "retained-accounting", + "spend": 0.5, + "startTime": "2026-11-02T12:00:00", + "endTime": "2026-11-02T12:00:01", + "model": "model", + "model_group": "group", + "api_key": "key", + "custom_llm_provider": "openai", + "user": "user", + "team_id": "team", + "agent_id": "agent", + "request_tags": '["tag"]', + "metadata": json.dumps(metadata), + } + stored: Final = spend_log_row_with_retained_metadata(row, SpendLogsMetadataFields(include=("usage_object",))) + parsed: Final = json.loads(str(stored["metadata"])) + assert "user_api_key_alias" not in parsed + assert parsed == {"usage_object": metadata["usage_object"], "status": metadata["status"]} diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py deleted file mode 100644 index 32710e594ee..00000000000 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ /dev/null @@ -1,1712 +0,0 @@ -import asyncio -import json -from collections.abc import Awaitable, Mapping -from datetime import datetime, timezone -from decimal import Decimal -from typing import Final - -import pytest - -from litellm.accounting import Metric, MetricName, Outcome -from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite -from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings -from litellm.types.utils import CallTypes -from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier, _manual_flush, _write - - -@pytest.mark.asyncio -@pytest.mark.parametrize("usage", [None, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}]) -async def test_capture_keeps_unknown_usage_distinct_from_measured_zero(usage: Mapping[str, int] | None) -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = CommitBarrier() - database.release.set() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - timestamp: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - captured: Final = capture_callback( - meter, - {"call_type": "acompletion", "litellm_call_id": "caller", "model": "unknown-model", "litellm_params": {}}, - {"id": "provider", "usage": usage}, - timestamp, - timestamp, - ) - await meter.close() - outbound: Final = captured.writes[1].event - assert outbound.metrics == ( - None - if usage is None - else tuple(Metric(name=MetricName(key), value=Decimal(value)) for key, value in usage.items()) - ) - envelope: Final = json.loads(captured.writes[1].envelope) - assert envelope["usage"] == usage - assert envelope["pricing"]["reported_cost"] is None - assert database.committed == ("caller:inbound", "provider") - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "call_type", - [ - call.value - for call in ( - CallTypes.acompletion, - CallTypes.arealtime, - CallTypes.call_mcp_tool, - CallTypes.asend_message, - CallTypes.pass_through, - ) - ], -) -async def test_shared_callback_capture_retains_usage_and_attribution_without_bodies(call_type: str) -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = CommitBarrier() - database.release.set() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - timestamp: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - captured: Final = capture_callback( - meter, - { - "call_type": call_type, - "litellm_call_id": "caller", - "model": "test-model", - "messages": "private prompt", - "litellm_params": {"api_key": "private-secret", "metadata": {"user_api_key_team_id": "team"}}, - }, - { - "id": "provider", - "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, - "output": "private answer", - }, - timestamp, - timestamp, - ) - await meter.close() - envelope: Final = captured.writes[1].envelope - assert all(secret not in envelope for secret in ("private prompt", "private-secret", "private answer")) - event: Final = captured.writes[1].event - assert event.call_type.value == call_type - assert event.metrics == ( - Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(3)), - Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)), - Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(5)), - ) - assert any(dimension.value == "team" for dimension in event.dimensions) - - -@pytest.mark.asyncio -async def test_cache_duplicate_uses_caller_identity_despite_timestamped_diagnostic_ids() -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = CommitBarrier() - database.release.set() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - timestamp: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - kwargs: Final = { - "call_type": "acompletion", - "litellm_call_id": "caller", - "model": "test-model", - "cache_hit": True, - "litellm_params": {}, - } - response: Final = {"id": "cached-provider-id", "usage": {"total_tokens": 5}} - first: Final = capture_callback(meter, kwargs, response, timestamp, timestamp) - second: Final = capture_callback(meter, kwargs, response, timestamp, timestamp) - await meter.close() - assert first.writes == second.writes - assert first.writes[1].event.event_id == "caller_cache_hit" - assert json.loads(first.writes[1].envelope)["charge_eligible"] is False - - -NOW: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - - -class StoredUsage: - def __init__(self) -> None: - self.rows: tuple[UsageWrite, ...] = () - - async def load(self, identities: tuple[str, ...]) -> tuple[UsageWrite, ...]: - return tuple(row for row in self.rows if row.event.event_id in identities) - - async def insert(self, writes: tuple[UsageWrite, ...]) -> None: - existing: Final = {row.event.event_id: row for row in self.rows} - if any(existing.get(row.event.event_id, row) != row for row in writes): - raise AccountingConflictError("Same identity has different content") - unique: Final = {row.event.event_id: row for row in writes} - if any(unique[row.event.event_id] != row for row in writes): - raise AccountingConflictError("Submission contains conflicting content") - self.rows += tuple(row for identity, row in unique.items() if identity not in existing) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("text", ["parent\x00value", "pärent-✓-日本", r"\u0000"]) -async def test_capture_normalizes_database_text_without_poisoning_neighbors(text: str) -> None: - from litellm.proxy.spend_tracking.usage_capture import PricingCapture, capture_callback - - database: Final = StoredUsage() - batches: Final = asyncio.Queue[tuple[UsageWrite, ...]]() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - assert all("\x00" not in (json.loads(write.envelope).get("parent_request_id") or "") for write in writes) - batches.put_nowait(writes) - await database.insert(writes) - - meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(), wait_for_batch=_manual_flush) - nested: Final = json.loads("[" * 100 + "3" + "]" * 100) - kwargs: Final = { - "call_type": "acompletion", - "litellm_call_id": "caller", - "model": "capture-model", - "search_context_size": text, - "_accounting_pricing_snapshot": PricingCapture( - "{}", "{}", json.dumps({"request": {"search_context_size": text}, "nested": nested}) - ), - "litellm_params": {"metadata": {"parent_request_id": text}}, - } - response: Final = {"id": "provider", "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}} - captured: Final = capture_callback(meter, kwargs, response, NOW, NOW) - neighbor: Final = capture_callback( - meter, - {**kwargs, "litellm_call_id": "neighbor", "litellm_params": {}}, - {**response, "id": "neighbor-provider"}, - NOW, - NOW, - ) - try: - await meter.flush() - await asyncio.gather(captured.receipt, neighbor.receipt) - assert batches.qsize() == 1 - assert len(batches.get_nowait()) == 4 - envelope: Final = json.loads(captured.writes[-1].envelope) - assert envelope["parent_request_id"] == text.replace("\x00", "") - assert envelope["pricing"]["request"]["search_context_size"] == text.replace("\x00", "") - assert envelope["pricing"]["nested"] == nested - assert envelope["event"] == captured.writes[-1].event.model_dump(mode="json") - assert envelope["usage"] == response["usage"] - assert kwargs["litellm_params"] == {"metadata": {"parent_request_id": text}} - repeated: Final = capture_callback(meter, kwargs, response, NOW, NOW) - await meter.flush() - await repeated.receipt - assert repeated.writes == captured.writes - assert len(database.rows) == 4 - finally: - await meter.stop(0) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("invalid_field", ["identity", "context"]) -async def test_invalid_capture_text_is_rejected_before_admission(invalid_field: str) -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - kwargs: Final = {"call_type": "acompletion", "litellm_call_id": "caller", "litellm_params": {}} - try: - accepted: Final = capture_callback(meter, kwargs, {"id": "provider"}, NOW, NOW) - invalid: Final = ( - {**kwargs, "litellm_call_id": "caller\x00"} - if invalid_field == "identity" - else {**kwargs, "litellm_params": {"metadata": {"parent_request_id": "parent\ud800"}}} - ) - with pytest.raises((UsageCaptureError, ValueError)): - capture_callback(meter, invalid, {"id": "other"}, NOW, NOW) - assert meter.pending_records == 2 - await meter.flush() - await accepted.receipt - assert tuple(write.event.event_id for write in database.rows) == ("caller:inbound", "provider") - finally: - await meter.stop(0) - - -def _logger(call_type: str = "acompletion", *, stream: bool = False) -> Logging: - from litellm.litellm_core_utils.litellm_logging import Logging - - logger: Final = Logging( - model="capture-model", - messages=[], - stream=stream, - call_type=call_type, - start_time=NOW, - litellm_call_id="caller", - function_id="function", - ) - logger.update_environment_variables( - model="capture-model", custom_llm_provider="openai", litellm_params={}, optional_params={} - ) - return logger - - -@pytest.mark.asyncio -async def test_terminal_retrieval_reuses_accepted_snapshot_but_rejects_changed_usage() -> None: - from datetime import timedelta - - from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY - from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, capture_callback - from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN - - database: Final = StoredUsage() - writer: Final = UsageBatchWriter(database.insert, database.load) - meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - kwargs: Final = { - "call_type": "aget_responses", - "litellm_call_id": "poll-one", - "model": "capture-model", - "response_cost": 2.0, - "litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}}, - } - response: Final = { - "id": "job", - "status": "completed", - "background": True, - "created_at": NOW.timestamp(), - "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7}, - } - first: Final = capture_callback(meter, kwargs, response, NOW, NOW) - await meter.flush() - later: Final = capture_callback( - meter, {**kwargs, "litellm_call_id": "poll-two", "response_cost": 9.0}, response, NOW, NOW + timedelta(hours=2) - ) - await meter.flush() - await later.receipt - assert database.rows == first.writes - assert len(database.rows) == 1 - assert database.rows[0].event.event_id == "job_background_cost" - changed: Final = capture_callback(meter, kwargs, {**response, "usage": {"total_tokens": 8}}, NOW, NOW) - await meter.close() - with pytest.raises(UsageCaptureError, match="identity"): - await changed.receipt - assert database.rows == first.writes - - -@pytest.mark.asyncio -@pytest.mark.parametrize("terminal", [False, True]) -async def test_competing_writers_reuse_only_terminal_pricing_snapshots(terminal: bool) -> None: - from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY - from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, capture_callback - from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN - - database: Final = StoredUsage() - attempts: Final = asyncio.Queue[tuple[UsageWrite, ...]]() - competing: Final = asyncio.Event() - - async def insert(writes: tuple[UsageWrite, ...]) -> None: - attempts.put_nowait(writes) - if attempts.qsize() >= 2: - competing.set() - await competing.wait() - await database.insert(writes) - - meters: Final = tuple( - DurableUsageMeter(UsageBatchWriter(insert, database.load).write, UsageCaptureSettings()) for _ in range(2) - ) - kwargs: Final = { - "call_type": "aget_responses" if terminal else "acompletion", - "litellm_call_id": "caller", - "model": "capture-model", - "litellm_params": { - "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN} - if terminal - else {} - }, - } - response: Final = { - "id": "job", - "status": "completed", - "background": terminal, - "created_at": NOW.timestamp(), - "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7}, - } - captured: Final = tuple( - capture_callback(meter, {**kwargs, "response_cost": price}, response, NOW, NOW) - for meter, price in zip(meters, (2.0, 9.0), strict=True) - ) - try: - await asyncio.gather(*(meter.flush() for meter in meters)) - results: Final = await asyncio.gather(*(capture.receipt for capture in captured), return_exceptions=True) - assert sum(result is None for result in results) == (2 if terminal else 1) - assert sum(isinstance(result, UsageCaptureError) for result in results) == (0 if terminal else 1) - assert database.rows in tuple(capture.writes for capture in captured) - assert len(database.rows) == (1 if terminal else 2) - assert attempts.qsize() == (3 if terminal else 2) - finally: - await asyncio.gather(*(meter.stop(0) for meter in meters)) - - -@pytest.mark.asyncio -async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_chargeable() -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - kwargs: Final = { - "call_type": "aretrieve_batch", - "litellm_call_id": "poll", - "model": "capture-model", - "litellm_params": {}, - } - pending: Final = capture_callback(meter, kwargs, {"id": "batch", "status": "in_progress"}, NOW, NOW) - completed: Final = capture_callback( - meter, kwargs, {"id": "batch", "status": "completed", "usage": {"total_tokens": 9}}, NOW, NOW - ) - await meter.close() - assert pending.writes[-1].event.event_id == "poll:status" - assert json.loads(pending.writes[-1].envelope)["charge_eligible"] is False - assert completed.writes[-1].event.event_id == "batch_batch_cost" - assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is True - assert len(completed.writes) == 1 - - -@pytest.mark.asyncio -async def test_internal_capture_does_not_claim_reused_parent_logger(monkeypatch: pytest.MonkeyPatch) -> None: - import litellm - from litellm._internal_context import internal_sub_call - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - from litellm.types.utils import ModelResponse, Usage - - monkeypatch.setitem( - litellm.model_cost, - "capture-model", - { - "input_cost_per_token": 0.1, - "output_cost_per_token": 0.2, - "litellm_provider": "openai", - "mode": "chat", - }, - ) - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger() - response: Final = ModelResponse( - id="response", model="capture-model", usage=Usage(prompt_tokens=2, completion_tokens=1, total_tokens=3) - ) - with internal_sub_call("caller"): - capture_logging_usage(meter, logger, response, NOW, NOW) - monkeypatch.setitem( - litellm.model_cost, - "capture-model", - { - "input_cost_per_token": 0.7, - "output_cost_per_token": 0.9, - "litellm_provider": "openai", - "mode": "chat", - }, - ) - litellm.register_model({"capture-model": litellm.model_cost["capture-model"]}) - capture_logging_usage(meter, logger, response, NOW, NOW) - capture_logging_usage(meter, logger, response, NOW, NOW) - await meter.close() - assert tuple(row.event.event_id for row in database.rows) == ("response:included", "caller:inbound", "response") - child: Final = json.loads(database.rows[0].envelope) - parent: Final = json.loads(database.rows[-1].envelope) - assert (child["ownership"], child["charge_eligible"], child["parent_request_id"]) == ( - "included_in_parent", - False, - "caller", - ) - assert parent["charge_eligible"] is True - assert child["pricing"]["model_map_information"]["model_map_value"]["input_cost_per_token"] == 0.1 - assert parent["pricing"]["model_map_information"]["model_map_value"]["input_cost_per_token"] == 0.7 - - -@pytest.mark.asyncio -async def test_stream_capture_ignores_chunks_and_preserves_terminal_usage_extras() -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - from litellm.types.utils import ModelResponse, ModelResponseStream, Usage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger(stream=True) - capture_logging_usage(meter, logger, ModelResponseStream(id="chunk"), NOW, NOW) - assert meter.pending_records == 0 - capture_logging_usage( - meter, logger, ModelResponse(id="stream", usage=Usage(prompt_tokens=3, provider_units=4)), NOW, NOW - ) - await meter.close() - assert json.loads(database.rows[-1].envelope)["usage"]["provider_units"] == 4 - assert database.rows[-1].event.event_id == "stream" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("call_type", ["acompletion", "atext_completion", "aembedding", "aresponses"]) -async def test_supported_sync_ack_waits_for_commit(call_type: str, monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy import proxy_server - from litellm.proxy._types import ConfigGeneralSettings - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) - deadlines: Final = asyncio.Queue[float | None]() - original_wait: Final = asyncio.wait_for - - async def wait_with_deadline(awaitable: Awaitable[object], timeout: float | None) -> object: - deadlines.put_nowait(timeout) - return await original_wait(awaitable, timeout) - - monkeypatch.setattr(asyncio, "wait_for", wait_with_deadline) - response: Final = asyncio.create_task( - capture_proxy_response( - _logger(call_type), - ModelResponse(id="provider", usage=Usage(prompt_tokens=3)), - ) - ) - try: - await database.entered.wait() - deadline: Final = deadlines.get_nowait() - assert deadline == ConfigGeneralSettings().accounting_sync_ack_timeout_seconds - assert deadline is not None and deadline > UsageCaptureSettings().flush_interval_seconds + 10 - assert not response.done() - assert database.committed == () - database.release.set() - await response - assert database.committed == ("caller:inbound", "provider") - finally: - database.release.set() - await runtime.stop(1) - await response - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure_policy", ["fail_closed", "log"]) -async def test_sync_ack_commit_failure_follows_response_policy( - failure_policy: str, monkeypatch: pytest.MonkeyPatch -) -> None: - from fastapi import HTTPException - - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier(failure=ValueError("invalid envelope")) - database.release.set() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2)) - meter: Final = runtime.meter - assert meter is not None - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr( - proxy_server, - "general_settings", - {"accounting_sync_ack": True, "accounting_sync_ack_failure_policy": failure_policy}, - ) - logger: Final = _logger() - response: Final = ModelResponse(id="provider", usage=Usage(prompt_tokens=4)) - if failure_policy == "fail_closed": - with pytest.raises(HTTPException) as error: - await capture_proxy_response(logger, response) - assert error.value.status_code == 503 - else: - await capture_proxy_response(logger, response) - await runtime.stop(1.0) - assert database.committed == () - assert meter.pending_records == 0 - - -@pytest.mark.asyncio -async def test_failed_attempt_keeps_its_usage_without_claiming_the_callers_success() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - try: - for _ in range(2): - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7), 0.4) - logger.carry_failed_attempt_usage() - logger.capture_usage_before_logging( - ModelResponse(id="fallback", usage=Usage(prompt_tokens=3, total_tokens=3)), NOW, NOW - ) - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ( - "caller-attempt-0", - "caller-attempt-1", - "caller:inbound", - "fallback", - ) - failed: Final = json.loads(database.rows[0].envelope) - assert failed["usage"]["total_tokens"] == 7 - assert failed["parent_request_id"] == "caller" - assert failed["request_count_eligible"] is False - assert failed["charge_eligible"] is True - assert failed["event"]["retry_of_attempt_id"] is None - assert failed["event"]["outcome"] == "failed" - assert failed["pricing"]["reported_cost"] == 0.4 - assert all(json.loads(row.envelope)["included_attempt_ids"] == [] for row in database.rows[:2]) - successful: Final = json.loads(database.rows[-1].envelope) - assert successful["related_attempt_ids"] == ["caller-attempt-0", "caller-attempt-1"] - assert successful["included_attempt_ids"] == [] - assert successful["usage"]["total_tokens"] == 3 - assert successful["charge_eligible"] is True - assert database.rows[2].event.outcome is Outcome.SUCCEEDED - - -@pytest.mark.asyncio -@pytest.mark.parametrize("estimated", [False, True]) -async def test_cancelled_usage_keeps_measured_or_estimated_provenance(estimated: bool) -> None: - from fastapi import HTTPException - - from litellm.litellm_core_utils.litellm_logging import UsageObservation - from litellm.proxy.spend_tracking.usage_capture import capture_callback - from litellm.types.utils import Usage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - capture_callback( - meter, - { - "litellm_call_id": "cancelled", - "call_type": "acompletion", - "model": "capture-model", - "litellm_params": {}, - "combined_usage_object": Usage(prompt_tokens=4, total_tokens=4), - "_accounting_usage_observed": UsageObservation.ESTIMATED if estimated else UsageObservation.KNOWN, - }, - HTTPException(499, "Client disconnected"), - NOW, - NOW, - ) - await meter.close() - payload: Final = json.loads(database.rows[-1].envelope) - assert payload["event"]["outcome"] == "cancelled" - assert payload["usage"]["total_tokens"] == 4 - assert payload["usage_source"] == ("estimated" if estimated else "measured") - assert payload["pricing"]["reported_cost"] is None - - -@pytest.mark.asyncio -async def test_realtime_capture_keeps_metered_units_and_provider_usage_without_frames() -> None: - from litellm.constants import REALTIME_METERED_USAGE_KEY - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - from litellm.types.llms.xai import RealtimeMeteredUsage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger(CallTypes.arealtime.value) - logger.model_call_details[REALTIME_METERED_USAGE_KEY] = RealtimeMeteredUsage(True, 1.5, 2.5, 3) - capture_logging_usage( - meter, - logger, - [ - { - "type": "response.done", - "response": { - "id": "turn", - "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7}, - "output": "private realtime transcript", - }, - } - ], - NOW, - NOW, - ) - await meter.close() - payload: Final = json.loads(database.rows[-1].envelope) - assert payload["usage"]["total_tokens"] == 7 - assert payload["usage"]["metered_usage"] == { - "push_to_talk": True, - "audio_seconds_sent": 1.5, - "audio_seconds_received": 2.5, - "text_messages": 3, - } - assert "private realtime transcript" not in database.rows[-1].envelope - - -@pytest.mark.asyncio -async def test_default_http_capture_returns_before_database_commit(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "general_settings", {}) - try: - await capture_proxy_response( - _logger(), - ModelResponse(id="asynchronous", usage=Usage(prompt_tokens=1)), - ) - await database.entered.wait() - assert database.committed == () - finally: - database.release.set() - await runtime.stop(1) - assert database.committed == ("caller:inbound", "asynchronous") - - -@pytest.mark.asyncio -async def test_deferred_capture_freezes_pricing_before_configuration_changes(monkeypatch: pytest.MonkeyPatch) -> None: - import litellm - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_logging_usage - from litellm.types.utils import ModelResponse, Usage - from litellm.utils import dispatch_success_logging - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - logger.defer_async_logging = True - logger.custom_cost_per_second = 0.25 - monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) - response: Final = ModelResponse(id="deferred", usage=Usage(prompt_tokens=3)) - try: - dispatch_success_logging(logger, response, NOW, NOW, False, False) - assert database.rows == () - logger.custom_cost_per_second = 9 - monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.9}) - assert runtime.meter is not None - capture_logging_usage(runtime.meter, logger, response, NOW, NOW) - await runtime.meter.flush() - finally: - await runtime.stop(1) - pricing: Final = json.loads(database.rows[-1].envelope)["pricing"] - assert pricing["logging"]["custom_cost_per_second"] == 0.25 - assert pricing["adjustments"]["discount"] == 0.1 - - -@pytest.mark.asyncio -async def test_synchronous_internal_capture_reserves_capacity_before_scheduling() -> None: - from litellm._internal_context import internal_sub_call - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(max_pending_records=2, batch_size=2)) - logger: Final = _logger() - - def capture_child() -> None: - with internal_sub_call("caller"): - logger.capture_usage_before_logging(ModelResponse(id="child", usage=Usage(prompt_tokens=2)), NOW, NOW) - - try: - await asyncio.to_thread(capture_child) - assert runtime.meter is not None - assert runtime.meter.pending_records == 1 - receipt: Final = await asyncio.to_thread(runtime.meter.submit, (_write("second"),)) - await database.entered.wait() - with pytest.raises(UsageCaptureError, match="capacity"): - await asyncio.to_thread(runtime.meter.submit, (_write("overflow"),)) - database.release.set() - await receipt - finally: - database.release.set() - await runtime.stop(1) - assert database.committed == ("child:included", "second") - - -@pytest.mark.asyncio -@pytest.mark.parametrize("has_metered_usage", [False, True]) -async def test_realtime_without_provider_token_usage_does_not_invent_zero(has_metered_usage: bool) -> None: - from litellm.constants import REALTIME_METERED_USAGE_KEY - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - from litellm.types.llms.xai import RealtimeMeteredUsage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger(CallTypes.arealtime.value) - if has_metered_usage: - logger.model_call_details[REALTIME_METERED_USAGE_KEY] = RealtimeMeteredUsage(True, 0, 0, 0) - capture_logging_usage(meter, logger, [], NOW, NOW) - await meter.close() - usage: Final = json.loads(database.rows[-1].envelope)["usage"] - assert usage == ( - { - "metered_usage": { - "push_to_talk": True, - "audio_seconds_sent": 0, - "audio_seconds_received": 0, - "text_messages": 0, - } - } - if has_metered_usage - else None - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("committed_before_failure", [False, True]) -async def test_database_recovery_retries_frozen_usage_without_callback_or_duplicate_rows( - committed_before_failure: bool, -) -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = StoredUsage() - failures: Final = asyncio.Queue[None]() - failures.put_nowait(None) - retrying: Final = asyncio.Event() - recovered: Final = asyncio.Event() - delays: Final = asyncio.Queue[float]() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - if not failures.empty(): - failures.get_nowait() - if committed_before_failure: - await database.insert(writes) - raise ConnectionError("database unavailable or acknowledgement lost") - await database.insert(writes) - - async def retry_delay(delay: float) -> None: - delays.put_nowait(delay) - retrying.set() - await recovered.wait() - - meter: Final = DurableUsageMeter( - persist, UsageCaptureSettings(max_pending_records=2), wait_for_batch=_manual_flush, retry_delay=retry_delay - ) - kwargs: Final = { - "call_type": "acompletion", - "litellm_call_id": "caller", - "model": "capture-model", - "litellm_params": {}, - } - response: Final = {"id": "provider", "usage": {"total_tokens": 3}} - first: Final = capture_callback(meter, kwargs, response, NOW, NOW) - reserved: Final = meter.pending_bytes - flushing: Final = asyncio.create_task(meter.flush()) - await retrying.wait() - assert not first.receipt.done() - assert (meter.pending_records, meter.pending_bytes) == (2, reserved) - with pytest.raises(UsageCaptureError, match="capacity"): - meter.submit((_write("overflow"),)) - recovered.set() - await flushing - await first.receipt - duplicate: Final = capture_callback(meter, {**kwargs, "_accounting_capture": first}, response, NOW, NOW) - await meter.close() - assert duplicate is first - assert database.rows == first.writes - assert delays.get_nowait() == 0.2 - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - - -@pytest.mark.asyncio -async def test_cut_stream_keeps_original_model_usage_as_part_of_the_parent_aggregate() -> None: - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.router import Router - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger(stream=True) - stream: Final = CustomStreamWrapper(completion_stream=iter(()), model="capture-model", logging_obj=logger) - partial: Final = ModelResponse( - id="cut", model="capture-model", usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5) - ) - try: - Router._price_cut_partial(stream, partial) - finally: - await runtime.stop(1) - assert len(database.rows) == 1 - payload: Final = json.loads(database.rows[0].envelope) - assert payload["event"]["event_id"] == "cut:included" - assert payload["parent_request_id"] == "caller" - assert payload["charge_eligible"] is False - assert payload["usage"]["total_tokens"] == 5 - assert payload["pricing"]["model_map_information"]["model_map_key"] == "capture-model" - - -@pytest.mark.asyncio -async def test_direct_mcp_logging_captures_tool_pricing_and_identity_without_arguments() -> None: - from mcp.types import CallToolResult, TextContent - - from litellm.accounting import DimensionName - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger(CallTypes.call_mcp_tool.value) - logger.model_call_details["mcp_tool_call_metadata"] = { - "name": "lookup", - "namespaced_tool_name": "server.lookup", - "arguments": {"query": "private query"}, - "mcp_server_cost_info": {"default_cost_per_query": 0.2, "tool_name_to_cost_per_query": {"lookup": 0.3}}, - } - try: - await logger.async_success_handler( - CallToolResult(content=[TextContent(type="text", text="private result")]), NOW, NOW - ) - finally: - await runtime.stop(1) - outbound: Final = database.rows[-1] - assert any(d.name is DimensionName.TOOL and d.value == "server.lookup" for d in outbound.event.dimensions) - payload: Final = json.loads(outbound.envelope) - assert payload["pricing"]["tool"] == {"name": "lookup", "cost_per_query": 0.3} - assert payload["usage"] is None - assert all(secret not in outbound.envelope for secret in ("private query", "private result")) - - -@pytest.mark.asyncio -async def test_callback_overload_is_visible_without_suppressing_legacy_tracking( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, captured_callback_kwargs - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(max_pending_records=2, batch_size=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - kwargs: Final = {"call_type": "acompletion", "litellm_call_id": "one", "litellm_params": {}} - try: - for start, end in ((None, NOW), (NOW, None)): - missing_timestamp: Final = await captured_callback_kwargs(kwargs, {"id": "one"}, start, end) - assert missing_timestamp == kwargs - assert "Usage capture requires callback start and end timestamps" in caplog.text - admitted: Final = await captured_callback_kwargs(kwargs, {"id": "one"}, NOW, NOW) - await database.entered.wait() - rejected: Final = await captured_callback_kwargs({**kwargs, "litellm_call_id": "two"}, {"id": "two"}, NOW, NOW) - assert "_accounting_capture" in admitted - assert "_accounting_capture" not in rejected - assert "Usage capture failed before legacy cost tracking" in caplog.text - finally: - database.release.set() - await runtime.stop(1) - assert database.committed == ("one:inbound", "one") - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("call_type", "streaming", "background"), - [("acompletion", True, False), ("aresponses", False, True), ("aocr", False, False)], -) -@pytest.mark.parametrize("overloaded", [False, True]) -async def test_async_capture_preserves_response_without_commit_ack( - call_type: str, streaming: bool, background: bool, overloaded: bool, monkeypatch: pytest.MonkeyPatch -) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr( - proxy_server, - "general_settings", - { - "accounting_sync_ack": True, - "accounting_sync_ack_timeout_seconds": 0, - }, - ) - meter: Final = runtime.meter - assert meter is not None - if overloaded: - meter.submit((_write("occupied-one"), _write("occupied-two"))) - try: - await capture_proxy_response( - _logger(call_type, stream=streaming), - ModelResponse(id="provider", usage=Usage(prompt_tokens=3)), - background=background, - ) - assert database.committed == () - assert meter.pending_records == 2 - finally: - database.release.set() - await runtime.stop(1) - assert database.committed == (("occupied-one", "occupied-two") if overloaded else ("caller:inbound", "provider")) - - -@pytest.mark.asyncio -async def test_new_connection_turn_does_not_reuse_the_previous_turns_capture() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - try: - logger.capture_usage_before_logging(ModelResponse(id="first", usage=Usage(prompt_tokens=2)), NOW, NOW) - next_turn: Final = logger.copy_for_new_call("second-caller", NOW) - next_turn.capture_usage_before_logging(ModelResponse(id="second", usage=Usage(prompt_tokens=4)), NOW, NOW) - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ( - "caller:inbound", - "first", - "second-caller:inbound", - "second", - ) - assert json.loads(database.rows[-1].envelope)["usage"]["prompt_tokens"] == 4 - - -@pytest.mark.asyncio -async def test_billed_failure_is_captured_before_legacy_persistence_gates(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "prisma_client", None) - request: Final = { - "call_type": "acompletion", - "litellm_call_id": "failed", - "model": "capture-model", - "start_time": NOW, - "combined_usage_object": Usage(prompt_tokens=3, total_tokens=3), - "response_cost": 0.2, - "litellm_params": {}, - } - try: - await ProxyDBLogger().async_post_call_failure_hook(request, RuntimeError("provider failed"), UserAPIKeyAuth()) - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ("failed:inbound", "failed") - payload: Final = json.loads(database.rows[-1].envelope) - assert payload["event"]["outcome"] == "failed" - assert payload["usage"]["total_tokens"] == 3 - assert payload["charge_eligible"] is True - assert payload["pricing"]["reported_cost"] == 0.2 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("pages", [None, 0, 3]) -async def test_prepared_ocr_capture_preserves_reported_pages(pages: int | None) -> None: - from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger("aocr") - response: Final = OCRResponse(pages=[], model="capture-model", usage_info=OCRUsageInfo(pages_processed=pages)) - logger.prepare_response_cost(response, NOW) - capture_logging_usage(meter, logger, response, NOW, NOW) - await meter.close() - envelope: Final = json.loads(database.rows[-1].envelope) - assert (None if envelope["usage"] is None else envelope["usage"]["pages_processed"]) == pages - assert envelope["usage_source"] == ("unknown" if pages is None else "measured") - assert response.usage_info is not None and response.usage_info.pages_processed == pages - - -@pytest.mark.asyncio -@pytest.mark.parametrize("measured", [None, {"total_tokens": 0}, {"total_tokens": 3}]) -async def test_batch_capture_provenance_tracks_observations_not_envelope_presence( - measured: Mapping[str, int] | None, -) -> None: - from litellm.proxy.spend_tracking.usage_capture import capture_callback - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - usage: Final = { - "total_tokens": 0 if measured is None else measured["total_tokens"], - "batch_usage_by_model": {"missing-model": None, "other-model": measured}, - "batch_unknown_usage_requests": 2 if measured is None else 1, - } - capture_callback( - meter, - {"call_type": "aretrieve_batch", "litellm_call_id": "caller", "litellm_params": {}}, - {"id": "batch", "usage": usage}, - NOW, - NOW, - ) - await meter.close() - envelope: Final = json.loads(database.rows[-1].envelope) - assert envelope["usage_source"] == ("unknown" if measured is None else "measured") - assert envelope["usage"]["batch_usage_by_model"] == usage["batch_usage_by_model"] - assert database.rows[-1].event.metrics == ( - None if measured is None else (Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(measured["total_tokens"])),) - ) - - -@pytest.mark.asyncio -async def test_ack_admission_failure_does_not_turn_provider_success_into_failed_usage( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from fastapi import HTTPException - - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings(batch_size=2, max_pending_records=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) - monkeypatch.setattr(proxy_server, "prisma_client", None) - meter: Final = runtime.meter - assert meter is not None - logger: Final = _logger() - try: - meter.submit((_write("occupied-one"), _write("occupied-two"))) - with pytest.raises(HTTPException) as error: - await capture_proxy_response(logger, ModelResponse(id="caller", usage=Usage(prompt_tokens=3))) - assert error.value.status_code == 503 - await meter.flush() - request: Final = {**logger.model_call_details, "litellm_logging_obj": logger, "start_time": NOW} - await ProxyDBLogger().async_post_call_failure_hook(request, error.value, UserAPIKeyAuth()) - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ("occupied-one", "occupied-two") - - -@pytest.mark.asyncio -async def test_response_ack_timeout_preserves_success_and_later_commits(monkeypatch: pytest.MonkeyPatch) -> None: - from fastapi import HTTPException - - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=2)) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "prisma_client", None) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) - logger: Final = _logger() - - async def expired_deadline(future: object, timeout: float) -> None: - raise TimeoutError("controlled response deadline") - - try: - with monkeypatch.context() as clock: - clock.setattr(asyncio, "wait_for", expired_deadline) - with pytest.raises(HTTPException) as error: - await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3))) - assert error.value.status_code == 503 - await database.entered.wait() - captured: Final = logger.model_call_details["_accounting_capture"] - request: Final = {**logger.model_call_details, "litellm_logging_obj": logger, "start_time": NOW} - await ProxyDBLogger().async_post_call_failure_hook(request, error.value, UserAPIKeyAuth()) - database.release.set() - await captured.receipt - assert tuple(row.event.outcome for row in captured.writes) == (Outcome.SUCCEEDED, Outcome.SUCCEEDED) - assert database.committed == ("caller:inbound", "provider") - assert database.transactions == 1 - finally: - await runtime.stop(1) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("call_type", ["generate_content", "agenerate_content"]) -@pytest.mark.parametrize("tokens", [None, 0, 7]) -async def test_prepared_native_google_capture_preserves_reported_usage( - call_type: str, tokens: int | None, monkeypatch: pytest.MonkeyPatch -) -> None: - import httpx - - import litellm - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger(call_type) - monkeypatch.setitem( - litellm.model_cost, - "capture-model", - {"input_cost_per_token": 0.1, "output_cost_per_token": 0.2, "litellm_provider": "openai", "mode": "chat"}, - ) - reported: Final = ( - {"trafficType": "ON_DEMAND"} - if tokens is None - else { - "promptTokenCount": tokens, - "candidatesTokenCount": tokens, - "totalTokenCount": 2 * tokens, - "cachedContentTokenCount": tokens, - "promptTokensDetails": [{"modality": "DOCUMENT", "tokenCount": tokens}], - "cacheTokensDetails": [{"modality": "document", "token_count": tokens}], - } - ) - response: Final = { - "responseId": "native-google", - "candidates": [{"content": {"parts": [{"text": "answer"}], "role": "model"}, "finishReason": "STOP"}], - "usageMetadata": reported, - } - native_response: Final = httpx.Response(200, json=response) - logger.model_call_details["httpx_response"] = native_response - prepared: Final = logger.prepare_response_cost(response, NOW) - assert prepared is not None - capture_logging_usage(meter, logger, response, NOW, NOW) - await meter.close() - envelope: Final = json.loads(database.rows[-1].envelope) - assert (None if envelope["usage"] is None else envelope["usage"]["total_tokens"]) == ( - None if tokens is None else 2 * tokens - ) - assert (None if envelope["usage"] is None else envelope["usage"]["prompt_tokens_details"]["text_tokens"]) == tokens - assert ( - None - if envelope["usage"] is None - else envelope["usage"]["prompt_tokens_details"]["cached_tokens_details"]["text_tokens"] - ) == tokens - assert envelope["usage_source"] == ("unknown" if tokens is None else "measured") - assert response == native_response.json() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("rejected", [False, True]) -async def test_retry_links_only_admitted_usage_and_preserves_later_measurement(rejected: bool) -> None: - from litellm.litellm_core_utils.litellm_logging import UsageObservation - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings(max_pending_records=4)) - logger: Final = _logger() - assert runtime.meter is not None - try: - if rejected: - runtime.meter.submit(tuple(_write(f"occupied-{index}") for index in range(4))) - logger.model_call_details["_accounting_usage_observed"] = UsageObservation.UNKNOWN - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5, total_tokens=5), 0.4) - logger.carry_failed_attempt_usage() - await runtime.meter.flush() - logger.capture_usage_before_logging(ModelResponse(id="success", usage=Usage(prompt_tokens=3)), NOW, NOW) - finally: - await runtime.stop(1) - successful: Final = json.loads(next(row.envelope for row in database.rows if row.event.event_id == "success")) - assert successful.get("related_attempt_ids", successful["included_attempt_ids"]) == ( - [] if rejected else ["caller-attempt-0"] - ) - assert successful["usage"]["prompt_tokens"] == 3 - if not rejected: - failed: Final = json.loads( - next(row.envelope for row in database.rows if row.event.event_id == "caller-attempt-0") - ) - assert failed["usage"]["prompt_tokens"] == 5 - assert failed["usage_source"] == "measured" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("current_tokens", [None, 0, 3]) -async def test_final_failure_captures_current_attempt_without_legacy_retry_aggregate( - current_tokens: int | None, -) -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback - from litellm.proxy.utils import _failure_fields_to_lift - from litellm.types.utils import Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - try: - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5, total_tokens=5), 0.4) - logger.carry_failed_attempt_usage() - if current_tokens is not None: - logger.record_partial_usage_for_failure( - Usage(prompt_tokens=current_tokens, total_tokens=current_tokens), 0.2 - ) - lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logger}, provider_billed=True) - assert lifted["combined_usage_object"].total_tokens == 5 + (current_tokens or 0) - assert runtime.meter is not None - capture_callback( - runtime.meter, - {**logger.model_call_details, **lifted, "log_event_type": "failed_api_call"}, - None, - NOW, - NOW, - ) - finally: - await runtime.stop(1) - final: Final = json.loads(next(row.envelope for row in database.rows if row.event.event_id == "caller")) - assert (None if final["usage"] is None else final["usage"]["total_tokens"]) == current_tokens - assert final["usage_source"] == ("unknown" if current_tokens is None else "measured") - assert final["pricing"]["reported_cost"] == (None if current_tokens is None else 0.2) - assert final["included_attempt_ids"] == [] - assert final["related_attempt_ids"] == ["caller-attempt-0"] - - -@pytest.mark.asyncio -async def test_estimated_final_attempt_keeps_its_provenance_after_measured_retry() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback - from litellm.proxy.utils import _failure_fields_to_lift - from litellm.types.utils import Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - logger.messages = [{"role": "user", "content": "hello"}] - logger.model_call_details["messages"] = logger.messages - try: - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5, total_tokens=5), 0.4) - logger.carry_failed_attempt_usage() - logger.model_call_details["current_attempt_api_call_start_time"] = NOW - lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logger}, provider_billed=True) - assert runtime.meter is not None - capture_callback(runtime.meter, {**logger.model_call_details, **lifted}, None, NOW, NOW) - finally: - await runtime.stop(1) - final: Final = json.loads(next(row.envelope for row in database.rows if row.event.event_id == "caller")) - assert final["usage_source"] == "estimated" - assert final["usage"]["total_tokens"] > 0 - assert lifted["combined_usage_object"].total_tokens == final["usage"]["total_tokens"] + 5 - assert final["pricing"]["reported_cost"] == 0 - - -@pytest.mark.asyncio -async def test_parent_receipt_cannot_acknowledge_when_a_linked_attempt_conflicts() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - try: - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5), 0.4) - logger.carry_failed_attempt_usage() - assert runtime.meter is not None - await runtime.meter.flush() - other: Final = _logger() - other.record_partial_usage_for_failure(Usage(prompt_tokens=8), 0.8) - other.carry_failed_attempt_usage() - other.capture_usage_before_logging(ModelResponse(id="success", usage=Usage(prompt_tokens=3)), NOW, NOW) - captured: Final = other.model_call_details["_accounting_capture"] - await runtime.meter.flush() - with pytest.raises(UsageCaptureError, match="dependency"): - await captured.receipt - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ("caller-attempt-0",) - - -@pytest.mark.asyncio -async def test_unserializable_observation_does_not_interrupt_retry_or_legacy_usage() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = StoredUsage() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.insert, UsageCaptureSettings()) - logger: Final = _logger() - usage: Final = Usage(prompt_tokens=5, total_tokens=5, opaque_usage=object()) - try: - logger.record_partial_usage_for_failure(usage, 0.4) - assert logger.model_call_details["combined_usage_object"] is usage - logger.carry_failed_attempt_usage() - assert logger.model_call_details["failed_attempts_usage"] == ((usage, 0.4),) - logger.capture_usage_before_logging(ModelResponse(id="success", usage=Usage(prompt_tokens=3)), NOW, NOW) - finally: - await runtime.stop(1) - assert tuple(row.event.event_id for row in database.rows) == ("caller:inbound", "success") - assert json.loads(database.rows[-1].envelope)["related_attempt_ids"] == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("prior_committed", [False, True]) -@pytest.mark.parametrize("batch_size", [2, 3]) -async def test_retry_dependencies_do_not_resubmit_usage_or_expand_final_submission( - prior_committed: bool, batch_size: int -) -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.types.utils import ModelResponse, Usage - - database: Final = CommitBarrier() - database.release.set() - runtime: Final = UsageCaptureRuntime() - runtime.start(database.write, UsageCaptureSettings(batch_size=batch_size, max_pending_records=3)) - logger: Final = _logger() - try: - logger.record_partial_usage_for_failure(Usage(prompt_tokens=5), 0.4) - logger.carry_failed_attempt_usage() - assert runtime.meter is not None - if prior_committed: - await runtime.meter.flush() - logger.capture_usage_before_logging(ModelResponse(id="success", usage=Usage(prompt_tokens=3)), NOW, NOW) - captured: Final = logger.model_call_details.get("_accounting_capture") - assert captured is not None, "Final usage must be admitted independently of prior usage batch size" - await runtime.meter.close() - await captured.receipt - assert runtime.meter.pending_records == 0 - finally: - await runtime.stop(1) - assert database.committed == ("caller-attempt-0", "caller:inbound", "success") - final: Final = json.loads(captured.writes[-1].envelope) - assert final["related_attempt_ids"] == ["caller-attempt-0"] - assert final["usage"]["prompt_tokens"] == 3 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("prepared", [False, True]) -@pytest.mark.parametrize( - ("call_type", "response", "field", "quantity"), - [ - ("arerank", {"id": "rerank", "results": [], "meta": {"tokens": {"input_tokens": 7}}}, "prompt_tokens", 7), - ("aocr", {"id": "ocr", "usage_info": {"pages_processed": 3}}, "pages_processed", 3), - ( - "acompletion", - {"id": "google", "usageMetadata": {"promptTokenCount": 7, "totalTokenCount": 7}}, - "prompt_tokens", - 7, - ), - ], -) -async def test_capture_preserves_native_usage_before_and_after_preparation( - prepared: bool, call_type: str, response: dict[str, object], field: str, quantity: int -) -> None: - from litellm._internal_context import internal_sub_call - from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage - - database: Final = StoredUsage() - meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger(call_type) - logger.model_call_details["response_cost"] = 0.25 - if prepared: - assert logger.prepare_response_cost(response, NOW) is not None - with internal_sub_call("parent"): - capture_logging_usage(meter, logger, response, NOW, NOW) - await meter.close() - assert len(database.rows) == 1 - envelope: Final = json.loads(database.rows[0].envelope) - assert envelope["usage"][field] == quantity - assert envelope["usage_source"] == "measured" - assert envelope["ownership"] == "included_in_parent" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokens", [(120000, 120000), (210000, 30000)]) -@pytest.mark.parametrize("priced", [False, True]) -async def test_batch_capture_retains_request_basis_for_later_tiered_pricing( - tokens: tuple[int, int], priced: bool, monkeypatch: pytest.MonkeyPatch -) -> None: - import litellm - from litellm.batches.batch_utils import batch_cost_and_usage - from litellm.cost_calculator import batch_cost_calculator - from litellm.proxy.spend_tracking.usage_capture import ( - UsageBatchWriter, - capture_batch_callback, - capture_logging_usage, - ) - from litellm.types.utils import Usage - - rates: Final = { - "input_cost_per_token_batches": 0.001, - "input_cost_per_token_above_200k_tokens_batches": 0.002, - "output_cost_per_token_batches": 0.0, - "litellm_provider": "openai", - "mode": "chat", - } - monkeypatch.setitem(litellm.model_cost, "capture-model", rates if priced else {"litellm_provider": "openai"}) - rows: Final = [ - { - "response": { - "status_code": 200, - "body": { - "model": "capture-model", - "usage": {"prompt_tokens": n, "completion_tokens": 0, "total_tokens": n}, - }, - } - } - for n in tokens - ] - result: Final = batch_cost_and_usage(rows, "openai", model_info=rates if priced else {}) - database: Final = StoredUsage() - writer: Final = UsageBatchWriter(database.insert, database.load) - meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - logger: Final = _logger("aretrieve_batch") - logger.model_call_details["response_cost"] = result.cost - capture_logging_usage(meter, logger, {"id": "batch", "status": "completed", "usage": result.usage}, NOW, NOW) - await capture_batch_callback( - meter, - database.load, - logger.model_call_details, - {"id": "batch", "status": "completed", "usage": result.usage}, - NOW, - NOW, - ) - await meter.close() - retained: Final = tuple( - {"model": "capture-model", "usage": json.loads(write.envelope)["usage"]} for write in database.rows[1:] - ) - assert sorted(row["usage"]["prompt_tokens"] for row in retained) == sorted(tokens) - reconstructed: Final = sum( - sum(batch_cost_calculator(Usage(**row["usage"]), row["model"], "openai", model_info=rates)) for row in retained - ) - expected: Final = sum(n * (0.002 if n > 200000 else 0.001) for n in tokens) - assert reconstructed == pytest.approx(expected) - assert result.cost == pytest.approx(expected if priced else 0) - assert database.rows[0].event.metrics is None - assert json.loads(database.rows[0].envelope)["charge_eligible"] is False - assert all(json.loads(write.envelope)["finalized_cost"] is None for write in database.rows[1:]) - - -@pytest.mark.asyncio -async def test_large_batch_recovers_partial_commit_with_frozen_pricing_and_rejects_changed_usage() -> None: - from litellm.proxy.spend_tracking.usage_capture import ( - PricingCapture, - UsageBatchWriter, - capture_batch_callback, - capture_callback, - ) - - database: Final = StoredUsage() - transactions: Final = asyncio.Queue[tuple[UsageWrite, ...]]() - failure: Final = asyncio.Queue[bool]() - failure.put_nowait(True) - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - assert sum(len(write.envelope.encode()) for write in writes) <= 65536 - assert len(writes) <= 32 - if len(database.rows) > 1 and not failure.empty(): - failure.get_nowait() - raise RuntimeError("Interrupted batch persistence") - await database.insert(writes) - transactions.put_nowait(writes) - - writer: Final = UsageBatchWriter(persist, database.load) - meter: Final = DurableUsageMeter( - writer.write, - UsageCaptureSettings(batch_size=32, max_pending_records=32, max_pending_bytes=65536), - wait_for_batch=_manual_flush, - ) - rows: Final = tuple( - { - "model": "capture-model" if index % 7 else None, - "usage": {"prompt_tokens": index % 5 + 1} if index % 7 else None, - } - for index in range(1000) - ) - response: Final = {"id": "large-batch", "status": "completed", "usage": {"batch_usage_by_request": rows}} - pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"capture-model": {"rate": 1}}})) - kwargs: Final = { - "call_type": "aretrieve_batch", - "litellm_call_id": "poll", - "litellm_params": {}, - "_accounting_pricing_snapshot": pricing, - } - failed_database: Final = CommitBarrier(failure=RuntimeError("Earlier capture failed")) - failed_database.release.set() - prior_meter: Final = DurableUsageMeter(failed_database.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) - prior: Final = capture_callback( - prior_meter, - {"call_type": "acompletion", "litellm_call_id": "earlier", "litellm_params": {}}, - {"id": "earlier"}, - NOW, - NOW, - ) - await prior_meter.close() - with pytest.raises(RuntimeError, match="Earlier capture failed"): - await capture_batch_callback( - meter, - database.load, - {**kwargs, "_accounting_prior_captures": (prior,)}, - response, - NOW, - NOW, - ) - assert database.rows == () - with pytest.raises(RuntimeError, match="Interrupted batch persistence"): - await capture_batch_callback(meter, database.load, kwargs, response, NOW, NOW) - partial: Final = database.rows - assert 1 < len(partial) < len(rows) + 1 - changed_pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"capture-model": {"rate": 2}}})) - retried: Final = await capture_batch_callback( - meter, - database.load, - {**kwargs, "_accounting_pricing_snapshot": changed_pricing}, - {**response, "usage": {"batch_usage_by_request": tuple(reversed(rows))}}, - NOW, - NOW, - ) - assert retried.receipt.done() and retried.receipt.exception() is None - assert len(database.rows) == len(rows) + 1 - assert database.rows[: len(partial)] == partial - envelopes: Final = tuple(json.loads(write.envelope) for write in database.rows[1:]) - assert all( - envelope["pricing"]["model_map_information"] == ({"rate": 1} if envelope["usage"] is not None else None) - for envelope in envelopes - ) - assert sum(envelope["usage"] is None for envelope in envelopes) == sum(row["usage"] is None for row in rows) - assert all(envelope["request_count_eligible"] is False for envelope in envelopes) - assert sum(envelope["usage"]["prompt_tokens"] for envelope in envelopes if envelope["usage"] is not None) == sum( - row["usage"]["prompt_tokens"] for row in rows if row["usage"] is not None - ) - before_conflict: Final = database.rows - with pytest.raises(UsageCaptureError, match="identity conflicts"): - await capture_batch_callback( - meter, - database.load, - kwargs, - {**response, "usage": {"batch_usage_by_request": rows[:-1]}}, - NOW, - NOW, - ) - await meter.close() - assert database.rows == before_conflict - assert transactions.qsize() > 2 - assert meter.pending_bytes == meter.pending_records == 0 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("outcome", ["drain", "cancel", "failure"]) -async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence( - outcome: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, UsageCaptureRuntime, captured_callback_kwargs - - database: Final = StoredUsage() - entered: Final = asyncio.Event() - release: Final = asyncio.Event() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - entered.set() - await release.wait() - if outcome == "failure": - raise RuntimeError("Persistence unavailable") - await database.insert(writes) - - runtime: Final = UsageCaptureRuntime() - runtime.start(persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), load=database.load) - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} - response: Final = { - "id": "batch", - "status": "completed", - "usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},) * 3}, - } - callback: Final = asyncio.create_task(captured_callback_kwargs(kwargs, response, NOW, NOW)) - try: - await entered.wait() - assert callback.done(), "Batch persistence must not consume the legacy callback deadline" - captured: Final = (await callback)["_accounting_capture"] - assert isinstance(captured, CapturedUsage) and not captured.receipt.done() - second: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW) - assert isinstance(second.get("_accounting_capture"), CapturedUsage) - overflow: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW) - assert "_accounting_capture" not in overflow - assert "Batch capture producer capacity exhausted" in caplog.text - if outcome != "cancel": - release.set() - await runtime.stop(0 if outcome == "cancel" else 1) - assert captured.receipt.done() - assert captured.receipt.cancelled() is (outcome == "cancel") - if outcome == "failure": - assert isinstance(captured.receipt.exception(), RuntimeError) - assert "Batch usage capture failed" in caplog.text - assert len(database.rows) == (4 if outcome == "drain" else 0) - with pytest.raises(UsageCaptureError, match="active ledger writer"): - runtime.submit_batch(kwargs, response, NOW, NOW) - finally: - release.set() - await asyncio.gather(callback, return_exceptions=True) - await runtime.stop(1) - - -@pytest.mark.asyncio -async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() -> None: - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - - database: Final = StoredUsage() - release: Final = asyncio.Event() - runtime: Final = UsageCaptureRuntime() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - await release.wait() - assert runtime.meter is not None - assert runtime.meter.pending_bytes <= 65536 - await database.insert(writes) - - runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load) - from litellm.batches.batch_utils import batch_cost_and_usage - - parsed: Final = batch_cost_and_usage( - [ - { - "response": { - "status_code": 200, - "body": { - "model": "capture-model", - "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, - }, - } - } - for _ in range(1000) - ], - "openai", - model_info={}, - ) - rows: Final = ({"model": "capture-model", "usage": {"prompt_tokens": 2}},) - response: Final = {"id": "batch", "status": "completed", "usage": parsed.usage} - kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} - try: - large: Final = asyncio.Queue[asyncio.Future[None]]() - for index in range(100): - try: - captured: Final = runtime.submit_batch(kwargs, {**response, "id": f"batch-{index}"}, NOW, NOW) - except UsageCaptureError: - break - large.put_nowait(captured.receipt) - large_count: Final = large.qsize() - assert 0 < large_count < 100 - assert runtime.meter is not None and 0 < runtime.meter.pending_bytes <= 65536 - release.set() - await asyncio.gather(*(large.get_nowait() for _ in range(large_count))) - assert len(database.rows) == 1001 * large_count - observations: Final = tuple(json.loads(write.envelope) for write in database.rows) - assert sum(row["usage"].get("prompt_tokens", 0) for row in observations) == 2000 * large_count - assert sum(row["usage"].get("completion_tokens", 0) for row in observations) == 1000 * large_count - assert runtime.meter.pending_bytes == 0 - admitted: Final = asyncio.Queue[asyncio.Future[None]]() - for index in range(100): - try: - small: Final = runtime.submit_batch( - kwargs, - {**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows}}, - NOW, - NOW, - ) - except UsageCaptureError: - break - admitted.put_nowait(small.receipt) - count: Final = admitted.qsize() - assert 1 < count < 100 - await asyncio.gather(*(admitted.get_nowait() for _ in range(count))) - assert len(database.rows) == 1001 * large_count + 2 * count - assert runtime.meter.pending_bytes == 0 - finally: - release.set() - await runtime.stop(1) - - -@pytest.mark.asyncio -async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> None: - from litellm.proxy.spend_tracking.usage_capture import PricingCapture, UsageBatchWriter, UsageCaptureRuntime - - database: Final = StoredUsage() - writer: Final = UsageBatchWriter(database.insert, database.load) - runtime: Final = UsageCaptureRuntime() - runtime.start(writer.write, UsageCaptureSettings(max_pending_bytes=16384), load=database.load) - response: Final = { - "id": "batch", - "status": "completed", - "usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},)}, - } - kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} - pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"model": {"source": "x" * 1200}}})) - try: - initial: Final = runtime.submit_batch({**kwargs, "_accounting_pricing_snapshot": pricing}, response, NOW, NOW) - await asyncio.shield(initial.receipt) - accepted: Final = database.rows - replay: Final = runtime.submit_batch(kwargs, response, NOW, NOW) - await asyncio.shield(replay.receipt) - assert database.rows == accepted and len(accepted) == 2 - assert runtime.meter is not None and runtime.meter.pending_bytes == 0 - finally: - await runtime.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py deleted file mode 100644 index bd3a0b9ed70..00000000000 --- a/tests/unit/proxy/spend_tracking/test_usage_meter.py +++ /dev/null @@ -1,473 +0,0 @@ -import asyncio -import json -from datetime import datetime, timezone -from decimal import Decimal -from typing import Final - -import pytest - -from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome -from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite -from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings -from litellm.types.utils import CallTypes - - -def _write(identity: str) -> UsageWrite: - event: Final = InboundUsageEvent( - event_id=identity, - request_id=identity, - call_type=CallTypes.acompletion, - started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), - ended_at=datetime(2026, 1, 1, tzinfo=timezone.utc), - outcome=Outcome.SUCCEEDED, - dimensions=(), - metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), - ) - return UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) - - -class CommitBarrier: - def __init__(self, *, conflict: str | None = None, failure: Exception | None = None) -> None: - self.entered: Final = asyncio.Event() - self.release: Final = asyncio.Event() - self.committed: tuple[str, ...] = () - self.transactions = 0 - self.conflict: Final = conflict - self.failure: Final = failure - - async def write(self, writes: tuple[UsageWrite, ...]) -> None: - self.transactions += 1 - self.entered.set() - await self.release.wait() - if self.failure is not None: - raise self.failure - identities: Final = tuple(write.event.event_id for write in writes) - if self.conflict in identities: - raise AccountingConflictError("Conflicting test event") - self.committed += identities - - -async def _manual_flush(wakeup: asyncio.Event, timeout: float) -> None: - await wakeup.wait() - - -@pytest.mark.asyncio -async def test_receipts_wait_for_one_batched_commit_and_keep_inflight_capacity_reserved() -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2)) - first: Final = meter.submit((_write("one"),)) - second: Final = meter.submit((_write("two"),)) - await database.entered.wait() - assert not first.done() and not second.done() - assert meter.pending_records == 2 - with pytest.raises(UsageCaptureError, match="capacity"): - meter.submit((_write("three"),)) - database.release.set() - await asyncio.gather(first, second) - await meter.close() - assert database.committed == ("one", "two") - assert database.transactions == 1 - assert (meter.pending_bytes, meter.pending_records) == (0, 0) - - -@pytest.mark.asyncio -async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> None: - database: Final = CommitBarrier() - write: Final = _write("large" * 300) - size: Final = len(write.envelope.encode()) - meter: Final = DurableUsageMeter( - database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush - ) - reservation: Final = meter.reserve_producer(size // 2, size // 2) - with pytest.raises(UsageCaptureError, match="producer byte capacity"): - meter.reserve_producer(1, size) - with pytest.raises(UsageCaptureError, match="producer byte capacity"): - meter.resize_producer(reservation, size, 1) - assert meter.pending_bytes == size // 2 - meter.release_producer(reservation) - receipt: Final = meter.submit((write,)) - with pytest.raises(UsageCaptureError, match="capacity"): - meter.submit((_write("extra"),)) - assert meter.pending_bytes == size - database.release.set() - await meter.close() - await receipt - assert database.committed == (write.event.event_id,) - with pytest.raises(UsageCaptureError, match="closed"): - meter.submit((_write("after-close"),)) - with pytest.raises(UsageCaptureError, match="producer byte capacity"): - meter.reserve_producer(1, 1) - - -@pytest.mark.asyncio -async def test_conflicting_group_rolls_back_and_valid_neighbor_still_commits() -> None: - database: Final = CommitBarrier(conflict="conflict") - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3)) - rejected: Final = meter.submit((_write("caller"), _write("conflict"))) - accepted: Final = meter.submit((_write("valid"),)) - await database.entered.wait() - database.release.set() - with pytest.raises(UsageCaptureError, match="identity"): - await rejected - await accepted - await meter.close() - assert database.committed == ("valid",) - assert meter.pending_records == 0 - - -@pytest.mark.asyncio -async def test_permanent_persistence_failure_reaches_each_receipt_and_releases_capacity() -> None: - failure: Final = ValueError("invalid envelope") - database: Final = CommitBarrier(failure=failure) - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) - first: Final = meter.submit((_write("one"),)) - second: Final = meter.submit((_write("two"),)) - await database.entered.wait() - database.release.set() - results: Final = await asyncio.gather(first, second, return_exceptions=True) - await meter.close() - assert results == [failure, failure] - assert meter.pending_bytes == 0 - - -@pytest.mark.asyncio -async def test_cancelling_record_waiter_does_not_cancel_persistence() -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) - waiting: Final = asyncio.create_task(meter.record(_write("caller").event)) - await database.entered.wait() - waiting.cancel() - with pytest.raises(asyncio.CancelledError): - await waiting - database.release.set() - await meter.close() - assert database.committed == ("caller",) - assert meter.pending_records == 0 - - -@pytest.mark.asyncio -async def test_shutdown_deadline_rejects_unacknowledged_work_and_releases_capacity() -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) - first: Final = meter.submit((_write("in-flight"), _write("in-flight-two"))) - await database.entered.wait() - queued: Final = meter.submit((_write("queued"),)) - await meter.stop(0) - results: Final = await asyncio.gather(first, queued, return_exceptions=True) - assert all(isinstance(result, UsageCaptureError) for result in results) - assert meter.pending_records == 0 - assert meter.pending_bytes == 0 - assert database.committed == () - - -@pytest.mark.asyncio -async def test_shutdown_during_database_backoff_releases_reserved_capacity() -> None: - retrying: Final = asyncio.Event() - blocked: Final = asyncio.Event() - - async def unavailable(writes: tuple[UsageWrite, ...]) -> None: - raise ConnectionError("database unavailable") - - async def backoff(delay: float) -> None: - retrying.set() - await blocked.wait() - - meter: Final = DurableUsageMeter(unavailable, UsageCaptureSettings(batch_size=2), retry_delay=backoff) - receipt: Final = meter.submit((_write("one"), _write("two"))) - await retrying.wait() - await meter.stop(0) - with pytest.raises(UsageCaptureError, match="interrupted"): - await receipt - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("wrapped", [False, True]) -async def test_closed_pooled_connection_retains_batch_until_automatic_recovery(wrapped: bool) -> None: - import httpx - from prisma.engine.errors import EngineRequestError - from prisma.http import Response - - database: Final = CommitBarrier() - attempts: Final = asyncio.Queue[tuple[UsageWrite, ...]]() - retrying: Final = asyncio.Event() - recovered: Final = asyncio.Event() - closure: Final = EngineRequestError( - Response(httpx.Response(500)), '{"error_code":"P1017","message":"Server has closed the connection."}' - ) - failure: Final = RuntimeError("ledger transaction failed") if wrapped else closure - if wrapped: - failure.__cause__ = closure - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - attempts.put_nowait(writes) - if attempts.qsize() == 1: - raise failure - await database.write(writes) - - async def backoff(delay: float) -> None: - retrying.set() - await recovered.wait() - - meter: Final = DurableUsageMeter( - persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), retry_delay=backoff - ) - writes: Final = (_write("caller"), _write("attempt")) - receipt: Final = meter.submit(writes) - retry_signal: Final = asyncio.create_task(retrying.wait()) - reserved: Final = meter.pending_bytes - try: - await asyncio.wait((receipt, retry_signal), return_when=asyncio.FIRST_COMPLETED) - assert not receipt.done(), "A dropped pooled connection must retain admitted usage for retry" - assert (meter.pending_records, meter.pending_bytes) == (2, reserved) - with pytest.raises(UsageCaptureError, match="capacity"): - meter.submit((_write("overflow"),)) - database.release.set() - recovered.set() - await receipt - assert attempts.get_nowait() == attempts.get_nowait() == writes - assert database.committed == ("caller", "attempt") - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - finally: - retry_signal.cancel() - await asyncio.gather(retry_signal, return_exceptions=True) - await meter.stop(0) - await asyncio.gather(receipt, return_exceptions=True) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("first_commits", [False, True]) -async def test_shutdown_during_conflict_isolation_completes_every_receipt(first_commits: bool) -> None: - isolating: Final = asyncio.Event() - blocked: Final = asyncio.Event() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - if len(writes) > 1: - raise AccountingConflictError("one producer conflicts") - if first_commits and writes[0].event.event_id == "one": - return - isolating.set() - await blocked.wait() - - meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=2)) - first: Final = meter.submit((_write("one"),)) - second: Final = meter.submit((_write("two"),)) - await isolating.wait() - await meter.stop(0) - results: Final = await asyncio.gather(first, second, return_exceptions=True) - assert results[0] is None if first_commits else isinstance(results[0], UsageCaptureError) - assert isinstance(results[1], UsageCaptureError) - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - - -@pytest.mark.asyncio -async def test_elapsed_flush_deadline_commits_a_partial_batch() -> None: - from litellm.proxy.spend_tracking.usage_meter import _wait_for_batch - - database: Final = CommitBarrier() - database.release.set() - - async def expired_flush(wakeup: asyncio.Event, timeout: float) -> None: - assert timeout == 5.0 - await _wait_for_batch(wakeup, 0) - - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=10), wait_for_batch=expired_flush) - receipt: Final = meter.submit((_write("partial-batch"),)) - try: - await receipt - assert database.committed == ("partial-batch",) - assert database.transactions == 1 - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - finally: - await meter.stop(0) - - -@pytest.mark.asyncio -async def test_burst_drains_full_batches_without_waiting_for_partial_batch_deadline() -> None: - database: Final = CommitBarrier() - database.release.set() - waits: Final = asyncio.Queue[bool]() - - async def observe_wait(wakeup: asyncio.Event, timeout: float) -> None: - waits.put_nowait(wakeup.is_set()) - await wakeup.wait() - - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2), wait_for_batch=observe_wait) - receipts: Final = tuple(meter.submit((_write(str(index)),)) for index in range(5)) - try: - assert await waits.get() is True - await asyncio.gather(*receipts[:2]) - assert await waits.get() is True, "A full queued batch must not wait for the flush deadline" - await asyncio.gather(*receipts[2:4]) - assert await waits.get() is False - assert database.committed == ("0", "1", "2", "3") - assert meter.pending_records == 1 - assert not receipts[-1].done() - await meter.close() - await receipts[-1] - assert database.committed == ("0", "1", "2", "3", "4") - assert database.transactions == 3 - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - finally: - await meter.stop(0) - await asyncio.gather(*receipts, return_exceptions=True) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("sqlstate", ["22P05", "22003", "42P01"]) -async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqlstate: str) -> None: - from prisma.errors import DataError - - database: Final = CommitBarrier() - database.release.set() - attempts: Final = asyncio.Queue[tuple[str, ...]]() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - identities: Final = tuple(write.event.event_id for write in writes) - attempts.put_nowait(identities) - if "invalid" in identities or sqlstate == "42P01": - raise DataError( - data={"user_facing_error": {"message": "database rejected input", "meta": {"code": sqlstate}}} - ) - await database.write(writes) - - meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=3)) - rejected: Final = meter.submit((_write("caller"), _write("invalid"))) - neighbor: Final = meter.submit((_write("neighbor"),)) - try: - with pytest.raises(DataError): - await rejected - if sqlstate == "42P01": - with pytest.raises(DataError): - await neighbor - assert attempts.qsize() == 1 - else: - await neighbor - assert database.committed == ("neighbor",) - assert attempts.qsize() == 3 - finally: - await meter.close() - assert (meter.pending_records, meter.pending_bytes) == (0, 0) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("dependency_outcome", ["commit", "failure", "cancel"]) -async def test_commit_dependencies_preserve_capacity_and_isolate_failed_dependents(dependency_outcome: str) -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3, max_pending_records=4)) - prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() - dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) - neighbor: Final = meter.submit((_write("neighbor"),)) - assert meter.pending_records == 3 - assert not dependent.done() - with pytest.raises(UsageCaptureError, match="capacity"): - meter.submit((_write("excess-one"), _write("excess-two"))) - if dependency_outcome == "commit": - prerequisite.set_result(None) - elif dependency_outcome == "failure": - prerequisite.set_exception(ValueError("dependency failed")) - else: - prerequisite.cancel() - database.release.set() - await meter.close() - await neighbor - if dependency_outcome == "commit": - await dependent - assert database.committed == ("parent", "final", "neighbor") - else: - with pytest.raises(UsageCaptureError, match="dependency"): - await dependent - assert database.committed == ("neighbor",) - assert meter.pending_records == 0 - assert meter.pending_bytes == 0 - - -@pytest.mark.asyncio -async def test_shutdown_does_not_cancel_dependency_receipt_or_leak_reserved_capacity() -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) - prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() - dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) - await meter.stop(0) - with pytest.raises(UsageCaptureError, match="acknowledgement"): - await dependent - assert not prerequisite.done() - assert database.committed == () - assert meter.pending_records == 0 - assert meter.pending_bytes == 0 - prerequisite.set_result(None) - - -@pytest.mark.asyncio -async def test_large_write_sequence_drains_within_byte_and_record_bounds() -> None: - database: Final = CommitBarrier() - database.release.set() - meter: Final = DurableUsageMeter( - database.write, - UsageCaptureSettings(batch_size=8, max_pending_records=2, max_pending_bytes=1024), - wait_for_batch=_manual_flush, - ) - writes: Final = tuple(_write(str(index)) for index in range(20)) - pending: Final = meter.submit((_write("pending-a"), _write("pending-b"))) - await meter.record_writes(writes) - await pending - with pytest.raises(UsageCaptureError, match="One usage observation"): - await meter.record_writes((_write("oversized" * 300),)) - await meter.close() - assert database.committed == ("pending-a", "pending-b", *(write.event.event_id for write in writes)) - assert database.transactions > 1 - assert (meter.pending_bytes, meter.pending_records) == (0, 0) - with pytest.raises(UsageCaptureError): - await meter.record_writes((_write("closed"),)) - - -@pytest.mark.asyncio -async def test_cancelling_bounded_capture_preserves_admitted_writes_without_admitting_remaining_rows() -> None: - database: Final = CommitBarrier() - meter: Final = DurableUsageMeter( - database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2), wait_for_batch=_manual_flush - ) - recording: Final = asyncio.create_task(meter.record_writes(tuple(_write(str(index)) for index in range(8)))) - await database.entered.wait() - recording.cancel() - with pytest.raises(asyncio.CancelledError): - await recording - database.release.set() - await meter.close() - assert database.committed == ("0", "1") - assert meter.pending_bytes == meter.pending_records == 0 - - -@pytest.mark.asyncio -async def test_capacity_wait_retries_admission_before_other_producers_finish() -> None: - entered: Final = asyncio.Queue[tuple[str, ...]]() - release: Final = asyncio.Queue[None]() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - entered.put_nowait(tuple(write.event.event_id for write in writes)) - await release.get() - - meter: Final = DurableUsageMeter( - persist, UsageCaptureSettings(batch_size=2, max_pending_records=3), wait_for_batch=_manual_flush - ) - initial: Final = meter.submit((_write("first"), _write("second"))) - remaining: Final = meter.submit((_write("remaining"),)) - assert await entered.get() == ("first", "second") - recording: Final = asyncio.create_task(meter.record_writes((_write("target"),))) - try: - await asyncio.sleep(0) - release.put_nowait(None) - assert await entered.get() == ("remaining",) - await asyncio.sleep(0) - assert meter.pending_records == 2 - release.put_nowait(None) - assert await entered.get() == ("target",) - release.put_nowait(None) - await recording - await asyncio.gather(initial, remaining) - finally: - recording.cancel() - await asyncio.gather(recording, return_exceptions=True) - release.put_nowait(None) - release.put_nowait(None) - await meter.close() diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index a4bd47ccb74..c140641a71b 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -1,6 +1,7 @@ import sys from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager +from dataclasses import replace from types import ModuleType from typing import Final @@ -443,3 +444,45 @@ def test_root_authorization_server_metadata_is_the_gateways_not_the_byok_connect assert root.json()["token_endpoint"] == "https://gateway.example.com/token" assert connector.status_code == 200 assert connector.json()["token_endpoint"] == "https://gateway.example.com/v1/mcp/oauth/token" + + +@pytest.mark.parametrize( + "load_order", + (("/realtime/unrouted", "/mcp/unrouted"), ("/mcp/unrouted", "/realtime/unrouted")), + ids=("realtime_first", "realtime_last"), +) +def test_realtime_and_mcp_features_keep_every_route_once_in_either_load_order( + monkeypatch: pytest.MonkeyPatch, load_order: tuple[str, str] +) -> None: + monkeypatch.delenv(FLAG, raising=False) + + async def mcp_mount() -> Mapping[str, str]: + return {"route": "mcp_mount"} + + mounted_app: Final = FastAPI() + mounted_app.add_api_route("/{path:path}", mcp_mount, methods=["GET"]) + mcp_module: Final = ModuleType("tests.unit.proxy.lazy_fixture_mcp_app") + mcp_module.app = mounted_app # pyright: ignore[reportAttributeAccessIssue] # fixture module built at test time + monkeypatch.setitem(sys.modules, mcp_module.__name__, mcp_module) + features: Final = tuple( + replace(feature, module_path=mcp_module.__name__) if feature.name == "mcp_app" else feature + for feature in LAZY_FEATURES + if feature.name in ("mcp_app", "realtime") + ) + + async def mcp_proxy() -> Mapping[str, str]: + return {"route": "mcp_proxy"} + + app: Final = FastAPI() + attach_lazy_features(app, features) + app.add_api_route("/mcp/proxy", mcp_proxy, methods=["GET"]) + + with TestClient(app) as client: + for path in load_order: + client.get(path) + response: Final = client.get("/mcp/proxy") + + assert loaded_lazy_modules(app) == {feature.module_path for feature in features} + assert response.json() == {"route": "mcp_proxy"} + route_ids: Final = tuple(id(route) for route in app.routes) + assert len(route_ids) == len(frozenset(route_ids)) diff --git a/tests/unit/proxy/test__types.py b/tests/unit/proxy/test__types.py index 76ca49362b4..40e66f1da65 100644 --- a/tests/unit/proxy/test__types.py +++ b/tests/unit/proxy/test__types.py @@ -520,6 +520,39 @@ def test_mcp_metadata_rejects_unavailable_upstream_protocol(revision): model.model_validate(payload) +@pytest.mark.parametrize("request_model", (NewMCPServerRequest, UpdateMCPServerRequest)) +@pytest.mark.parametrize("value", ["true", "false", 1, 0, None]) +def test_mcp_metadata_rejects_a_non_boolean_skip_tool_output_validation( + request_model: type[NewMCPServerRequest] | type[UpdateMCPServerRequest], value: object +) -> None: + payload: Final = { + "server_id": "test", + "transport": "http", + "url": "https://example.com/mcp", + "mcp_info": {"skip_tool_output_validation": value}, + } + + with pytest.raises(ValidationError): + request_model.model_validate(payload) + + +@pytest.mark.parametrize("request_model", (NewMCPServerRequest, UpdateMCPServerRequest)) +@pytest.mark.parametrize("value", [True, False]) +def test_mcp_metadata_accepts_a_boolean_skip_tool_output_validation( + request_model: type[NewMCPServerRequest] | type[UpdateMCPServerRequest], value: bool +) -> None: + parsed: Final = request_model.model_validate( + { + "server_id": "test", + "transport": "http", + "url": "https://example.com/mcp", + "mcp_info": {"skip_tool_output_validation": value}, + } + ) + + assert parsed.mcp_info == {"skip_tool_output_validation": value} + + MCP_SERVER_REQUESTS = (NewMCPServerRequest, UpdateMCPServerRequest) STDIO_SERVER_FIELDS = {"server_id": "stdio-1", "transport": "stdio", "command": "python", "args": ["server.py"]} diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 4d1b67f9212..8143636a435 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -22,6 +22,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame from litellm.caching.caching import DualCache @@ -4293,6 +4294,62 @@ class TestHandleLLMApiExceptionPricesWhatTheServerDispatched: assert resent >= dispatched_after_trim +class TestHandleLLMApiExceptionReportsWhatAFailedAgenticFollowUpBills: + async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float: + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.types.utils import ModelResponse, Usage + + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "what is 6*7?"}]} + ) + caller: Final = ProxyUserAPIKeyAuth(request_route="/v1/chat/completions") + mock_request: Final = MagicMock(spec=Request) + mock_request.headers = {} + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + _, logging_obj = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="acompletion", + ) + logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now()) + follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + + async def failing_follow_up() -> ModelResponse: + raise follow_up_error + + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse( + model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ), + stream=False, + ) + with pytest.raises(ProxyException) as raised: + await processing_obj.handle_llm_api_exception( + e=follow_up_error + if raise_the_follow_up_error + else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o"), + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + ) + return float(raised.value.headers["x-litellm-response-cost"]) + + @pytest.mark.asyncio + async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None: + price: Final = litellm.model_cost["gpt-4o"] + + assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx( + 559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"] + ) + assert await self._error_response_cost(raise_the_follow_up_error=False) == 0 + + @pytest.mark.asyncio class TestHandleLLMApiExceptionDictDetail: """ @@ -4618,6 +4675,34 @@ class TestStreamCloseOnDisconnect: assert upstream.aclosed + async def test_async_streaming_data_generator_reraises_an_unchanged_http_exception_without_chaining_it_to_itself( + self, + ): + blocked = HTTPException(status_code=400, detail="Content blocked by guardrail") + + class FailingUpstream: + def __aiter__(self): + return self + + async def __anext__(self): + raise blocked + + ProxyLogging._callback_capabilities_cache.clear() + gen = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=FailingUpstream(), + user_api_key_dict=ProxyUserAPIKeyAuth(), + request_data={"model": "mock-model"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + serialize_chunk=lambda c: "data: x\n\n", + serialize_error=lambda e: "data: error\n\n", + ) + + with pytest.raises(HTTPException) as raised: + await gen.__anext__() + + assert raised.value is blocked + assert raised.value.__cause__ is not blocked + async def test_async_streaming_data_generator_closes_the_guardrail_chain_on_client_disconnect( self, ): @@ -5043,6 +5128,138 @@ class TestHandleLLMApiExceptionRetryAfter: assert "All deployments for selected model are in cooldown" not in body["message"] +class TestHandleLLMApiExceptionProviderOverload: + class _RecordingFailureHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.original_exception: Exception | None = None + + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.original_exception = original_exception + + async def _invoke( + self, + exc: Exception, + route_type: str, + monkeypatch: pytest.MonkeyPatch, + recording_hook: _RecordingFailureHook | None = None, + ) -> ProxyException: + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + processor.route_type = route_type + callback: Final = recording_hook if recording_hook is not None else self._RecordingFailureHook() + monkeypatch.setattr(litellm, "callbacks", [callback]) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + + with pytest.raises(ProxyException) as raised: + await processor.handle_llm_api_exception( + e=exc, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + return raised.value + + @pytest.mark.parametrize("route_type", ("acompletion", "atext_completion", "aresponses")) + async def test_openai_routes_answer_provider_overload_as_503( + self, route_type: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + + proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + @pytest.mark.parametrize( + ("route_type", "provider_status_code"), + (("acompletion", 500), ("anthropic_messages", 529)), + ) + async def test_non_overload_and_non_openai_routes_keep_their_status( + self, route_type: str, provider_status_code: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Internal server error", + llm_provider="anthropic", + model="claude", + provider_status_code=provider_status_code, + ) + + proxy_exc: Final = await self._invoke(exc, route_type, monkeypatch) + + assert proxy_exc.code == "500" + assert proxy_exc.type == "internal_server_error" + assert proxy_exc.to_dict()["code"] == "500" + + async def test_openai_route_answers_wrapped_provider_overload_as_503( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + original: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + exc: Final = MidStreamFallbackError( + message="Fallback failed", + llm_provider="anthropic", + model="claude", + original_exception=original, + ) + + proxy_exc: Final = await self._invoke(exc, "acompletion", monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + async def test_anthropic_text_api_overload_answers_503(self, monkeypatch: pytest.MonkeyPatch) -> None: + exc: Final = APIError( + status_code=529, + message="Overloaded", + llm_provider="anthropic_text", + model="claude", + ) + + proxy_exc: Final = await self._invoke(exc, "atext_completion", monkeypatch) + + assert proxy_exc.code == "503" + assert proxy_exc.type == "service_unavailable_error" + assert proxy_exc.to_dict()["code"] == "server_is_overloaded" + + @pytest.mark.parametrize( + ("route_type", "expected_status"), + (("acompletion", 503), ("anthropic_messages", 500)), + ) + async def test_failure_hook_receives_the_answered_overload_status( + self, route_type: str, expected_status: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + exc: Final = InternalServerError( + message="Overloaded", + llm_provider="anthropic", + model="claude", + provider_status_code=529, + ) + recording_hook: Final = self._RecordingFailureHook() + + await self._invoke(exc, route_type, monkeypatch, recording_hook) + + original_exception: Final = recording_hook.original_exception + assert original_exception is not None + assert original_exception.status_code == expected_status + + class TestHandleLLMApiExceptionFramingHeaders: """HTTP-framing headers on the provider exception must be stripped before the proxy builds its own response, or they conflict with the framing the proxy @@ -7671,7 +7888,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit: "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"}, "model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}, }, - {"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}}, + { + "model_name": "chat-model", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}, + }, ], fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}], ) @@ -9745,10 +9965,13 @@ def _hook_data(**kwargs: object) -> object: @pytest.mark.asyncio -@pytest.mark.parametrize("prompt, proxy_field, client_field", [ - ({"input": "hi"}, "litellm_metadata", "metadata"), - ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), -]) +@pytest.mark.parametrize( + "prompt, proxy_field, client_field", + [ + ({"input": "hi"}, "litellm_metadata", "metadata"), + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), + ], +) async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks( prompt: dict[str, str | list[dict[str, str]]], proxy_field: str, @@ -9758,7 +9981,9 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape logging: Final = MagicMock(spec=ProxyLogging) logging.pre_call_hook = AsyncMock(side_effect=_hook_data) - processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}) + processor: Final = ProxyBaseLLMRequestProcessing( + data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt} + ) data, _ = await processor.common_processing_pre_call_logic( request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}), general_settings={}, @@ -9774,6 +9999,7 @@ async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape assert "queue_time_seconds" not in data.get(client_field, {}) assert data["metadata"]["client"] == "cursor" + class TestPerRequestModelGroupAlias: """``router_settings.model_group_alias`` on a key or team has to be resolved by the proxy: the Router resolves aliases from its own shared instance @@ -12609,7 +12835,9 @@ class TestResponseCostFromLoggingObjForCacheHits: proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), ) - cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")} + cost_headers: Final = { + name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost") + } assert cost_headers == {"x-litellm-response-cost-output": "0.0012"} def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self): diff --git a/tests/unit/proxy/test_db_schema_changes.py b/tests/unit/proxy/test_db_schema_changes.py index 3a5d0fb1a0a..f43e2d2a8ce 100644 --- a/tests/unit/proxy/test_db_schema_changes.py +++ b/tests/unit/proxy/test_db_schema_changes.py @@ -12,6 +12,12 @@ def test_aaaaaschema_compatibility() -> None: assert check_breaking_changes(parse_model_fields(ORIGINAL), parse_model_fields(added)) == () +def test_changing_only_a_field_attribute_is_not_breaking() -> None: + previous: Final = parse_model_fields('model Item {\n id String @id @default(dbgenerated("a()::text"))\n}\n') + current: Final = parse_model_fields('model Item {\n id String @id @default(dbgenerated("(a())::text"))\n}\n') + assert check_breaking_changes(previous, current) == () + + def test_parse_model_fields_retains_types_and_attributes() -> None: assert parse_model_fields(ORIGINAL) == {"Item": {"id": "String @id", "label": "String?"}} @@ -40,3 +46,17 @@ def test_schema_compatibility_rejects_removed_models_fields_and_narrowed_types( schema: str, expected: tuple[str, ...] ) -> None: assert check_breaking_changes(parse_model_fields(ORIGINAL), parse_model_fields(schema)) == expected + + +def test_only_the_dropped_accounting_models_may_be_removed() -> None: + accounting: Final = ( + "model LiteLLM_AccountingEvent {\n event_id String @id\n}\n" + "model LiteLLM_AccountingDelivery {\n event_id String\n note String?\n}\n" + ) + previous: Final = parse_model_fields(ORIGINAL + accounting) + + assert check_breaking_changes(previous, parse_model_fields(ORIGINAL)) == () + assert check_breaking_changes(previous, parse_model_fields("")) == ("Breaking: Model Item was removed",) + assert check_breaking_changes( + previous, parse_model_fields(ORIGINAL + "model LiteLLM_AccountingDelivery {\n event_id String\n}\n") + ) == ("Breaking: Field LiteLLM_AccountingDelivery.note was removed",) diff --git a/tests/unit/proxy/test_model_deprecations_endpoint.py b/tests/unit/proxy/test_model_deprecations_endpoint.py index 62d6870e1fe..705f5ec7b39 100644 --- a/tests/unit/proxy/test_model_deprecations_endpoint.py +++ b/tests/unit/proxy/test_model_deprecations_endpoint.py @@ -1,10 +1,12 @@ +import logging +from typing import Final from unittest.mock import MagicMock import pytest from fastapi.testclient import TestClient - import litellm +from litellm._logging import verbose_logger from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -64,12 +66,153 @@ def test_should_bucket_configured_models_by_urgency(authenticated_client): def test_should_rebucket_with_warn_within_days_override(authenticated_client): - response = authenticated_client.get( - "/v1/model/deprecations", params={"warn_within_days": 40000} - ) + response = authenticated_client.get("/v1/model/deprecations", params={"warn_within_days": 40000}) assert response.status_code == 200 payload = response.json() assert [m["model_name"] for m in payload["imminent"]] == ["future-alias"] assert payload["upcoming"] == [] assert payload["warn_within_days"] == 40000 + + +@pytest.fixture +def client_with_cost_map_successors(monkeypatch): + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-9876" + ) + monkeypatch.setattr( + litellm, + "model_cost", + { + "retiring-model": { + "deprecation_date": "2020-01-01", + "litellm_provider": "openai", + "successor_model": "fresh-model", + }, + "orphan-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}, + }, + ) + router = route_entry_points_to_stubs(MagicMock()) + router.get_model_list.return_value = [ + { + "model_name": "retiring-alias", + "litellm_params": {"model": "retiring-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "retiring-overridden", + "litellm_params": {"model": "retiring-model"}, + "model_info": {"id": "2", "successor_model": "my-own-pick"}, + }, + { + "model_name": "shared-name", + "litellm_params": {"model": "retiring-model", "api_key": "fake"}, + "model_info": {"id": "4"}, + }, + { + "model_name": "shared-name", + "litellm_params": {"model": "retiring-model", "api_key": "fake"}, + "model_info": {"id": "5", "successor_model": "my-own-pick"}, + }, + { + "model_name": "dated-by-hand", + "litellm_params": {"model": "retiring-model", "api_key": "fake"}, + "model_info": {"id": "6", "deprecation_date": "2020-06-01"}, + }, + { + "model_name": "orphan-alias", + "litellm_params": {"model": "orphan-model"}, + "model_info": {"id": "3"}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + yield client + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_cost_map_successor_is_reported_unless_the_deployment_names_its_own(client_with_cost_map_successors): + response = client_with_cost_map_successors.get("/model/deprecations") + + assert response.status_code == 200, response.text + successors = {m["model_name"]: m["successor_model"] for m in response.json()["deprecated"]} + assert successors == { + "retiring-alias": "fresh-model", + "retiring-overridden": "my-own-pick", + "shared-name": "my-own-pick", + "dated-by-hand": "fresh-model", + "orphan-alias": None, + } + + +def test_unmatched_successor_logs_below_warning_through_endpoint(client_with_cost_map_successors, caplog, monkeypatch): + monkeypatch.setattr( + proxy_server.llm_router.get_model_list, + "side_effect", + lambda model_name=None, **kwargs: ( + proxy_server.llm_router.get_model_list.return_value if model_name is None else [] + ), + ) + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + response = client_with_cost_map_successors.get("/model/deprecations") + + assert response.status_code == 200, response.text + successor_records = tuple( + record + for record in caplog.records + if record.name == verbose_logger.name and "successor" in record.getMessage() + ) + assert successor_records + assert all(record.levelno < logging.WARNING for record in successor_records) + + +def test_mixed_successor_group_logs_below_warning_through_endpoint( + client_with_cost_map_successors: TestClient, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + router: Final = proxy_server.llm_router + models: Final[tuple[dict[str, object], ...]] = ( + { + "model_name": "mixed-successor-group", + "litellm_params": {"model": "retiring-model"}, + "model_info": {"id": "10", "successor_model": "successor-a"}, + }, + { + "model_name": "mixed-successor-group", + "litellm_params": {"model": "retiring-model"}, + "model_info": {"id": "11", "successor_model": "successor-b"}, + }, + { + "model_name": "successor-a", + "litellm_params": {"model": "fresh-model"}, + "model_info": {"id": "12"}, + }, + { + "model_name": "successor-b", + "litellm_params": {"model": "fresh-model"}, + "model_info": {"id": "13"}, + }, + ) + + def filtered_model_list( + model_name: str | None = None, **_kwargs: object + ) -> tuple[dict[str, object], ...]: + return models if model_name is None else tuple(model for model in models if model["model_name"] == model_name) + + monkeypatch.setattr(router.get_model_list, "side_effect", filtered_model_list) + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + response = client_with_cost_map_successors.get("/model/deprecations") + + assert response.status_code == 200, response.text + successor_records: Final = tuple( + record + for record in caplog.records + if record.name == verbose_logger.name and "successor" in record.getMessage() + ) + assert successor_records + assert all( + "names several successors" in record.getMessage() and record.levelno < logging.WARNING + for record in successor_records + ) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 643c01e3e7f..6347c165865 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1,3 +1,4 @@ +import asyncio import gc import importlib import inspect @@ -30,7 +31,7 @@ from uvicorn.importer import import_from_string import litellm from litellm._logging import turn_on_json -from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT @@ -39,12 +40,114 @@ from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SH @pytest.fixture(autouse=True) def fork_reservation(): """Reserving is irreversible: it would forbid native routes in this pytest worker for good""" - with patch( # test-quality-ok: process-global native state, a real reservation would poison every later test in the worker - "litellm.rust_bridge.fork_guard.reserve_process_for_forking" - ) as reserve: + with ( + patch( # test-quality-ok: process-global native state, a real reservation would poison every later test in the worker + "litellm.rust_bridge.fork_guard.reserve_process_for_forking" + ) as reserve + ): yield reserve +@pytest.mark.parametrize( + ("cli_value", "env_value", "expected"), + [(None, None, None), (None, "131072", 131072), ("65536", "131072", 65536)], +) +def test_max_request_header_size_cli_precedence( + monkeypatch: pytest.MonkeyPatch, cli_value: str | None, env_value: str | None, expected: int | None +) -> None: + monkeypatch.delenv("MAX_REQUEST_HEADER_SIZE", raising=False) + if env_value is not None: + monkeypatch.setenv("MAX_REQUEST_HEADER_SIZE", env_value) + args: Final = [] if cli_value is None else ["--max_request_header_size", cli_value] + with run_server.make_context("litellm", args) as context: + assert context.params["max_request_header_size"] == expected + + +@pytest.mark.parametrize("value", ["0", "-1", "invalid"]) +@pytest.mark.parametrize("source", ["cli", "env"]) +def test_max_request_header_size_rejects_invalid_values( + monkeypatch: pytest.MonkeyPatch, value: str, source: str +) -> None: + monkeypatch.delenv("MAX_REQUEST_HEADER_SIZE", raising=False) + if source == "env": + monkeypatch.setenv("MAX_REQUEST_HEADER_SIZE", value) + args: Final = ["--max_request_header_size", value] if source == "cli" else [] + with pytest.raises(click.BadParameter): + run_server.make_context("litellm", args) + + +def _request_header_test_config(server: str, limit: int | None) -> uvicorn.Config: + from starlette.responses import PlainTextResponse + + app: Final = PlainTextResponse("ok") + if server == "uvicorn": + args: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 4000, max_request_header_size=limit + ) + return uvicorn.Config(**{**args, "app": app, "log_config": None}) + + import gunicorn.config + import gunicorn.glogging + + from litellm.proxy.shutdown.graceful_uvicorn_worker import build_graceful_shutdown_uvicorn_worker + + gunicorn_config: Final = gunicorn.config.Config() + worker: Final = build_graceful_shutdown_uvicorn_worker(7.5, limit)( + 0, os.getpid(), [], MagicMock(), 30, gunicorn_config, gunicorn.glogging.Logger(gunicorn_config) + ) + worker.config.app = app + return worker.config + + +@pytest.mark.parametrize("server", ["uvicorn", "gunicorn"]) +@pytest.mark.parametrize("limit", [None, DEFAULT_MAX_REQUEST_HEADER_SIZE, 128 * 1024]) +def test_request_header_size_preserves_parser_selection(server: str, limit: int | None) -> None: + config: Final = _request_header_test_config(server, limit) + assert config.http == "auto" + assert config.h11_max_incomplete_event_size == (limit if limit is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("server", ["uvicorn", "gunicorn"]) +@pytest.mark.parametrize("parser", ["h11", "httptools"]) +@pytest.mark.parametrize( + ("limit", "header_bytes", "expected_status"), + [(16384, 16384, 200), (16384, 16896, 400), (16384, 39936, 400), (131072, 39936, 200), (131072, 131073, 400)], +) +async def test_segmented_request_headers( + monkeypatch: pytest.MonkeyPatch, server: str, parser: str, limit: int, header_bytes: int, expected_status: int +) -> None: + from uvicorn.server import ServerState + + from uvicorn.protocols.http.h11_impl import H11Protocol + from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol + + selected_protocol: Final = H11Protocol if parser == "h11" else HttpToolsProtocol + monkeypatch.setattr("uvicorn.protocols.http.auto.AutoHTTPProtocol", selected_protocol) + config: Final = _request_header_test_config(server, limit) + config.load() + assert config.http_protocol_class is selected_protocol + protocol: Final = config.http_protocol_class(config, ServerState(), {}, _loop=asyncio.get_running_loop()) + transport: Final = MagicMock(spec=asyncio.Transport) + transport.get_extra_info.return_value = None + transport.is_closing.return_value = False + protocol.connection_made(transport) + prefix: Final = b"GET /health/liveliness HTTP/1.1\r\nHost: localhost\r\nX-Large: " + pending_headers: Final = prefix + b"x" * (header_bytes - len(prefix)) + try: + for offset in range(0, len(pending_headers), 4096): + protocol.data_received(pending_headers[offset : offset + 4096]) + if transport.close.called: + break + if not transport.close.called: + protocol.data_received(b"\r\n\r\n") + await asyncio.gather(*protocol.tasks) + response: Final = b"".join(call.args[0] for call in transport.write.call_args_list) + assert response.startswith(f"HTTP/1.1 {expected_status if parser == 'h11' else 200} ".encode()) + finally: + protocol.connection_lost(None) + + @pytest.mark.xdist_group("proxy_cli") class TestProxyInitializationHelpers: @patch("importlib.metadata.version") @@ -1550,6 +1653,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=30, timeout_worker_healthcheck=None, + max_request_header_size=None, ) mock_uvicorn_run.assert_called_once() @@ -1626,6 +1730,7 @@ class TestProxyInitializationHelpers: log_config=None, keepalive_timeout=None, timeout_worker_healthcheck=15, + max_request_header_size=None, ) @patch("uvicorn.run") @@ -2092,7 +2197,10 @@ class TestProxyInitializationHelpers: assert captured["options"]["max_requests_jitter"] == 50 @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") - def test_gunicorn_workers_bound_graceful_shutdown_by_graceful_shutdown_timeout(self, monkeypatch): + @pytest.mark.parametrize("max_request_header_size", [DEFAULT_MAX_REQUEST_HEADER_SIZE, 128 * 1024]) + def test_gunicorn_workers_bound_graceful_shutdown_by_graceful_shutdown_timeout( + self, monkeypatch, max_request_header_size: int + ): pytest.importorskip("gunicorn") import gunicorn.config import gunicorn.glogging @@ -2117,6 +2225,7 @@ class TestProxyInitializationHelpers: num_workers=2, ssl_certfile_path=None, ssl_keyfile_path=None, + max_request_header_size=max_request_header_size, ) for name in ("uvicorn.error", "uvicorn.access"): @@ -2132,6 +2241,7 @@ class TestProxyInitializationHelpers: assert worker.config.timeout_graceful_shutdown == 7.5 assert worker.config.loop == UvicornWorker.CONFIG_KWARGS["loop"] assert worker.config.http == UvicornWorker.CONFIG_KWARGS["http"] + assert worker.config.h11_max_incomplete_event_size == max_request_header_size assert captured["options"]["graceful_timeout"] == 8 + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index e4af36d99f2..dae9ffcf7fd 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -42,6 +42,7 @@ from fastapi import FastAPI, HTTPException, WebSocket # test /chat/completion request to the proxy from fastapi.testclient import TestClient +from starlette.testclient import WebSocketTestSession from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -267,76 +268,6 @@ async def test_shutdown_commits_spend_a_logging_task_queued_while_the_first_comm token = MASTER_KEY -@pytest.mark.asyncio -async def test_shutdown_persists_accounting_from_worker_callbacks_before_returning( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from datetime import datetime - - from litellm.litellm_core_utils.logging_worker import LoggingWorker - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - from litellm.responses.streaming_iterator import spawn_logging_task - - worker: Final = LoggingWorker() - started: Final = asyncio.Event() - release: Final = asyncio.Event() - first_flush: Final = asyncio.Event() - pending: Final[asyncio.Queue[str]] = asyncio.Queue() - persisted: Final[asyncio.Queue[str]] = asyncio.Queue() - - captured: Final[asyncio.Queue[str]] = asyncio.Queue() - - async def persist_usage(writes: tuple[UsageWrite, ...]) -> None: - for write in writes: - captured.put_nowait(write.event.event_id) - - capture: Final = UsageCaptureRuntime() - capture.start(persist_usage, UsageCaptureSettings(flush_interval_seconds=10)) - monkeypatch.setattr(litellm.proxy.proxy_server, "usage_capture", capture) - - async def enqueue_spend() -> None: - assert capture.meter is not None - capture_callback( - capture.meter, - {"call_type": "acompletion", "litellm_call_id": "shutdown-usage", "litellm_params": {}}, - {"id": "shutdown-attempt", "usage": {"total_tokens": 3}}, - datetime(2026, 1, 1), - datetime(2026, 1, 1), - ) - pending.put_nowait("accounted request") - - async def callback() -> None: - started.set() - await release.wait() - spawn_logging_task(asyncio.get_running_loop(), enqueue_spend(), task_name="accounting") - - async def flush_spend() -> None: - while not pending.empty(): - persisted.put_nowait(pending.get_nowait()) - first_flush.set() - - worker.ensure_initialized_and_enqueue(callback()) - await started.wait() - shutdown: Final = asyncio.create_task( - litellm.proxy.proxy_server._flush_spend_around_logging_drain_on_shutdown(flush_spend, worker=worker) - ) - try: - await first_flush.wait() - waited_for_callback: Final = not shutdown.done() - release.set() - await shutdown - assert waited_for_callback, "Shutdown returned while accounting callbacks were still running" - assert persisted.get_nowait() == "accounted request" - assert pending.empty() - assert (captured.get_nowait(), captured.get_nowait()) == ("shutdown-usage:inbound", "shutdown-attempt") - assert capture.meter is None - finally: - release.set() - await worker.flush() - await worker.stop() - await capture.stop(0) @pytest.mark.asyncio @@ -3661,6 +3592,91 @@ def test_live_session_pre_call_abort_releases_reservations( route_call.assert_not_awaited() +def _receive_until_closed(websocket: WebSocketTestSession) -> None: + while True: + websocket.receive_text() + + +@pytest.mark.parametrize( + ("litellm_params", "expected_upstream_url"), + ( + pytest.param( + {"model": "openai/gateway-transcribe", "api_base": "https://gateway.example/v1"}, + "wss://gateway.example/v1/realtime?model=gateway-transcribe&intent=transcription", + id="gateway", + ), + pytest.param( + {"model": "xai/grok-voice", "api_base": "https://api.x.ai/v1"}, + "wss://api.x.ai/v1/realtime?model=grok-voice&intent=transcription", + id="xai", + ), + pytest.param( + {"model": "openai/gpt-live-transcribe", "api_base": "https://api.openai.com/v1"}, + "wss://api.openai.com/v1/realtime?intent=transcription", + id="openai", + ), + ), +) +def test_realtime_transcription_websocket_keeps_the_model_upstream_except_on_openai_hosts( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, str], + expected_upstream_url: str, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "team-transcriber", "litellm_params": {**litellm_params, "api_key": "fake"}}] + ), + ) + connect: Final = MagicMock(side_effect=RuntimeError("upstream url captured")) + monkeypatch.setattr("websockets.connect", connect) + + with ( + client_no_auth.websocket_connect( + "/v1/realtime?model=team-transcriber&intent=transcription", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket, + pytest.raises(WebSocketDisconnect), + ): + _receive_until_closed(websocket) + + assert connect.call_args.args[0] == expected_upstream_url + + +def test_openai_prefixed_translation_websocket_takes_the_openai_websocket_pass_through( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _OPENAI_WS_DISABLED_REFUSAL + + route_call: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "route_request", route_call) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth_websocket, lambda: UserAPIKeyAuth()) + + with client_no_auth.websocket_connect( + "/openai/v1/realtime/translations?model=gpt-realtime-translate", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket: + refusal: Final = websocket.receive_json() + with pytest.raises(WebSocketDisconnect) as closed: + websocket.receive_json() + + assert refusal["error"]["message"] == _OPENAI_WS_DISABLED_REFUSAL.message + assert closed.value.code == 1008 + route_call.assert_not_awaited() + + @pytest.mark.parametrize("path", ["/openai/v1/live/sessions", "/v1/live/sessions", "/live/sessions"]) def test_live_session_invalid_start_is_rejected_without_routing( path: str, @@ -4291,276 +4307,11 @@ async def test_shutdown_waits_for_enqueue_producers_before_stopping_worker() -> await worker.stop(discard_pending=True) -@pytest.mark.asyncio -@pytest.mark.parametrize("unsupported", ["database", "collector", "remote", "free_tier"]) -async def test_capture_enablement_rejects_unrecoverable_writer_modes( - unsupported: str, monkeypatch: pytest.MonkeyPatch -) -> None: - import litellm - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - - monkeypatch.setattr(proxy_server, "usage_capture", UsageCaptureRuntime()) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {}}) - monkeypatch.setattr(proxy_server, "prisma_client", None if unsupported == "database" else object()) - monkeypatch.setattr(litellm, "apply_search_free_tier", unsupported == "free_tier") - monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "true" if unsupported == "collector" else "false") - monkeypatch.delenv("SPEND_LOGS_URL", raising=False) - if unsupported == "remote": - monkeypatch.setenv("SPEND_LOGS_URL", "https://writer.invalid") - with pytest.raises(ValueError, match="requires local Prisma"): - proxy_server._configure_usage_capture() - assert proxy_server.usage_capture.meter is None -@pytest.mark.asyncio -async def test_capture_configuration_requires_restart_without_replacing_the_active_writer( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from litellm.proxy import proxy_server - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.utils import PrismaClient - - runtime: Final = UsageCaptureRuntime() - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) - with pytest.raises(ValueError, match="requires accounting_capture"): - proxy_server._configure_usage_capture() - client: Final = PrismaClient("postgresql://localhost/unused", ProxyLogging(UserApiKeyCache())) - monkeypatch.setattr(proxy_server, "prisma_client", client) - monkeypatch.setattr(litellm, "apply_search_free_tier", False) - monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "false") - monkeypatch.delenv("SPEND_LOGS_URL", raising=False) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {"batch_size": 4}}) - try: - proxy_server._configure_usage_capture() - active: Final = runtime.meter - assert active is not None - assert active.settings.batch_size == 4 - proxy_server._configure_usage_capture() - assert runtime.meter is active - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {"batch_size": 6}}) - with pytest.raises(ValueError, match="Changing accounting_capture requires a worker restart"): - proxy_server._configure_usage_capture() - monkeypatch.setattr(proxy_server, "general_settings", {}) - with pytest.raises(ValueError, match="Disabling accounting_capture requires a worker restart"): - proxy_server._configure_usage_capture() - assert runtime.meter is active - assert active.settings.batch_size == 4 - finally: - await runtime.stop(1) - assert runtime.meter is None -@pytest.mark.asyncio -@pytest.mark.parametrize("sync_ack", [False, True]) -@pytest.mark.parametrize( - ("path", "payload"), - [ - ( - "/v1/chat/completions", - {"model": "openai/gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True}, - ), - ("/v1/responses", {"model": "openai/gpt-5-mini", "input": "hi", "background": True}), - ( - "/v1/messages", - {"model": "openai/gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 1}, - ), - ], -) -async def test_sync_ack_preserves_async_request_support( - sync_ack: bool, - path: str, - payload: dict[str, object], - monkeypatch: pytest.MonkeyPatch, - respx_mock: respx.MockRouter, -) -> None: - from fastapi import Request - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - - persisted: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - for write in writes: - persisted.put_nowait(write) - - async def authorize(request: Request) -> UserAPIKeyAuth: - return UserAPIKeyAuth(api_key="test-key", request_route=request.url.path) - - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setenv("OPENAI_API_KEY", "test-provider-key") - upstream: Final = { - "id": "provider-result", - "object": "chat.completion", - "created": 1, - "model": "gpt-5-mini", - "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, - } - if payload.get("stream"): - chunk: Final = { - **upstream, - "object": "chat.completion.chunk", - "choices": [{"index": 0, "finish_reason": "stop", "delta": {"role": "assistant", "content": "hi"}}], - } - respx_mock.post().respond( - 200, text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n", headers={"content-type": "text/event-stream"} - ) - elif path == "/v1/responses": - respx_mock.post().respond( - 200, - json={ - "id": "resp_background", - "object": "response", - "created_at": 1, - "model": "gpt-5-mini", - "status": "in_progress", - "background": True, - "output": [], - "usage": None, - }, - ) - else: - respx_mock.post(url__regex=r".*/responses$").respond( - 200, - json={ - "id": "resp_message", - "object": "response", - "created_at": 1, - "model": "gpt-5-mini", - "status": "completed", - "output": [ - { - "id": "msg_reply", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "hi", "annotations": []}], - } - ], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - }, - ) - respx_mock.post(url__regex=r".*/chat/completions$").respond(200, json=upstream) - runtime: Final = UsageCaptureRuntime() - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr( - proxy_server, "general_settings", {"accounting_sync_ack": sync_ack, "accounting_sync_ack_timeout_seconds": 0} - ) - monkeypatch.setattr(proxy_server, "prisma_client", None) - monkeypatch.setattr( - proxy_server, - "llm_router", - litellm.Router( - model_list=[ - { - "model_name": payload["model"], - "litellm_params": {"model": payload["model"], "api_key": "test-provider-key"}, - } - ], - num_retries=0, - ), - ) - monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(UserApiKeyCache())) - monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger()]) - monkeypatch.setitem(proxy_server.app.dependency_overrides, proxy_server.user_api_key_auth, authorize) - runtime.start(persist, UsageCaptureSettings()) - try: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test" - ) as client: - response: Final = await client.post(path, json=payload) - finally: - await runtime.stop(1) - assert response.status_code == 200, response.text - assert len(respx_mock.calls) == 1 - if payload.get("background"): - assert response.json()["status"] == "in_progress" - else: - assert "hi" in response.text - assert not persisted.empty(), "Terminal usage must still be captured" - -@pytest.mark.asyncio -@pytest.mark.parametrize("sync_ack", [False, True]) -async def test_text_completion_provider_rejection_captures_unknown_usage( - sync_ack: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: - from fastapi import Request - - from litellm.accounting import Outcome - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - - persisted: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - for write in writes: - persisted.put_nowait(write) - - async def authorize(request: Request) -> UserAPIKeyAuth: - return UserAPIKeyAuth(api_key="test-key", request_route=request.url.path) - - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setenv("OPENAI_API_KEY", "test-provider-key") - respx_mock.post().respond( - 404, - json={ - "error": {"message": "The model is unavailable", "type": "invalid_request_error", "code": "model_not_found"} - }, - ) - runtime: Final = UsageCaptureRuntime() - monkeypatch.setattr(proxy_server, "usage_capture", runtime) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": sync_ack}) - monkeypatch.setattr(proxy_server, "prisma_client", None) - monkeypatch.setattr( - proxy_server, - "llm_router", - litellm.Router( - model_list=[ - { - "model_name": "instruct", - "litellm_params": {"model": "openai/gpt-3.5-turbo-instruct", "api_key": "test-provider-key"}, - } - ], - num_retries=0, - ), - ) - monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(UserApiKeyCache())) - monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger()]) - monkeypatch.setitem(proxy_server.app.dependency_overrides, proxy_server.user_api_key_auth, authorize) - runtime.start(persist, UsageCaptureSettings()) - try: - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test" - ) as client: - response: Final = await client.post( - "/v1/completions", json={"model": "instruct", "prompt": "hi", "max_tokens": 1} - ) - finally: - await runtime.stop(1) - assert response.status_code == 404, response.text - assert len(respx_mock.calls) == 1 - assert persisted.qsize() == 2 - inbound: Final = persisted.get_nowait() - outbound: Final = persisted.get_nowait() - assert outbound.event.outcome is Outcome.FAILED - assert outbound.event.call_type.value == "atext_completion" - assert outbound.event.metrics is None - assert inbound.event.request_id == outbound.event.request_id - envelope: Final = json.loads(outbound.envelope) - assert envelope["usage"] is None - assert envelope["usage_source"] == "unknown" - assert envelope["pricing"]["reported_cost"] is None @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 487638d6919..62d8db6f810 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -30,6 +30,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient +from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -1793,6 +1794,38 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_reads_fips_mode_from_the_config_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": True}) + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode is on but OpenSSL does not enforce FIPS"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_a_fips_mode_reference_to_an_unset_variable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings( + monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": "os.environ/FIPS_MODE_NOT_SET"} + ) + monkeypatch.delenv("FIPS_MODE_NOT_SET", raising=False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode has no value"): + async with proxy_startup_event(FastAPI()): + pass + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext @@ -4116,7 +4149,11 @@ async def test_load_environment_variables_litellm_license_and_edge_cases(): mock_license_check = MagicMock() mock_license_check.is_premium.return_value = True - with patch("litellm.proxy.proxy_server._license_check", mock_license_check): + with ( + patch("litellm.proxy.proxy_server._license_check", mock_license_check), + patch("litellm.proxy.proxy_server.premium_user", False), + patch("litellm.proxy.proxy_server.premium_user_data", None), + ): with patch.dict(os.environ, {}, clear=False): # Call the method under test proxy_config._load_environment_variables(test_config_with_license) @@ -11117,7 +11154,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via three path aliases stacked on + """Realtime sessions reach the proxy via five path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -11133,7 +11170,13 @@ def test_realtime_websocket_route_aliases_registered(): websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)} openai_routes = LiteLLMRoutes.openai_routes.value - for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + for expected in ( + "/openai/v1/realtime", + "/v1/realtime", + "/realtime", + "/v1/realtime/translations", + "/realtime/translations", + ): assert expected in websocket_paths, ( f"{expected!r} missing from registered WebSocket routes; the " f"realtime endpoint will 405 for clients hitting this path." @@ -11152,7 +11195,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock: ws = MagicMock() ws.headers = {} ws.scope = {"headers": [], "type": "websocket"} - ws.url = "ws://testserver/v1/realtime" + ws.url = URL("ws://testserver/v1/realtime") ws.accept = AsyncMock() ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -16546,3 +16589,63 @@ async def test_initialize_prints_the_banner_as_one_json_record_in_json_mode( banner_records: Final = [record for record in records if record["message"] == LITELLM_BANNER] assert len(banner_records) == 1, records assert banner_records[0]["level"] == "INFO" + + +@contextlib.contextmanager +def _accounting_window_env(cached: Mapping[str, object], client: MagicMock) -> Iterator[None]: + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + cache: Final = UserApiKeyCache() + for key, value in cached.items(): + cache.set_cache(key=key, value=value) + originals: Final = (proxy_server_module.user_api_key_cache, proxy_server_module.prisma_client) + proxy_server_module.user_api_key_cache = cache + proxy_server_module.prisma_client = client + try: + yield + finally: + proxy_server_module.user_api_key_cache, proxy_server_module.prisma_client = originals + + +@pytest.mark.asyncio +async def test_accounting_window_keys_cover_the_stored_key_and_the_cached_team(): + key_reset: Final = datetime(2026, 11, 13, tzinfo=timezone.utc) + team_reset: Final = datetime(2026, 11, 6, tzinfo=timezone.utc) + key_row: Final = _StoredRow( + budget_limits=[{"budget_duration": "30d", "max_budget": 100.0, "reset_at": key_reset.isoformat()}] + ) + team_obj: Final = MagicMock() + team_obj.budget_limits = [{"budget_duration": "7d", "max_budget": 50.0, "reset_at": team_reset.isoformat()}] + + with _accounting_window_env({"team_id:team-1": team_obj}, _prisma_with_rows(keys={"hashed-key": key_row})): + windows = await proxy_server_module.accounting_window_keys("hashed-key", "team-1") + + assert windows == ( + { + "entity_type": "key", + "entity_id": "hashed-key", + "window_duration": "30d", + "window_start": (key_reset - timedelta(days=30)).isoformat(), + }, + { + "entity_type": "team", + "entity_id": "team-1", + "window_duration": "7d", + "window_start": (team_reset - timedelta(days=7)).isoformat(), + }, + ) + + +@pytest.mark.asyncio +async def test_accounting_window_keys_remember_a_key_without_windows_but_retry_a_failed_read(): + client: Final = _prisma_with_rows(keys={"no-windows": _StoredRow(budget_limits=[])}) + + with _accounting_window_env({}, client): + assert await proxy_server_module.accounting_window_keys("no-windows", None) == () + assert await proxy_server_module.accounting_window_keys("no-windows", None) == () + assert client.db.litellm_verificationtoken.find_unique.await_count == 1 + + client.db.litellm_verificationtoken.find_unique.side_effect = RuntimeError("database unavailable") + assert await proxy_server_module.accounting_window_keys("unreadable", None) == () + assert await proxy_server_module.accounting_window_keys("unreadable", None) == () + assert client.db.litellm_verificationtoken.find_unique.await_count == 3 diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index 1b9cdd003ab..698f4787a42 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -3716,3 +3716,55 @@ async def test_evict_config_param_clears_the_local_layer_and_survives_a_redis_ou await evict_config_param("router_settings", cache=cache) assert await cache.in_memory_cache.async_get_cache(_config_cache_key("router_settings")) is None + + +@pytest.mark.asyncio +async def test_query_raw_within_statement_timeout_has_postgres_cancel_the_read_at_30s() -> None: + from datetime import timedelta + + from litellm.proxy.utils import query_raw_within_statement_timeout + from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_reads + + prisma: Final = MagicMock() + transaction: Final = statement_timeout_reads(prisma, AsyncMock(return_value=[{"spend": 1.5}])) + + rows: Final = await query_raw_within_statement_timeout( + prisma, 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE "user" = $1', "u1" + ) + + assert rows == [{"spend": 1.5}] + assert transaction.statements == ("SET LOCAL statement_timeout = 30000",) + transaction.query_raw.assert_awaited_once_with('SELECT spend FROM "LiteLLM_SpendLogs" WHERE "user" = $1', "u1") + assert prisma.read_db.tx.call_args.kwargs["timeout"] > timedelta(seconds=30), "Prisma must outlive the Postgres cancel" + + +@pytest.mark.asyncio +async def test_query_raw_within_statement_timeout_reports_a_cancelled_read_as_503() -> None: + from litellm.proxy.utils import query_raw_within_statement_timeout + from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_error, statement_timeout_reads + + prisma: Final = MagicMock() + statement_timeout_reads(prisma, AsyncMock(side_effect=statement_timeout_error())) + + with pytest.raises(HTTPException) as exc_info: + await query_raw_within_statement_timeout(prisma, "SELECT 1", what="Spend per user") + + assert exc_info.value.status_code == 503 + assert exc_info.value.detail == {"error": "Spend per user took longer than 30s. Pick a shorter date range"} + + +@pytest.mark.asyncio +async def test_query_raw_within_statement_timeout_leaves_other_database_errors_alone() -> None: + from prisma.errors import DataError + + from litellm.proxy.utils import query_raw_within_statement_timeout + from tests.unit.proxy.db.fake_prisma_engine import statement_timeout_reads + + prisma: Final = MagicMock() + failure: Final = DataError(data={"user_facing_error": {"is_panic": False, "message": "relation does not exist"}}) + statement_timeout_reads(prisma, AsyncMock(side_effect=failure)) + + with pytest.raises(DataError) as exc_info: + await query_raw_within_statement_timeout(prisma, "SELECT 1") + + assert exc_info.value is failure diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index db0db97cd88..3b000ba741f 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -3735,42 +3735,6 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest assert sent == earlier_attempt_prompt_tokens([COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]) -@pytest.mark.asyncio -async def test_a_billed_responses_route_failure_captures_usage_from_the_newest_compaction_item() -> None: - from litellm.litellm_core_utils.get_litellm_params import get_litellm_params - from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - from litellm.proxy.utils import _failure_fields_to_lift - - async def persist(_: tuple[UsageWrite, ...]) -> None: - return None - - def captured_prompt_tokens(request_input: list[dict[str, str]]) -> int: - logging_obj: Final = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, get_litellm_params()) - logging_obj.model_call_details["call_type"] = "aresponses" - logging_obj.model_call_details["messages"] = request_input - lifted: Final = _failure_fields_to_lift( - {"litellm_logging_obj": logging_obj}, provider_billed=True, route_call_type="aresponses" - ) - snapshot: Final = lifted["_accounting_usage_snapshot"] - assert isinstance(snapshot, UsageCaptureObservation) and snapshot.usage_json is not None - return json.loads(snapshot.usage_json)["prompt_tokens"] - - runtime: Final = UsageCaptureRuntime() - runtime.start(persist, UsageCaptureSettings()) - try: - sent: Final = captured_prompt_tokens( - [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION] - ) - since_compaction: Final = captured_prompt_tokens( - [COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION] - ) - finally: - await runtime.stop(1) - - assert sent == since_compaction @pytest.mark.parametrize( @@ -3814,6 +3778,56 @@ def test_a_retry_failure_bills_input_only_for_dispatched_attempts( assert lifted["response_cost"] == 0.25 +@pytest.mark.asyncio +async def test_an_agentic_follow_up_failure_row_bills_each_served_first_call_without_an_estimate() -> None: + import litellm + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.proxy.utils import _failure_fields_to_lift + from litellm.types.utils import ModelResponse, Usage + + logging_obj: Final = Logging( + model=FAILURE_USAGE_MODEL, + messages=ONE_USER_MESSAGE, + stream=False, + call_type="acompletion", + start_time=real_datetime.datetime.now(), + litellm_call_id="call-agentic-follow-up", + function_id="fn-agentic-follow-up", + ) + logging_obj.update_environment_variables( + model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"} + ) + + async def failing_follow_up() -> ModelResponse: + raise litellm.RateLimitError(message="rate limited", llm_provider="openai", model=FAILURE_USAGE_MODEL) + + async def serve_first_call_then_fail_its_follow_up(usage: Usage) -> float: + logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test") + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse(model=FAILURE_USAGE_MODEL, usage=usage), + stream=False, + ) + return logging_obj.model_call_details["response_cost"] + + primary_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ) + logging_obj.carry_failed_attempt_usage() + backup_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=31, completion_tokens=5, total_tokens=36) + ) + + lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}) + + usage: Final = lifted["combined_usage_object"] + assert (usage.prompt_tokens, usage.completion_tokens) == (559 + 31, 113 + 5) + assert primary_cost > 0 and backup_cost > 0 + assert lifted["response_cost"] == pytest.approx(primary_cost + backup_cost) + + @pytest.mark.parametrize( "model_call_details, dispatched", [ @@ -4355,57 +4369,3 @@ def test_is_client_error_exception_existing_behaviour_unchanged() -> None: ProxyException(message="x", type="x", param=None, code="429") ) is True assert _is_client_error_exception(ValueError("x")) is False - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokens", [None, 0, 3]) -async def test_failure_capture_preserves_prepared_usage_observation(tokens: int | None) -> None: - from litellm.proxy.db.accounting_ledger import UsageWrite - from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback - from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings - - timestamp: Final = real_datetime.datetime(2026, 1, 1) - committed: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue() - - async def persist(writes: tuple[UsageWrite, ...]) -> None: - for write in writes: - committed.put_nowait(write) - - logger: Final = Logging( - model=FAILURE_USAGE_MODEL, - messages=[], - stream=False, - call_type="acompletion", - start_time=timestamp, - litellm_call_id="post-response-failure", - function_id="post-response-failure", - ) - logger.update_environment_variables( - model=FAILURE_USAGE_MODEL, custom_llm_provider="openai", litellm_params={}, optional_params={} - ) - logger.model_call_details["response_cost"] = 0.2 - runtime: Final = UsageCaptureRuntime() - runtime.start(persist, UsageCaptureSettings()) - try: - prepared: Final = logger.prepare_response_cost( - {"id": "provider", "usage": None if tokens is None else {"prompt_tokens": tokens, "total_tokens": tokens}}, - timestamp, - ) - assert prepared is not None - lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logger}, provider_billed=True) - assert lifted["combined_usage_object"] is logger.model_call_details["combined_usage_object"] - assert lifted["response_cost"] == 0.2 - assert runtime.meter is not None - capture_callback( - runtime.meter, - {**logger.model_call_details, **lifted, "log_event_type": "failed_api_call"}, - None, - timestamp, - timestamp, - ) - finally: - await runtime.stop(1) - committed.get_nowait() - envelope: Final = json.loads(committed.get_nowait().envelope) - assert (None if envelope["usage"] is None else envelope["usage"]["total_tokens"]) == tokens - assert envelope["usage_source"] == ("unknown" if tokens is None else "measured") diff --git a/tests/unit/proxy/test_update_spend.py b/tests/unit/proxy/test_update_spend.py index 571341e85e7..e2c027408c1 100644 --- a/tests/unit/proxy/test_update_spend.py +++ b/tests/unit/proxy/test_update_spend.py @@ -351,3 +351,125 @@ async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_re assert sorted(written) == ["good-1", "good-2"] assert prisma_client.spend_log_transactions == [] proxy_logging_obj.failure_handler.assert_not_called() + + +def _raw_query_error(sqlstate: str): + from prisma.errors import RawQueryError + + return RawQueryError( + { + "user_facing_error": { + "error_code": "P2010", + "meta": {"code": sqlstate, "message": f"ERROR: sqlstate {sqlstate}"}, + } + } + ) + + +class _RawInsertRepository: + def __init__(self, sqlstate: str): + self.sqlstate = sqlstate + self.written: list[str] = [] + + async def insert_with_deliveries(self, rows): + if any(row["request_id"] == "rejected" for row in rows): + raise _raw_query_error(self.sqlstate) + self.written.extend(str(row["request_id"]) for row in rows) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["22P02", "23502", "54000"]) +async def test_raw_insert_rejecting_one_row_drops_only_that_row(sqlstate): + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + repo = _RawInsertRepository(sqlstate) + rows = [{"request_id": "good-1"}, {"request_id": "rejected"}, {"request_id": "good-2"}] + + await _create_spend_logs_with_poison_isolation(repo, rows, failure_budget=10) + + assert sorted(repo.written) == ["good-1", "good-2"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["42P01", "40P01"]) +async def test_systemic_raw_insert_failure_is_retried_not_dropped(sqlstate): + from prisma.errors import RawQueryError + + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + repo = _RawInsertRepository(sqlstate) + rows = [{"request_id": "good-1"}, {"request_id": "rejected"}] + + with pytest.raises(RawQueryError): + await _create_spend_logs_with_poison_isolation(repo, rows, failure_budget=10) + + assert repo.written == [] + + +class _RescueFailsRepository: + def __init__(self, rescue_error: Exception): + self.rescue_error = rescue_error + + async def insert_with_deliveries(self, rows): + if any(row.get("accounting_log_disabled") for row in rows): + raise self.rescue_error + if any(row["request_id"] == "rejected" for row in rows): + raise _raw_query_error("22P02") + + +@pytest.mark.asyncio +async def test_a_transient_failure_while_keeping_a_dropped_rows_charge_is_retried_not_lost(): + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + outage = _raw_query_error("40P01") + repo = _RescueFailsRepository(outage) + rows = [{"request_id": "rejected", "accounting_payload": {"spend": 1.0}}] + + with pytest.raises(type(outage)): + await _create_spend_logs_with_poison_isolation(repo, rows, failure_budget=10) + + +@pytest.mark.asyncio +async def test_a_rejected_charge_rescue_is_logged_and_the_flush_continues(): + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + repo = _RescueFailsRepository(_raw_query_error("22P02")) + rows = [{"request_id": "rejected", "accounting_payload": {"spend": 1.0}}] + + assert await _create_spend_logs_with_poison_isolation(repo, rows, failure_budget=10) == 9 + + +class _StatementValidatingRepository: + def __init__(self): + self.written = [] + self.charges_kept = [] + + async def insert_with_deliveries(self, rows): + from litellm.repositories.spend_log_insert import spend_log_insert_statement + + if all(row.get("accounting_log_disabled") for row in rows): + self.charges_kept.extend(row["request_id"] for row in rows) + return len(rows) + spend_log_insert_statement(rows) + self.written.extend(row["request_id"] for row in rows) + return len(rows) + + +@pytest.mark.asyncio +async def test_a_row_the_insert_statement_rejects_is_dropped_alone_and_keeps_its_charge(): + from litellm.proxy.utils import _create_spend_logs_with_poison_isolation + + repo = _StatementValidatingRepository() + rows = [ + {"request_id": "good", "call_type": "acompletion", "messages": "[]"}, + { + "request_id": "unparseable", + "call_type": "acompletion", + "messages": "not json", + "accounting_payload": {"spend": 1.0}, + }, + ] + + assert await _create_spend_logs_with_poison_isolation(repo, rows, failure_budget=10) < 10 + assert repo.written == ["good"] + assert repo.charges_kept == ["unparseable"] diff --git a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py index c2931470482..ce1dfce1646 100644 --- a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py +++ b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py @@ -32,6 +32,29 @@ from litellm.router import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +@pytest.mark.parametrize( + ("model", "expected_free"), + [("parallel_ai/decision", False), ("fireworks_ai/accounts/fireworks/routers/rune", True)], +) +def test_query_price_keeps_budgets_only_where_it_is_billed(model: str, expected_free: bool) -> None: + router = Router( + model_list=[ + { + "model_name": "query-priced", + "litellm_params": { + "model": model, + "api_key": "test-key", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_query": 0.025, + }, + } + ] + ) + + assert is_model_cost_zero("query-priced", router) is expected_free + + @pytest.fixture def mock_router_with_zero_cost_model(): """Create a mock router with a zero-cost model.""" diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index e66050877df..77d0132ec72 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -18,6 +18,27 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( client = TestClient(app) +@pytest.mark.asyncio +async def test_sso_settings_without_prisma_reports_database_not_connected(monkeypatch: pytest.MonkeyPatch) -> None: + import runpy + import sys + + from fastapi import HTTPException + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.ui_crud_endpoints import proxy_setting_endpoints + + monkeypatch.setattr(proxy_server, "prisma_client", None) + with patch.dict(sys.modules, {"prisma": None, "prisma.errors": None}): + endpoints: Final = runpy.run_path(proxy_setting_endpoints.__file__) + with pytest.raises(HTTPException) as error: + await endpoints["update_sso_settings"](SSOConfig(), UserAPIKeyAuth()) + + assert error.value.status_code == 500 + assert error.value.detail == {"error": "Database not connected. Please connect a database."} + + @pytest.mark.asyncio @pytest.mark.parametrize("config_value,stored_value,expected", [ (None, None, False), (None, False, False), (None, True, True), @@ -874,6 +895,25 @@ class TestProxySettingEndpoints: env_row_update = prisma_client.db.litellm_config.update.call_args.kwargs assert json.loads(env_row_update["data"]["param_value"]) == {"LANGFUSE_PUBLIC_KEY": "unrelated"} + def test_update_sso_settings_blank_stored_value_keeps_env_row_secret( + self, mock_proxy_config, mock_auth, monkeypatch + ): + import json + + response, saved = self._patch_sso_settings( + monkeypatch, + {**self._STORED_GENERIC_SSO, "generic_client_secret": ""}, + {"user_email": "new@example.com"}, + environment_variables={"GENERIC_CLIENT_SECRET": "env-row-secret", "LANGFUSE_PUBLIC_KEY": "unrelated"}, + ) + + from litellm.proxy.proxy_server import prisma_client + + assert response.status_code == 200, response.text + assert saved["generic_client_secret"] == "env-row-secret", saved + env_row_update = prisma_client.db.litellm_config.update.call_args.kwargs + assert json.loads(env_row_update["data"]["param_value"]) == {"LANGFUSE_PUBLIC_KEY": "unrelated"} + def test_update_sso_settings_maps_saml_fields_to_env_vars( self, mock_proxy_config, mock_auth, monkeypatch ): diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py index fc7ae531c34..293f2d49f43 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -23,9 +23,9 @@ import pytest from litellm.constants import REDIS_SPEND_LOGS_BUFFER_KEY from litellm.proxy.utils import ( MAX_SPEND_LOG_DRAIN_ITERATIONS, + drain_spend_logs_queue, monitor_spend_logs_queue, raise_failed_update_spend_exception, - drain_spend_logs_queue, recover_parked_spend_logs, update_daily_tag_spend, update_spend, @@ -985,6 +985,85 @@ async def test_update_spend_logs_job_requeues_whole_batch_when_postgres_is_out_o } == {"create_many_calls": 1, "queue_after": ["r1", "r2", "r3"], "backoff_sleeps": []} +def _postgres_restarting() -> Exception: + """prisma's shape for Postgres SQLSTATE 57P03 while the server runs crash recovery after + a restart: a base ``DataError`` carrying only the connector message.""" + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error in connector: Error querying the database: FATAL: the database system is starting up", + "backtrace": None, + } + } + ) + + +def _postgres_unreachable() -> Exception: + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Can't reach database server at `db.internal`:`5432`", + "error_code": "P1001", + } + } + ) + + +def _postgres_terminated_session() -> Exception: + from prisma.errors import DataError + + return DataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error in connector: Error querying the database: FATAL: terminating connection due to " + "administrator command", + "meta": {"code": "57P01", "message": "terminating connection due to administrator command"}, + } + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [_postgres_restarting(), _postgres_terminated_session()], ids=["57P03", "57P01"]) +async def test_update_spend_logs_job_keeps_every_row_while_postgres_restarts( + error: Exception, + mock_prisma_client: MagicMock, + make_spend_log_row: Callable[..., dict[str, object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A restarting server refuses or cuts every statement, so the batch is not bisected + into per-row drops: it is retried whole and then goes back to the queue head.""" + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + proxy_logging: Final = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [ + make_spend_log_row(request_id="r1"), + make_spend_log_row(request_id="r2"), + make_spend_log_row(request_id="r3"), + make_spend_log_row(request_id="r4"), + ] + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=error) + + with suppress(Exception): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert { + "create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count, + "queue_after": [row["request_id"] for row in mock_prisma_client.spend_log_transactions], + } == {"create_many_calls": 4, "queue_after": ["r1", "r2", "r3", "r4"]} + + def _tool_usage_transaction(request_id: str) -> object: from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -1177,6 +1256,53 @@ async def test_model_usage_flush_requeues_and_stops_the_job_when_postgres_is_out } == {"queue_after": [first, second], "autorouter_queue_after": 1} +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [_postgres_restarting(), _postgres_unreachable()], ids=["57P03", "P1001"]) +async def test_rollups_requeue_and_stop_the_job_while_postgres_cannot_take_writes( + error: Exception, + mock_prisma_client: MagicMock, +) -> None: + """A server that is restarting or unreachable never saw the rollup batch, so the tool + usage increments are kept for the next flush, and the job stops before draining the + auto-router queue into the same outage.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=error) + mock_prisma_client.spend_log_transactions = [] + first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2") + mock_prisma_client.tool_usage_transactions = [first, second] + mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")] + + with suppress(Exception): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert { + "queue_after": mock_prisma_client.tool_usage_transactions, + "autorouter_queue_after": len(mock_prisma_client.autorouter_turn_transactions), + } == {"queue_after": [first, second], "autorouter_queue_after": 1} + + +@pytest.mark.asyncio +async def test_rollups_drop_a_batch_whose_session_the_server_terminated( + mock_prisma_client: MagicMock, +) -> None: + """A terminated session (57P01) may have committed before it was cut, and the rollup + increments are not idempotent, so the batch is dropped rather than counted twice.""" + mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=_postgres_terminated_session()) + mock_prisma_client.spend_log_transactions = [] + mock_prisma_client.tool_usage_transactions = [_tool_usage_transaction("r1")] + + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=MagicMock(), + ) + + assert mock_prisma_client.tool_usage_transactions == [] + + @pytest.mark.asyncio async def test_model_usage_flush_still_drops_a_batch_that_may_have_applied( mock_prisma_client: MagicMock, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 46777b32319..ad3036560a3 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1,28 +1,40 @@ import asyncio +import functools import json import time from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime, timezone from pathlib import Path from types import TracebackType -from typing import Final, Protocol +from typing import ClassVar, Final, Literal, Protocol from unittest.mock import AsyncMock, MagicMock, patch import httpx import openai import pytest import respx -from websockets.exceptions import ConnectionClosedError +from openai import AsyncOpenAI +from websockets.exceptions import ConnectionClosed, ConnectionClosedError from websockets.frames import Close import litellm +from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, RealTimeStreaming +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.models.credentials import CredentialItem +from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy +from litellm.proxy.litellm_pre_call_utils import _apply_resolved_guardrails_to_metadata from litellm.realtime_api import main as realtime_main -from litellm.realtime_api.main import _with_resolved_session_model +from litellm.realtime_api.main import with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyMatchContext +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep +from litellm.types.realtime import RealtimeQueryParams @pytest.fixture @@ -48,19 +60,19 @@ class _CallCapture: def test_resolves_top_level_session_model(): - resolved = _with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") + resolved = with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") assert resolved == {"model": "gpt-realtime"} def test_session_without_model_is_returned_unchanged(): session = {"type": "realtime", "audio": {"input": {}}} - assert _with_resolved_session_model(session, "gpt-realtime") == session + assert with_resolved_session_model(session, "gpt-realtime") == session def test_does_not_clobber_flat_transcription_model(): """The nested transcription model is a different model than the realtime conversation model and must not be overwritten with the routing model.""" - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "whisper-1"}}, "gpt-4o-realtime-preview", ) @@ -68,7 +80,7 @@ def test_does_not_clobber_flat_transcription_model(): def test_does_not_clobber_nested_audio_transcription_model(): - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( { "model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "whisper-1"}}}, @@ -80,7 +92,7 @@ def test_does_not_clobber_nested_audio_transcription_model(): def test_original_session_is_not_mutated(): session = {"model": "alias/gpt-realtime"} - _with_resolved_session_model(session, "gpt-realtime") + with_resolved_session_model(session, "gpt-realtime") assert session == {"model": "alias/gpt-realtime"} @@ -388,6 +400,52 @@ async def test_arealtime_live_session_rejects_streaming_only_pre_call_guardrail( assert live_handler.called is False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_metadata", + ( + {"user_api_key_metadata": {"disable_global_guardrails": True}}, + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["default-on-blocker"]}}, + ), +) +async def test_arealtime_key_metadata_cannot_opt_out_of_a_default_on_realtime_guardrail( + monkeypatch: pytest.MonkeyPatch, key_metadata: dict[str, object] +) -> None: + class FakeLiveHandler: + def __init__(self) -> None: + self.called = False + + async def async_live_session(self, **kwargs: object) -> None: + self.called = True + + live_handler: Final = FakeLiveHandler() + monkeypatch.setattr(realtime_main, "openai_live_sessions", live_handler) + monkeypatch.setattr( + litellm, + "callbacks", + [ + CustomGuardrail( + guardrail_name="default-on-blocker", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + ) + ], + ) + + with pytest.raises(ValueError, match="Guardrails are not supported on OpenAI Live sessions"): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-live-1", + websocket=MagicMock(), + api_key="fake-key", + api_base="https://live-gateway.example/v1", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + live_session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + ) + + assert live_handler.called is False + + @pytest.mark.asyncio async def test_arealtime_live_session_rejects_non_openai_provider(monkeypatch: pytest.MonkeyPatch) -> None: def mock_get_llm_provider( @@ -986,6 +1044,114 @@ async def test_realtime_http_routes_send_a_bare_model_name_to_its_explicit_provi assert gateway.url == f"https://realtime-gateway.example/v1/realtime/{path}" +class _GatewayThatRecordsTheBody: + body: object = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.body = json.loads(request.content) + return httpx.Response(200, json={"client_secret": {"value": "ek_test"}}) + + +@pytest.mark.asyncio +async def test_transcription_session_sends_the_deployment_model_upstream_in_every_transcription_shape() -> None: + gateway: Final = _GatewayThatRecordsTheBody() + await realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={ + "audio": {"input": {"transcription": {"model": "team-transcriber", "language": "en"}}}, + "input_audio_transcription": {"model": "team-transcriber"}, + "input_audio_format": "pcm16", + }, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + assert gateway.body == { + "audio": {"input": {"transcription": {"model": "gpt-transcribe", "language": "en"}}}, + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_format": "pcm16", + } + + +def _create_realtime_session_with_an_explicit_null(route: str, client: AsyncHTTPHandler) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + session={ + "type": "realtime", + "model": "gpt-realtime", + "instructions": None, + "tools": None, + "audio": {"input": {"turn_detection": None}}, + }, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + session={"include": None, "audio": {"input": {"noise_reduction": None}}}, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=_realtime_http_logging(route), + client=client, + ) + return realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={"include": None, "audio": {"input": {"turn_detection": None}}}, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "body_path"), + ( + ("client_secrets", ("session", "audio", "input", "turn_detection")), + ("translation_client_secrets", ("session", "audio", "input", "noise_reduction")), + ("transcription_sessions", ("audio", "input", "turn_detection")), + ), +) +async def test_realtime_session_requests_forward_an_explicit_null_upstream( + route: str, body_path: tuple[str, ...] +) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + await _create_realtime_session_with_an_explicit_null( + route, AsyncHTTPHandler(transport=httpx.MockTransport(gateway)) + ) + + parent: Final = functools.reduce(lambda node, key: node[key], body_path[:-1], gateway.body) + assert body_path[-1] in parent + assert parent[body_path[-1]] is None + session_fields: Final = gateway.body["session"] if body_path[0] == "session" else gateway.body + assert [key for key, value in session_fields.items() if value is None] == [] + + +@pytest.mark.asyncio +async def test_transcription_client_secret_sends_the_deployment_model_as_the_session_model() -> None: + gateway: Final = _GatewayThatRecordsTheSession() + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + session={"type": "transcription", "model": "openai/gpt-live-transcribe"}, + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=_realtime_http_logging("client_secrets"), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-live-transcribe" + assert gateway.session["audio"] == {"input": {"transcription": {"model": "gpt-live-transcribe"}}} + + class _ClientWebSocketWithHeaders: def __init__(self, headers: tuple[tuple[bytes, bytes], ...]) -> None: self.scope: Final = {"headers": headers} @@ -995,11 +1161,51 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) +_GA_DEFAULT_SESSIONS: Final = ( + pytest.param("gpt-realtime-2.1", "realtime", None, id="ga-only-model"), + pytest.param("gpt-realtime-2", "realtime", None, id="ga-only-alias"), + pytest.param("gpt-realtime-2-2026-05-06", "realtime", None, id="ga-only-snapshot"), + # Azure answers beta sessions on these with "The Realtime preview API is no longer available" + # (https://aka.ms/aoai/realtime-ga), checked live 2026-10-28 + pytest.param("gpt-realtime", "realtime", None, id="beta-retired-gpt-realtime"), + pytest.param("gpt-realtime-mini", "realtime", None, id="beta-retired-gpt-realtime-mini"), + pytest.param("gpt-realtime-1.5", "realtime", None, id="beta-retired-gpt-realtime-1.5"), + pytest.param("my-transcribe-deployment", "realtime", {"intent": "transcription"}, id="transcription-intent"), + pytest.param("my-translate-deployment", "translation", None, id="translation-mode"), +) + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_defaults_to_ga_when_none_is_configured( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + None, model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_BETA_CLIENT + ) + + assert protocol.name == "GA" + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_configured_by_the_operator_wins( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + "beta", model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_GA_CLIENT + ) + + assert protocol.name == "beta" + + async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: connect: Final = _ConnectThatStopsAfterCapturingTheUrl() with patch("websockets.connect", connect): await realtime_main._arealtime.__wrapped__( - model="azure/gpt-realtime", + model="azure/my-realtime-deployment", websocket=websocket, api_base="https://my-endpoint.openai.azure.com", api_key="fake-key", @@ -1014,7 +1220,7 @@ async def test_arealtime_azure_ga_client_without_beta_header_dials_the_ga_upstre monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert ( await _azure_backend_url_dialed_for(_GA_CLIENT) - == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-realtime" + == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment" ) @@ -1022,7 +1228,7 @@ async def test_arealtime_azure_ga_client_without_beta_header_dials_the_ga_upstre async def test_arealtime_azure_beta_header_client_keeps_the_beta_upstream(monkeypatch): monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert await _azure_backend_url_dialed_for(_BETA_CLIENT) == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" ) @@ -1030,7 +1236,7 @@ async def test_arealtime_azure_beta_header_client_keeps_the_beta_upstream(monkey async def test_arealtime_azure_explicit_beta_protocol_wins_over_a_ga_client(monkeypatch): monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert await _azure_backend_url_dialed_for(_GA_CLIENT, realtime_protocol="beta") == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" ) @@ -1038,7 +1244,197 @@ async def test_arealtime_azure_explicit_beta_protocol_wins_over_a_ga_client(monk async def test_arealtime_azure_env_beta_protocol_wins_over_a_ga_client(monkeypatch): monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", "beta") assert await _azure_backend_url_dialed_for(_GA_CLIENT) == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + ) + + +class _BetaClientThatStaysConnected: + def __init__(self) -> None: + self.scope: Final = {"headers": ((b"openai-beta", b"realtime=v1"),)} + self.sent: tuple[str, ...] = () + + async def receive_text(self) -> str: + return await asyncio.get_running_loop().create_future() + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + return None + + +class _ConnectToABackendThatSendsOneFrame: + def __init__(self, frame: Mapping[str, object]) -> None: + self.url: str | None = None + self._backend: Final = MagicMock(recv=AsyncMock(side_effect=[json.dumps(frame), ConnectionClosed(None, None)])) + + def __call__( + self, url: str, *, additional_headers: Mapping[str, str], max_size: int, ssl: object + ) -> "_ConnectToABackendThatSendsOneFrame": + self.url = url + return self + + async def __aenter__(self) -> MagicMock: + return self._backend + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + +def _realtime_websocket_logging(*failure_callbacks: CustomLogger) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-realtime-websocket", + function_id="test-realtime-websocket", + dynamic_async_failure_callbacks=list(failure_callbacks), + ) + + +_GA_SESSION_CREATED: Final = { + "type": "session.created", + "event_id": "event_1", + "session": { + "type": "realtime", + "id": "sess_1", + "model": "my-realtime-deployment", + "output_modalities": ["text"], + "audio": {"output": {"voice": "marin"}}, + }, +} + + +async def _azure_session_created_seen_by_a_beta_client( + session_kwargs: Mapping[str, object], +) -> tuple[str | None, object]: + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="azure/my-realtime-deployment", + websocket=client, + api_base="https://my-endpoint.openai.azure.com", + api_key="fake-key", + litellm_logging_obj=_realtime_websocket_logging(), + **session_kwargs, + ) + return connect.url, json.loads(client.sent[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("session_kwargs", "expected_url"), + ( + pytest.param( + {"query_params": {"intent": "transcription"}}, + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription", + id="transcription-intent", + ), + ), +) +async def test_arealtime_azure_beta_client_on_a_ga_session_it_did_not_need_bridging_for_keeps_ga_session_events( + monkeypatch: pytest.MonkeyPatch, session_kwargs: Mapping[str, object], expected_url: str +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(session_kwargs) == (expected_url, _GA_SESSION_CREATED) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("env_protocol", "session_kwargs"), + ( + pytest.param(None, {"realtime_protocol": "GA"}, id="operator-set-ga"), + pytest.param("GA", {}, id="env-set-ga"), + ), +) +async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_operator_pins_ga( + monkeypatch: pytest.MonkeyPatch, env_protocol: str | None, session_kwargs: Mapping[str, object] +) -> None: + if env_protocol is None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + else: + monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", env_protocol) + + assert await _azure_session_created_seen_by_a_beta_client(session_kwargs) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "base_model_kwargs", + ( + pytest.param({"base_model": "azure/gpt-realtime-2.1"}, id="base-model-param"), + pytest.param({"metadata": {"model_info": {"base_model": "azure/gpt-realtime-2.1"}}}, id="model-info"), + ), +) +async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_deployment_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, base_model_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(base_model_kwargs) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +_GA_SESSION_CREATED_BRIDGED_TO_BETA: Final = { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, +} + +_ROUTER_REQUEST_SHAPES: Final = ( + pytest.param({}, id="sdk-router-call"), + pytest.param({"metadata": {"user_api_key_hash": "hashed-key"}}, id="proxy-request-metadata"), +) + + +def _router_with_an_azure_deployment_aliasing(deployment_model: str, base_model: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": deployment_model, + "api_base": "https://my-endpoint.openai.azure.com", + "api_key": "fake-key", + }, + "model_info": {"base_model": base_model}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_realtime_bridges_a_beta_client_to_ga_when_model_info_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-realtime-deployment", "azure/gpt-realtime-2.1-mini" + ) + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + + with patch("websockets.connect", connect): + await router._arealtime(model="team-alias", websocket=client, **request_kwargs) + + assert (connect.url, json.loads(client.sent[0])) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, ) @@ -1130,6 +1526,30 @@ async def test_arealtime_openai_forwards_the_intent_query_param_to_the_upstream_ assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime&intent=chat" +@pytest.mark.asyncio +@pytest.mark.parametrize("intent", (None, "transcription")) +async def test_arealtime_openai_connects_to_api_base_rather_than_the_callers_client_base_url( + intent: str | None, +) -> None: + connect: Final = _ConnectThatStopsAfterCapturingTheUrl() + query_params: Final[RealtimeQueryParams] = ( + {"model": "openai/gateway-realtime", "intent": intent} if intent else {"model": "openai/gateway-realtime"} + ) + with patch("websockets.connect", connect), patch("websockets.asyncio.client.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gateway-realtime", + websocket=_ClosableGaClientWebSocket(), + api_key="fake-key", + api_base="https://configured-gateway.example/", + client=AsyncOpenAI(api_key="caller-key", base_url="https://caller-client.example/v1"), + query_params=query_params, + litellm_logging_obj=FakeLogging(), + ) + + expected_query: Final = "model=gateway-realtime&intent=transcription" if intent else "model=gateway-realtime" + assert connect.url == f"wss://configured-gateway.example/v1/realtime?{expected_query}" + + @pytest.mark.parametrize( ("model", "api_base", "client_query_params", "expected_backend_url"), [ @@ -1416,26 +1836,82 @@ class _GatewayThatRecordsTheAuthorization: @pytest.mark.asyncio @pytest.mark.parametrize( - "route", - [realtime_main.acreate_realtime_client_secret, realtime_main.acreate_realtime_transcription_session], - ids=["client_secrets", "transcription_sessions"], + ("route", "model"), + [ + (realtime_main.acreate_realtime_client_secret, "gpt-realtime"), + (realtime_main.acreate_realtime_transcription_session, "gpt-realtime"), + (realtime_main.acreate_realtime_translation_client_secret, "gpt-realtime-translate"), + ], + ids=["client_secrets", "transcription_sessions", "translation_client_secrets"], ) @respx.mock async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyless_deployment( - federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]] + federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]], model: str ) -> None: _mock_token_exchange() gateway: Final = _GatewayThatRecordsTheAuthorization() await route.__wrapped__( - model="gpt-realtime", + model=model, custom_llm_provider="openai", - litellm_logging_obj=FakeLogging(), + litellm_logging_obj=Logging( + model=model, + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-federated-realtime", + function_id="test-federated-realtime", + ), client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), **federated_deployment, ) assert gateway.authorization == "Bearer federated-bearer" +@pytest.mark.asyncio +@pytest.mark.parametrize("flow", ("client_secret", "calls")) +async def test_translation_http_routes_preserve_explicit_azure_provider(flow: str) -> None: + expected_path: Final = ( + "/openai/v1/realtime/translations/" + + { + "client_secret": "client_secrets", + "calls": "calls", + }[flow] + ) + + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.host == "azure-deployment.example" + assert request.url.path == expected_path + return httpx.Response(200, json={"value": "ek-test", "routed_path": request.url.path}) + + params: Final[Mapping[str, object]] = { + "model": "translation-deployment", + "custom_llm_provider": "azure", + "api_base": "https://azure-deployment.example/openai/v1/", + "api_key": "test-key", + "client": AsyncHTTPHandler(transport=httpx.MockTransport(send_response)), + "litellm_logging_obj": Logging( + model="translation-deployment", + messages=[], + stream=False, + call_type="acreate_realtime_translation_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-translation-routing", + function_id="test-translation-routing", + ), + } + response: Final = await ( + realtime_main.acreate_realtime_translation_client_secret.__wrapped__(**params) + if flow == "client_secret" + else realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek-test", sdp_body=b"v=0", **params + ) + ) + + assert response.status_code == 200 + assert response.json() == {"value": "ek-test", "routed_path": expected_path} + + @pytest.mark.asyncio @respx.mock async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_keyless_deployment( @@ -1453,6 +1929,532 @@ async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_key assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer federated-bearer"} +def _realtime_http_route_call( + route: str, + logging_obj: Logging, + custom_llm_provider: str, + api_base: str, + client: AsyncHTTPHandler | None = None, + metadata: Mapping[str, object] | None = None, +) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + if route == "calls": + return realtime_main.arealtime_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + return realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + + +def _realtime_http_logging(route: str) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id=f"test-realtime-{route}", + function_id=f"test-realtime-{route}", + ) + + +_OPENAI_REALTIME_HTTP_ROUTE_PATHS: Final = { + "client_secrets": "/v1/realtime/client_secrets", + "translation_client_secrets": "/v1/realtime/translations/client_secrets", + "calls": "/v1/realtime/calls", + "translation_calls": "/v1/realtime/translations/calls", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS)) +async def test_openai_realtime_http_routes_surface_the_upstream_error_body_verbatim(route: str) -> None: + upstream_error: Final = { + "error": { + "message": "Invalid model", + "type": "invalid_request_error", + "param": "session.model", + "code": "invalid_value", + } + } + + with pytest.raises(BaseLLMException) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler( + transport=httpx.MockTransport(lambda request: httpx.Response(400, json=upstream_error)) + ), + ) + + assert raised.value.status_code == 400 + assert json.loads(raised.value.message) == upstream_error + + +class _GatewayThatRecordsTheSession: + session: Mapping[str, object] | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.session = json.loads(request.content)["session"] + return httpx.Response(200, json={"value": "ek_test"}) + + +@pytest.mark.asyncio +async def test_client_secret_keeps_the_realtime_session_model_when_input_transcription_names_another() -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + session={ + "type": "realtime", + "model": "gpt-realtime", + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, + }, + custom_llm_provider="openai", + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime" + assert gateway.session["audio"] == {"input": {"transcription": {"model": "gpt-realtime-whisper"}}} + assert logging_obj.model == "gpt-realtime" + + +@pytest.mark.asyncio +async def test_client_secret_session_model_wins_over_a_provider_qualified_top_level_model( + local_model_cost_map: None, +) -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="openai/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1-mini"}, + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime-2.1-mini" + assert logging_obj.model == "gpt-realtime-2.1-mini" + + +class _UpstreamThatMustNotBeCalled: + called: bool = False + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.called = True + return httpx.Response(200, json={"value": "ek_test"}) + + +class _RecordingClientWebSocket: + def __init__(self) -> None: + self.scope: Final = {"headers": ()} + self.sent: tuple[str, ...] = () + self.close_code: int | None = None + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + self.close_code = code + + +def _block_every_request_with_a_guardrail( + monkeypatch: pytest.MonkeyPatch, event_hook: GuardrailEventHooks = GuardrailEventHooks.pre_call +) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [CustomGuardrail(guardrail_name="translation-blocker", event_hook=event_hook, default_on=True)], + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_http_routes_refuse_a_guarded_request_before_any_upstream_call( + route: str, + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +_DEFAULT_ON_GUARDRAIL_OPT_OUTS: Final = ( + pytest.param({"user_api_key_metadata": {"disable_global_guardrails": True}}, id="key-disables-global"), + pytest.param( + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["translation-blocker"]}}, id="team-opts-out" + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "path"), + (("translation_client_secrets", "client_secrets"), ("translation_calls", "calls")), +) +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_http_routes_serve_a_key_opted_out_of_the_default_on_guardrail( + route: str, + path: str, + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + gateway: Final = _GatewayThatRecordsTheUrl() + + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + metadata=key_metadata, + ) + + assert gateway.url == f"https://upstream.example/v1/realtime/translations/{path}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +async def test_translation_http_routes_refuse_a_provider_other_than_openai_and_azure_before_any_upstream_call( + route: str, +) -> None: + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match="supports only the azure, openai providers, got xai") as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="xai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_websocket_closes_1008_for_a_guarded_request_before_the_upstream_connect( + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_websocket_opens_for_a_key_opted_out_of_the_default_on_guardrail( + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[object, ...] = () + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures = (*self.failures, kwargs.get("exception")) + + +@pytest.mark.asyncio +async def test_translation_websocket_refusal_logs_the_session_as_a_failure(monkeypatch: pytest.MonkeyPatch) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + recorder: Final = _FailureRecorder() + logging_obj: Final = _realtime_websocket_logging(recorder) + + with patch("websockets.connect", _CapturingConnect()): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=_RecordingClientWebSocket(), + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=logging_obj, + ) + + assert [(type(failure), getattr(failure, "status_code", None)) for failure in recorder.failures] == [ + (litellm.BadRequestError, 400) + ] + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in str(recorder.failures[0]) + assert logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] is True + + +class _NativeLifecycleGuardrail(CustomGuardrail): + use_native_lifecycle_hooks: ClassVar[bool] = True + + +@pytest.mark.asyncio +async def test_translation_websocket_opens_under_a_native_pre_call_guardrail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [ + _NativeLifecycleGuardrail( + guardrail_name="runner-skipped", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + ], + ) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls", "websocket")) +@pytest.mark.parametrize( + ("key_metadata", "blocked"), + ( + pytest.param({"user_api_key_metadata": {"disable_global_guardrails": True}}, False, id="key-disables-global"), + pytest.param({}, True, id="no-opt-out"), + ), +) +async def test_translation_lets_only_an_opted_out_key_past_a_legacy_string_callback_guardrail( + route: str, + key_metadata: Mapping[str, object], + blocked: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + initialize_callbacks_on_proxy( + value=["lakera_prompt_injection"], premium_user=False, config_file_path="", litellm_settings={} + ) + if route == "websocket": + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + metadata=key_metadata, + ) + if blocked: + assert connect.url is None + assert websocket.close_code == 1008 + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in websocket.sent[0] + else: + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + return + gateway: Final = _GatewayThatRecordsTheUrl() + call: Final = _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + metadata=key_metadata, + ) + if blocked: + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE): + await call + assert gateway.url is None + else: + await call + assert gateway.url is not None + + +@pytest.mark.asyncio +async def test_translation_websocket_closes_1008_for_a_provider_other_than_openai_and_azure() -> None: + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="xai/grok-voice", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + "Realtime translation supports only the azure, azure_ai, openai providers, got xai" + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls", "websocket")) +@pytest.mark.parametrize("mode", ("pre_call", "post_call")) +@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("selected,condition_matches", ((True, True), (True, False), (False, True))) +async def test_translation_rejects_only_applicable_policy_pipelines( + route: str, + mode: Literal["pre_call", "post_call"], + metadata_bucket: str, + selected: bool, + condition_matches: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + request_data: Final[dict[str, object]] = {"metadata": {}, "litellm_metadata": {}} + _apply_resolved_guardrails_to_metadata( + data=request_data, + metadata_variable_name=metadata_bucket, + context=PolicyMatchContext(model="translation-deployment"), + policy_names=["translation-policy"] if selected else [], + policies={ + "translation-policy": Policy( + condition=PolicyCondition(model="translation-deployment" if condition_matches else "unrelated-model"), + pipeline=GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="speech-policy", on_fail="block")]), + ) + }, + ) + metadata: Final = realtime_main._build_litellm_metadata(request_data) + blocked: Final = selected and condition_matches + if route == "websocket": + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + litellm_metadata=metadata, + ) + if blocked: + assert connect.url is None + assert websocket.close_code == 1008 + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in websocket.sent[0] + else: + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + return + upstream: Final = _UpstreamThatMustNotBeCalled() + call: Final = _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + metadata=metadata, + ) + if blocked: + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await call + assert raised.value.status_code == 400 + assert upstream.called is False + else: + response: Final = await call + assert response.status_code == 200 + assert upstream.called is True + + @pytest.mark.asyncio async def test_arealtime_routes_soniox_with_the_deployment_options(monkeypatch): from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig diff --git a/tests/unit/repositories/test_spend_log_insert.py b/tests/unit/repositories/test_spend_log_insert.py new file mode 100644 index 00000000000..a099551086b --- /dev/null +++ b/tests/unit/repositories/test_spend_log_insert.py @@ -0,0 +1,31 @@ +from datetime import datetime, timezone +import json + +import pytest + +from litellm.repositories.spend_log_insert import spend_log_insert_statement + + +@pytest.mark.parametrize( + "rows", + [(), ({"request_id": "request", "unknown_column": 1},), ({"request_id": "first"}, {"spend": 1})], +) +def test_insert_rejects_empty_unknown_or_inconsistent_columns(rows): + with pytest.raises(ValueError, match="Spend log insert requires"): + spend_log_insert_statement(rows) + + +def test_insert_keeps_json_and_timestamps_in_bound_data(): + timestamp = datetime(2026, 11, 2, tzinfo=timezone.utc) + request_id = "request'); DROP TABLE unrelated; --" + statement, encoded = spend_log_insert_statement(({ + "request_id": request_id, "startTime": timestamp, "metadata": '{"project":"project"}', + "request_tags": '["tag"]', "messages": '[{"role":"user","content":"hello"}]', + },)) + row = json.loads(encoded)[0] + assert request_id not in statement + assert row["request_id"] == request_id + assert row["startTime"] == timestamp.isoformat() + assert row["metadata"] == {"project": "project"} + assert row["request_tags"] == ["tag"] + assert row["messages"] == [{"role": "user", "content": "hello"}] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f3de49b8bfc..0c39146cdf5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6540,6 +6540,31 @@ def test_an_application_inference_profile_gets_no_item_marks_under_drop_params() assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 0 + +def test_a_bedrock_deployment_keeps_item_marks_by_its_own_flag_not_a_siblings_on_the_same_model( + local_model_cost_map: None, +): + from litellm import Router + + Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": {"model": "bedrock/moonshotai.kimi-k3", "aws_region_name": "us-east-1"}, + "model_info": {"id": "opted-in-id", "supports_prompt_cache_breakpoint": True}, + } + ] + ) + marked_system: Final = {"role": "system", "content": "House rules", "cache_control": _EPHEMERAL} + + def bridged_system(model_info: Mapping[str, object]) -> Mapping[str, object]: + return _bridged_request(marked_system, model="bedrock/moonshotai.kimi-k3", model_info=model_info)[1] + + assert ( + bridged_system({"id": "flagless-id"}), + bridged_system({"id": "opted-in-id", "supports_prompt_cache_breakpoint": True}), + ) == ({"role": "system", "content": "House rules"}, marked_system) + def test_the_shared_input_transform_keeps_every_item_mark_for_a_request_it_sends(): messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast(ResponseInputParam, list(_marked_turns(5))), # cast-ok: cache_control is not an input TypedDict key diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index ec0be896095..037004ab999 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -540,13 +540,9 @@ async def test_mcp_allowed_tools_filtering(): ] # First filter by allowed tools - filtered_tools_with_duplicates = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_with_duplicates, - mcp_tools_with_litellm_proxy=cast( - List[ToolParam], mcp_tool_config_with_duplicates - ), - ) + filtered_tools_with_duplicates = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_with_duplicates, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], mcp_tool_config_with_duplicates), ) # Then deduplicate the filtered tools @@ -605,11 +601,9 @@ async def test_mcp_allowed_tools_filtering(): }, ] - filtered_tools_multiple = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_from_server, - mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), - ) + filtered_tools_multiple = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_from_server, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), ) # Should return union of all allowed tools (3 unique tools) diff --git a/tests/unit/router_rust/__init__.py b/tests/unit/router_rust/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/router_rust/test_surface_fallbacks.py b/tests/unit/router_rust/test_surface_fallbacks.py new file mode 100644 index 00000000000..dec6e864dae --- /dev/null +++ b/tests/unit/router_rust/test_surface_fallbacks.py @@ -0,0 +1,11 @@ +from dataclasses import fields + +from litellm.router_rust.surface.fallbacks import MOCK_FLAGS +from litellm.types.router import MockRouterTestingParams + + +def test_mock_flags_cover_every_mock_testing_param(): + """The Rust core serves exactly the ``mock_testing_*`` flags the Python core reads: the + ``MockRouterTestingParams`` fields and ``mock_testing_rate_limit_error``. A new flag fails + here until ``MOCK_FLAGS`` serves it.""" + assert set(MOCK_FLAGS) == {f.name for f in fields(MockRouterTestingParams)} | {"mock_testing_rate_limit_error"} diff --git a/tests/unit/router_strategy/adaptive_router/test_signals.py b/tests/unit/router_strategy/adaptive_router/test_signals.py index 2773c13a812..bd58b4c668d 100644 --- a/tests/unit/router_strategy/adaptive_router/test_signals.py +++ b/tests/unit/router_strategy/adaptive_router/test_signals.py @@ -81,6 +81,12 @@ def test_exhaustion_fires_on_infra_signal(fixture): assert state.exhaustion_count >= 1 +@pytest.mark.parametrize("status", [503, 529]) +def test_exhaustion_fires_on_an_overloaded_answer(status): + state, _ = _replay([{"user_content": "fix the build", "response_status": status}]) + assert state.exhaustion_count == 1 + + def test_no_signals_on_clean_session(): state, _ = _replay(_load("clean_no_signals")) assert state.misalignment_count == 0 diff --git a/tests/unit/router_strategy/test_budget_limiter.py b/tests/unit/router_strategy/test_budget_limiter.py index c0583566531..d24d0598d87 100644 --- a/tests/unit/router_strategy/test_budget_limiter.py +++ b/tests/unit/router_strategy/test_budget_limiter.py @@ -7,6 +7,7 @@ the callback raise before any spend was recorded, so those budgets never moved. """ from datetime import datetime, timezone +from functools import partial from typing import Final import litellm @@ -17,6 +18,7 @@ from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.types.utils import BudgetConfig +from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG import sys, os, asyncio, time, random @@ -38,6 +40,7 @@ def _success_kwargs( call_type: str = "aresponses", response_cost: float = 0.25, model_id: str = "deployment-1", + guardrail_information: tuple[dict[str, object], ...] = (), ) -> dict[str, object]: provider_params: Final[dict[str, str]] = ( {} if provider_in_litellm_params is None else {"custom_llm_provider": provider_in_litellm_params} @@ -51,6 +54,7 @@ def _success_kwargs( "response_cost": response_cost, "model_id": model_id, "custom_llm_provider": provider_in_payload, + "guardrail_information": guardrail_information, }, } @@ -462,6 +466,7 @@ def _completed_budget_batch(batch_id: str): input_file_id="file-budget", object="batch", status="completed", + output_file_id="file-budget-output", ) @@ -498,6 +503,130 @@ async def test_batch_polls_charge_each_router_budget_once(monkeypatch, call_type assert tuple([await cache.async_get_cache(key) for key in keys]) == (1.0, 1.0, 1.0) + +@pytest.mark.asyncio +async def test_a_batch_retrieve_before_its_cost_is_final_charges_no_router_budget_even_for_its_guardrail(): + from litellm.types.utils import LiteLLMBatch + + cache: Final = DualCache() + limiter: Final = RouterBudgetLimiting( + dual_cache=cache, + provider_budget_config={"openai": {"budget_limit": 100, "time_period": "1d"}}, + model_list=[ + { + "model_name": "batch-budget", + "litellm_params": {"model": "openai/batch-budget", "max_budget": 100, "budget_duration": "1d"}, + "model_info": {"id": "deployment-1"}, + } + ], + ) + retrieve: Final = partial( + _success_kwargs, + provider_in_litellm_params=None, + provider_in_payload="openai", + call_type="aretrieve_batch", + guardrail_information=({"guardrail_name": "priced", "guardrail_cost": 0.01},), + ) + + def polled(status: str, output_file_id: str | None = None) -> LiteLLMBatch: + return LiteLLMBatch( + id="batch-running", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-budget", + object="batch", + status=status, + output_file_id=output_file_id, + ) + + for response_cost, batch in ( + (0.01, polled("in_progress")), + (0.01, polled("completed")), + (0.26, polled("completed", "file-out")), + (0.26, polled("completed", "file-out")), + ): + await limiter.async_log_success_event(retrieve(response_cost=response_cost), batch, None, None) + + keys: Final = ("provider_spend:openai:1d", "deployment_spend:deployment-1:1d") + assert tuple([await cache.async_get_cache(key) for key in keys]) == pytest.approx((0.26, 0.26)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ("avideo_status", "video_status")) +async def test_video_polls_charge_each_router_budget_once(call_type): + from litellm.types.videos.main import VideoObject + + cache: Final = DualCache() + limiter: Final = RouterBudgetLimiting( + dual_cache=cache, + provider_budget_config={"openai": {"budget_limit": 100, "time_period": "1d"}}, + model_list=[ + { + "model_name": "video-budget", + "litellm_params": {"model": "openai/sora-2", "max_budget": 100, "budget_duration": "1d"}, + "model_info": {"id": "deployment-1"}, + } + ], + ) + kwargs: Final = { + **_success_kwargs(provider_in_litellm_params=None, provider_in_payload="openai", call_type=call_type), + POLLED_VIDEO_STATUS_KWARG: "completed", + } + video: Final = VideoObject(id="video-first", object="video", status="completed") + keys: Final = ("provider_spend:openai:1d", "deployment_spend:deployment-1:1d") + + await asyncio.gather(*(limiter.async_log_success_event(kwargs, video, None, None) for _ in range(10))) + assert tuple([await cache.async_get_cache(key) for key in keys]) == (0.25, 0.25) + + await limiter.async_log_success_event( + kwargs, VideoObject(id="video-second", object="video", status="completed"), None, None + ) + assert tuple([await cache.async_get_cache(key) for key in keys]) == (0.5, 0.5) + + +@pytest.mark.asyncio +async def test_every_video_poll_pays_its_guardrail_and_only_the_finished_poll_pays_the_router_for_the_video(): + from litellm.types.videos.main import VideoObject + + cache: Final = DualCache() + limiter: Final = RouterBudgetLimiting( + dual_cache=cache, + provider_budget_config={"openai": {"budget_limit": 100, "time_period": "1d"}}, + model_list=[ + { + "model_name": "video-budget", + "litellm_params": {"model": "openai/sora-2", "max_budget": 100, "budget_duration": "1d"}, + "model_info": {"id": "deployment-1"}, + } + ], + ) + priced: Final = ({"guardrail_name": "priced", "guardrail_cost": 0.01},) + poll: Final = partial( + _success_kwargs, + provider_in_litellm_params=None, + provider_in_payload="openai", + call_type="avideo_status", + guardrail_information=priced, + ) + + await limiter.async_log_success_event( + {**poll(response_cost=0.01), POLLED_VIDEO_STATUS_KWARG: "in_progress"}, + VideoObject(id="video-first", object="video", status="in_progress"), + None, + None, + ) + for _ in range(3): + await limiter.async_log_success_event( + {**poll(response_cost=0.26), POLLED_VIDEO_STATUS_KWARG: "completed"}, + VideoObject(id="video-first", object="video", status="completed"), + None, + None, + ) + + keys: Final = ("provider_spend:openai:1d", "deployment_spend:deployment-1:1d") + assert tuple([await cache.async_get_cache(key) for key in keys]) == pytest.approx((0.29, 0.29)) + @pytest.mark.asyncio async def test_batch_charge_is_shared_across_workers_and_requires_redis_confirmation(): from fakeredis import FakeServer, FakeRedis as FakeSyncRedis diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index ef7b9e485d4..d0e2b381253 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -359,6 +359,52 @@ class TestComplexityRouterInit: "tier-probability:MEDIUM=0.900000", ) + @pytest.mark.asyncio + async def test_jev_choice_without_confidence_routes_and_reports_no_confidence(self, mock_router_instance): + client = _StaticJevClient( + JevSystemOneResponse( + answers={ + "tier": JevChoiceAnswer( + type="choice", + choice="MEDIUM", + probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9}, + ) + }, + ) + ) + router = ComplexityRouter( + "test-router", + mock_router_instance, + { + "classifier_type": "jev", + "jev_classifier_config": {"api_key": "test", "timeout_ms": 100}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + jev_client=client, + ) + + outcome = await router.aclassify("Explain this") + result = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={}, + messages=[{"role": "user", "content": "Explain this"}], + ) + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.confidence is None + assert outcome.signals == ( + "jev-classifier:MEDIUM", + "tier-probability:SIMPLE=0.100000", + "tier-probability:MEDIUM=0.900000", + ) + assert result is not None + assert result.routing_decision is not None + assert result.routing_decision["classifier_probabilities"] == {"SIMPLE": 0.1, "MEDIUM": 0.9} + assert "classifier_confidence" not in result.routing_decision + @pytest.mark.asyncio async def test_jev_pre_routing_hook_exposes_routing_decision_provenance( self, mock_router_instance, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/router_strategy/test_least_busy.py b/tests/unit/router_strategy/test_least_busy.py index 5b3af388ae5..f156a5ee405 100644 --- a/tests/unit/router_strategy/test_least_busy.py +++ b/tests/unit/router_strategy/test_least_busy.py @@ -1,9 +1,16 @@ import asyncio +import contextlib +import contextvars +import datetime import json import logging +import threading +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor from typing import Final import httpx +import openai import pytest import respx @@ -11,11 +18,19 @@ import litellm from litellm import Router from litellm.caching.caching import DualCache -from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler -from litellm.types.utils import ModelResponse +from litellm.router_strategy.least_busy import ( + IN_FLIGHT_COUNT_TTL_SECONDS, + SELECTION_RESERVATION_TTL_SECONDS, + LeastBusyLoggingHandler, + owning_picks, + picks_reserve, + release_unstarted_picks, +) +from litellm.types.utils import CallTypes, ModelResponse GROUP: Final = "least-busy-group" DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}} @@ -23,8 +38,11 @@ DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}} HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B] -def _call_kwargs(deployment_id: str) -> dict[str, object]: - return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}} +def _call_kwargs(deployment_id: str, call_id: str = "call-1") -> dict[str, object]: + return { + "litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}, + "litellm_call_id": call_id, + } class SharedRedisCounters: @@ -58,6 +76,20 @@ class SharedRedisCounters: return self.batch_get_counts(key_list) +class SlowRedisRead(SharedRedisCounters): + """Answers a count read with the counts from when the read started, after letting other work run.""" + + def __init__(self) -> None: + super().__init__() + self.while_reading: Callable[[], None] = lambda: None + + async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]: + counts_when_asked: Final = self.batch_get_counts(key_list) + await asyncio.sleep(0) + self.while_reading() + return counts_when_asked + + def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler: cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared) # pyright: ignore[reportArgumentType] # duck-typed Redis double return LeastBusyLoggingHandler(router_cache=cache) @@ -81,6 +113,16 @@ async def test_worker_routes_around_a_request_another_worker_started() -> None: assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A +@pytest.mark.asyncio +async def test_a_request_that_starts_while_redis_is_read_still_counts_on_its_deployment() -> None: + redis: Final = SlowRedisRead() + worker: Final = _worker(redis) + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + redis.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + def test_sync_pick_reads_the_shared_counts() -> None: shared: Final = SharedRedisCounters() streaming_worker: Final = _worker(shared) @@ -98,6 +140,44 @@ def test_sync_pick_reads_the_shared_counts() -> None: assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A +def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]: + return { + "litellm_params": { + "metadata": {"tags": ["caller-tag"]}, + "litellm_metadata": {"model_group": GROUP}, + "model_info": {"id": deployment_id}, + }, + "litellm_call_id": "call-1", + } + + +def test_a_request_routed_through_litellm_metadata_counts_while_in_flight() -> None: + shared: Final = SharedRedisCounters() + calling_worker: Final = _worker(shared) + picking_worker: Final = _worker(shared) + + assert calling_worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + calling_worker.log_pre_api_call(model="m", messages=[], kwargs=_routed_through_litellm_metadata("dep-a")) + + assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + calling_worker.log_success_event(_routed_through_litellm_metadata("dep-a"), None, None, None) + + assert calling_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_closed_realtime_session_leaves_its_deployment_idle() -> None: + shared: Final = SharedRedisCounters() + worker: Final = _worker(shared) + session_event: Final = {**_routed_through_litellm_metadata("dep-a"), "call_type": CallTypes.arealtime.value} + + for _ in range(5): + worker.log_pre_api_call(model="m", messages=[], kwargs=session_event) + worker.log_success_event(session_event, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + def test_the_handler_never_pushes_a_counters_ttl_forward() -> None: """A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that stuck count an hour after the key was created, which only works while nothing writes the TTL @@ -259,12 +339,16 @@ ROUTER_GROUP: Final = "least-busy-router-group" ROUTER_DEPLOYMENT_IDS: Final = ("1", "2", "3") -def _least_busy_router() -> Router: +def _least_busy_router(group: str = ROUTER_GROUP, *, use_in_pass_through: bool = False) -> Router: return Router( model_list=[ { - "model_name": ROUTER_GROUP, - "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": f"key-{deployment_id}"}, + "model_name": group, + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_key": f"key-{deployment_id}", + "use_in_pass_through": use_in_pass_through, + }, "model_info": {"id": deployment_id}, } for deployment_id in ROUTER_DEPLOYMENT_IDS @@ -391,3 +475,575 @@ async def test_router_picks_least_busy_deployment_and_completion_restores_counts assert response._hidden_params["model_id"] == "1" assert route.calls.last.request.headers["authorization"] == "Bearer key-1" assert _router_counts(router) == seeded + + +@pytest.mark.asyncio +async def test_router_spreads_requests_that_start_together( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + burst: Final = 2 * len(ROUTER_DEPLOYMENT_IDS) + all_started: Final = asyncio.Barrier(burst) + + async def completion_after_all_started(_: httpx.Request) -> httpx.Response: + await all_started.wait() + return httpx.Response( + 200, + json={ + "id": "chatcmpl-least-busy", + "object": "chat.completion", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + side_effect=completion_after_all_started + ) + router: Final = _least_busy_router() + + await asyncio.gather( + *(router.acompletion(model=ROUTER_GROUP, messages=[{"role": "user", "content": "hi"}]) for _ in range(burst)) + ) + + one_per_deployment: Final = tuple(f"Bearer key-{deployment_id}" for deployment_id in ROUTER_DEPLOYMENT_IDS) + assert sorted(call.request.headers["authorization"] for call in route.calls) == sorted(one_per_deployment * 2) + + +class _ManualClock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +def test_a_reservation_that_never_reaches_the_api_call_expires() -> None: + clock: Final = _ManualClock() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), clock=clock) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = 1.0 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + clock.now = 2.0 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.5 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +def test_a_fresh_reservation_outlives_the_ttl_of_the_first_one_on_its_deployment() -> None: + clock: Final = _ManualClock() + cache: Final = DualCache(in_memory_cache=InMemoryCache(clock=clock)) + worker: Final = LeastBusyLoggingHandler(router_cache=cache, clock=clock) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + worker.log_success_event(kwargs=_call_kwargs("dep-a"), response_obj=None, start_time=None, end_time=None) + clock.now = SELECTION_RESERVATION_TTL_SECONDS - 0.1 + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.1 + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def test_a_started_request_moves_its_reservation_into_the_in_flight_count() -> None: + worker: Final = _worker(None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +def test_a_request_started_through_another_selector_of_the_router_releases_the_reservation() -> None: + cache: Final = DualCache() + reservations: Final = InMemoryCache() + picker: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + registered: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations) + + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + registered.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + +class _StoreWhereThreadsReadBeforeWriting: + def __init__(self) -> None: + self._store: Final = InMemoryCache() + self._first_reads: Final = threading.Barrier(2, timeout=0.5) + self._thread_state: Final = threading.local() + self._meeting: Final = threading.Event() + + @contextlib.contextmanager + def first_reads_meet(self) -> Generator[None, None, None]: + self._meeting.set() + yield + self._meeting.clear() + + def get_cache(self, key: str) -> object: + value: Final = self._store.get_cache(key) + if self._meeting.is_set() and not getattr(self._thread_state, "has_read", False): + self._thread_state.has_read = True + with contextlib.suppress(threading.BrokenBarrierError): + self._first_reads.wait() + return value + + def set_cache(self, key: str, value: tuple[object, ...], *, ttl: float) -> None: + self._store.set_cache(key, value, ttl=ttl) + + def delete_cache(self, key: str) -> None: + self._store.delete_cache(key) + + +def test_two_threads_picking_at_once_reserve_different_deployments() -> None: + store: Final = _StoreWhereThreadsReadBeforeWriting() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store) + + with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool: + picks: Final = tuple(pool.map(lambda _: worker.get_available_deployments(GROUP, HEALTHY, reserve=True), range(2))) + + assert sorted(pick["model_info"]["id"] for pick in picks if pick is not None) == ["dep-a", "dep-b"] + + +def test_a_reservation_and_a_release_on_one_deployment_at_once_both_count() -> None: + store: Final = _StoreWhereThreadsReadBeforeWriting() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b")) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b")) + + with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool: + reserved: Final = pool.submit(worker.get_available_deployments, GROUP, HEALTHY, True) + started: Final = pool.submit(worker.log_pre_api_call, "m", [], _call_kwargs("dep-a")) + assert reserved.result() is DEPLOYMENT_A + started.result() + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + + +class _SlowLocalRead(InMemoryCache): + """Answers a count read with the counts from when the read started, once another thread has begun its request.""" + + def __init__(self) -> None: + super().__init__() + self.while_reading: Callable[[], None] = lambda: None + self.other_thread: threading.Thread | None = None + + def batch_get_cache(self, keys: list[str], **kwargs: object) -> list[object]: + counts_when_asked: Final = super().batch_get_cache(keys, **kwargs) + began: Final = threading.Event() + + def begin_then_run() -> None: + began.set() + self.while_reading() + + self.other_thread = threading.Thread(target=begin_then_run) + self.other_thread.start() + began.wait() + self.other_thread.join(timeout=0.2) + return counts_when_asked + + +def test_a_request_that_starts_on_another_thread_while_counts_are_read_still_counts_on_its_deployment() -> None: + local: Final = _SlowLocalRead() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(in_memory_cache=local)) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + local.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + picked_while_the_other_thread_started: Final = worker.get_available_deployments(GROUP, HEALTHY, reserve=True) + if local.other_thread is not None: + local.other_thread.join() + + assert picked_while_the_other_thread_started is DEPLOYMENT_B + + +def _router_pick(router: Router, group: str = ROUTER_GROUP) -> str: + deployment: Final = router.get_available_deployment(group, request_kwargs={"metadata": {}}) + return deployment["model_info"]["id"] + + +def test_a_pick_stays_reserved_while_other_router_keys_fill_its_cache() -> None: + router: Final = _least_busy_router() + + with picks_reserve(True): + first: Final = _router_pick(router) + for index in range(DEFAULT_MAX_SIZE_IN_MEMORY): + router.health().store.set_cache(f"deployment-usage-{index}", 1, local_only=True, ttl=60) + + assert _router_pick(router) != first + + +@pytest.mark.asyncio +async def test_looking_up_a_deployment_does_not_hold_it_for_later_picks() -> None: + router: Final = _least_busy_router() + + sync_picks: Final = [_router_pick(router) for _ in range(3)] + async_picks: Final = [ + (await router.async_get_available_deployment(ROUTER_GROUP, request_kwargs={"metadata": {}}))["model_info"]["id"] + for _ in range(3) + ] + + assert sync_picks + async_picks == [ROUTER_DEPLOYMENT_IDS[0]] * 6 + + +@pytest.mark.asyncio +async def test_looking_up_a_pass_through_deployment_does_not_hold_it_for_later_picks() -> None: + router: Final = _least_busy_router(use_in_pass_through=True) + + selected: Final = [ + (await router.select(ROUTER_GROUP, {"metadata": {}}, kind="pass_through"))["model_info"]["id"] for _ in range(2) + ] + async_picks: Final = [ + (await router.async_get_available_deployment_for_pass_through(ROUTER_GROUP, request_kwargs={"metadata": {}}))[ + "model_info" + ]["id"] + for _ in range(2) + ] + sync_picks: Final = [ + router.get_available_deployment_for_pass_through(ROUTER_GROUP, request_kwargs={"metadata": {}})["model_info"]["id"] + for _ in range(2) + ] + + assert selected + async_picks + sync_picks == [ROUTER_DEPLOYMENT_IDS[0]] * 6 + + +@pytest.mark.asyncio +async def test_a_cache_hit_releases_the_pick_it_reserved() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + await worker.async_log_success_event({**_call_kwargs("dep-b"), "cache_hit": True}, None, None, None) + + assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + + worker.log_success_event({**_call_kwargs("dep-a", "cached-call"), "cache_hit": True}, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event]) +def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick( + outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None], +) -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + outcome(worker, _call_kwargs("dep-a"), None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None: + worker: Final = _worker(None) + rejected: Final = {**_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True} + + worker.log_pre_api_call(model="m", messages=[], kwargs=rejected) + worker.log_failure_event(rejected, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_call_logged_under_its_callers_group_name_frees_the_pick_made_for_another_group() -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments("tier-group", HEALTHY, reserve=True) is DEPLOYMENT_A + + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a")) + worker.log_success_event(_call_kwargs("dep-a"), None, None, None) + + assert worker.get_available_deployments("tier-group", HEALTHY) is DEPLOYMENT_A + + +def test_a_fallback_attempt_that_fails_before_reaching_its_provider_frees_its_pick() -> None: + worker: Final = _worker(None) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "busy-call")) + first_attempt_started_at: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + first_attempt: Final = {**_call_kwargs("dep-a", "fallback-call"), "api_call_start_time": first_attempt_started_at} + worker.log_pre_api_call(model="m", messages=[], kwargs=first_attempt) + worker.log_failure_event(first_attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + second_attempt: Final = {**_call_kwargs("dep-b", "fallback-call"), "api_call_start_time": first_attempt_started_at} + worker.log_failure_event(second_attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +def test_ending_one_request_frees_only_the_picks_it_never_started() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="ended-call") is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="running-call") is DEPLOYMENT_A + + release_unstarted_picks(reservations, "ended-call") + + assert [worker.get_available_deployments(GROUP, HEALTHY, reserve=True) for _ in range(2)] == [ + DEPLOYMENT_A, + DEPLOYMENT_B, + ] + + +def test_a_call_without_a_call_id_turns_its_own_pick_into_its_in_flight_count() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + + with owning_picks(reservations, None): + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "attempt-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_another_call_starting_leaves_the_pick_of_a_call_without_a_call_id_reserved() -> None: + reservations: Final = InMemoryCache() + worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=reservations) + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b", "busy-call")) + other_call: Final = contextvars.copy_context() + + with owning_picks(reservations, None): + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + other_call.run(worker.log_pre_api_call, model="m", messages=[], kwargs=_call_kwargs("dep-a", "other-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +def test_a_call_started_without_a_pick_of_its_own_leaves_another_calls_pick_reserved() -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True, call_id="picked-call") is DEPLOYMENT_A + + worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a", "looked-up-call")) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +def _fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": "unreachable-group", + "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": "key-unreachable"}, + "model_info": {"id": "unreachable"}, + }, + *( + { + "model_name": ROUTER_GROUP, + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": f"key-{deployment_id}"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ROUTER_DEPLOYMENT_IDS + ), + ], + routing_strategy="least-busy", + num_retries=0, + fallbacks=[{"unreachable-group": [ROUTER_GROUP]}], + ) + + +@pytest.mark.asyncio +async def test_a_fallback_hop_rejected_before_its_provider_frees_its_pick_when_every_hop_shares_one_logging_object( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(500, json={"error": {"message": "upstream down"}}) + ) + router: Final = _fallback_router() + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model="unreachable-group", + messages=[{"role": "user", "content": "hi"}], + logit_bias={"1": 1}, + litellm_call_id="fallback-call", + ) + + with pytest.raises(litellm.InternalServerError): + await router.acompletion(**request, litellm_logging_obj=logging_obj) + await _drain_logging() + + assert {call.request.headers["authorization"] for call in route.calls} == {"Bearer key-unreachable"} + assert _router_pick(router) == ROUTER_DEPLOYMENT_IDS[0] + + +def test_a_retry_that_starts_before_the_failed_attempt_is_logged_counts_each_attempt_once() -> None: + worker: Final = _worker(None) + attempt: Final = _call_kwargs("dep-a", "retried-call") + + worker.log_pre_api_call(model="m", messages=[], kwargs=attempt) + worker.log_pre_api_call(model="m", messages=[], kwargs=attempt) + worker.log_failure_event(attempt, None, None, None) + worker.log_success_event(attempt, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B + + +async def _drain_logging() -> None: + for _ in range(10): + await asyncio.sleep(0) + await GLOBAL_LOGGING_WORKER.flush() + + +def _called_deployment(route: respx.Route) -> str: + return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-") + + +_BATCH: Final = { + "id": "batch_least_busy", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-least-busy", + "completion_window": "24h", + "status": "validating", + "created_at": 1, +} +_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)]) +async def test_a_batch_create_does_not_keep_its_deployment_busy( + provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response) + router: Final = _least_busy_router() + + with contextlib.suppress(openai.BadRequestError): + await router.acreate_batch( + model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy" + ) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +_VECTOR_STORE: Final = { + "id": "vs_least_busy", + "object": "vector_store", + "created_at": 1, + "name": "least-busy", + "usage_bytes": 0, + "file_counts": {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0}, + "status": "completed", +} +_REJECTED_VECTOR_STORE: Final = {"error": {"message": "Invalid name", "type": "invalid_request_error"}} +_CALL_IDENTITIES: Final = ({"litellm_call_id": "least-busy-call"}, {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +@pytest.mark.parametrize( + "provider_response", [httpx.Response(200, json=_VECTOR_STORE), httpx.Response(400, json=_REJECTED_VECTOR_STORE)] +) +async def test_a_vector_store_create_does_not_keep_its_deployment_busy( + provider_response: httpx.Response, + call_identity: dict[str, str], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores").mock(return_value=provider_response) + router: Final = _least_busy_router() + + with contextlib.suppress(openai.BadRequestError): + await router.avector_store_create(model=ROUTER_GROUP, name="least-busy", **call_identity) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +async def test_listing_fine_tuning_jobs_does_not_keep_its_deployment_busy( + call_identity: dict[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.get(url__startswith="https://api.openai.com/v1/fine_tuning/jobs").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [], "has_more": False}) + ) + router: Final = _least_busy_router() + + await router.alist_fine_tuning_jobs(model=ROUTER_GROUP, **call_identity) + await _drain_logging() + + assert _router_pick(router) == _called_deployment(route) + + +TEXT_GROUP: Final = "openai/least-busy-text-group" + + +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +def test_a_sync_text_completion_does_not_keep_its_deployment_busy( + call_identity: dict[str, str], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "cmpl-least-busy", + "object": "text_completion", + "created": 1, + "model": "least-busy-text-group", + "choices": [{"text": "ok", "index": 0, "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ) + ) + router: Final = _least_busy_router(TEXT_GROUP) + + router.text_completion(model=TEXT_GROUP, prompt="Say ok", **call_identity) + + assert _router_pick(router, TEXT_GROUP) == _called_deployment(route) + + + +@pytest.mark.parametrize("call_identity", _CALL_IDENTITIES) +def test_a_sync_responses_call_that_fails_before_reaching_its_provider_frees_its_pick( + call_identity: dict[str, str], +) -> None: + router: Final = _least_busy_router() + + with pytest.raises(litellm.InternalServerError): + router.responses(model=ROUTER_GROUP, input="hi", api_base="not-a-url", **call_identity) + + assert _router_pick(router) == ROUTER_DEPLOYMENT_IDS[0] + +@pytest.mark.asyncio +async def test_a_batch_cancel_does_not_keep_its_deployment_busy( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock( + return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"}) + ) + router: Final = _least_busy_router() + + await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy") + + assert _router_pick(router) == _called_deployment(route) + + +@pytest.mark.asyncio +async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None: + router: Final = _least_busy_router() + + picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}}) + + assert _router_pick(router) == picked["model_info"]["id"] diff --git a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py index 129bb10eccc..f04ade1ce2d 100644 --- a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py +++ b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py @@ -2,7 +2,9 @@ # This tests the router's ability to pick deployment with lowest tpm using 'usage-based-routing-v2-v2' import asyncio +import base64 import importlib +import io import os import time import traceback @@ -15,6 +17,7 @@ from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outco from unittest.mock import patch import pytest +from PIL import Image import litellm from litellm import Router @@ -23,7 +26,7 @@ from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_tpm_rpm_v2 import ( LowestTPMLoggingHandler_v2 as LowestTPMLoggingHandler, ) -from litellm.types.router import DeploymentTypedDict +from litellm.types.router import DeploymentTypedDict, RouterRateLimitError from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -347,6 +350,55 @@ def test_router_skip_rate_limited_deployments(): print(f"An exception occurred! {str(e)}") +def _embedding_image_uri() -> str: + image: Final = io.BytesIO() + Image.new("RGB", (1024, 1024)).save(image, format="PNG", compress_level=0) + return f"data:image/png;base64,{base64.b64encode(image.getvalue()).decode()}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "routing_strategy", + ["usage-based-routing", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"], +) +@pytest.mark.parametrize( + ("litellm_params", "model_info", "admitted"), + [ + ({"model": "vertex_ai/gemini-embedding-2"}, {}, True), + ( + {"model": "openai/team-embedder", "api_base": "https://embedder.invalid"}, + {"base_model": "vertex_ai/gemini-embedding-2"}, + True, + ), + ({"model": "text-embedding-3-small"}, {}, False), + ], + ids=["image-model", "image-base-model", "text-model"], +) +async def test_deployment_tpm_counts_embedding_image_as_media_only_on_image_models( + routing_strategy: str, litellm_params: Dict[str, str], model_info: Dict[str, str], admitted: bool +) -> None: + uri: Final = _embedding_image_uri() + assert len(uri) // 4 > 75_000 + router: Final = Router( + model_list=[ + { + "model_name": "embedder", + "litellm_params": {**litellm_params, "tpm": 75_000}, + "model_info": {**model_info, "id": f"embedder-{index}"}, + } + for index in range(2) + ], + routing_strategy=routing_strategy, + ) + + if not admitted: + with pytest.raises((litellm.RateLimitError, RouterRateLimitError)): + await router.select("embedder", {}, input=[uri]) + return + deployment: Final = await router.select("embedder", {}, input=[uri]) + assert deployment["model_info"]["id"] in ("embedder-0", "embedder-1") + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_multiple_potential_deployments(sync_mode): diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.py index 5eeb46c3518..951182051c4 100644 --- a/tests/unit/router_strategy/test_router_routing_groups.py +++ b/tests/unit/router_strategy/test_router_routing_groups.py @@ -21,6 +21,7 @@ from litellm import Router from litellm.caching.redis_cache import RedisPipelineIncrementOperation from litellm.integrations.custom_logger import CustomLogger from litellm.router_api import RoutingContext +from litellm.router_strategy.least_busy import picks_reserve from litellm.types.router import DeploymentTypedDict, FallbackAccessCheck, RoutingGroup, RoutingStrategy from litellm.utils import Rules, function_setup @@ -482,6 +483,38 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch): assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0 +def _filtered_model_pick(router: Router) -> str: + deployment: Final = router.get_available_deployment( + "filtered-model", request_kwargs=_proxy_shaped_request(model="filtered-model") + ) + return deployment["model_info"]["id"] + + +def test_a_least_busy_group_pick_stops_counting_once_its_request_finishes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "input_callback", []) + router: Final = _build_router( + routing_strategy="least-busy", + routing_groups=[{"group_name": "fast", "models": ["filtered-model"], "routing_strategy": "least-busy"}], + ) + + request: Final = _proxy_shaped_request(model="filtered-model") + with picks_reserve(True): + first: Final = router.get_available_deployment("filtered-model", request_kwargs=request)["model_info"]["id"] + kwargs: Final = { + "litellm_params": {"metadata": {"model_group": "filtered-model"}, "model_info": {"id": first}}, + "litellm_call_id": request["litellm_call_id"], + } + for callback in litellm.input_callback: + if isinstance(callback, CustomLogger): + callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs) + for callback in litellm.callbacks: + if isinstance(callback, CustomLogger): + callback.log_success_event(kwargs, None, None, None) + + assert _filtered_model_pick(router) == first + + def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch): """ Least-busy hangs its counting off litellm's global callback lists, and those lists keep one @@ -614,6 +647,7 @@ async def test_select_deployment_async_returns_none_without_selector(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -628,6 +662,7 @@ def test_select_deployment_sync_returns_none_without_selector(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -662,6 +697,7 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing(): messages=None, input=None, request_kwargs=None, + reserve=False, ) assert result is None @@ -708,6 +744,7 @@ async def test_async_dispatch_falls_back_to_sync_for_usage_based_routing_v1(): messages=[{"role": "user", "content": "hi"}], input=None, request_kwargs={}, + reserve=False, ) assert v1_spy.called, "async dispatch must route v1 strategy through sync method" @@ -1022,7 +1059,11 @@ async def test_usage_based_v2_override_stays_scoped_to_the_request_that_asked_fo await _mock_completion(router, routing_strategy="usage-based-routing-v2") -def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for_it(): +def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for_it(monkeypatch): + monkeypatch.setattr( + "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", + lambda: datetime.datetime(2026, 1, 1, 12, 0, 30, tzinfo=datetime.timezone.utc), + ) router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle", num_retries=0) messages = [{"role": "user", "content": "hi"}] diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 26dbb96c60a..a386542d61b 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -257,23 +257,22 @@ _FIRST_HOP_FAILURE_MARKER: Final = datetime(2026, 1, 2) _FINAL_HOP_FAILURE_MARKER: Final = datetime(2026, 1, 3) -class _TwoFailureFallbackRouter(Router): - def __init__(self) -> None: - super().__init__( - model_list=[ - { - "model_name": "fallback-model", - "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, - "model_info": {"id": f"fallback-dep-{index}"}, - } - for index in (1, 2) - ], - allowed_fails=0, - cooldown_time=60, - num_retries=0, - ) +def _two_failure_fallback_router() -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": f"fallback-dep-{index}"}, + } + for index in (1, 2) + ], + allowed_fails=0, + cooldown_time=60, + num_retries=0, + ) - async def async_function_with_fallbacks(self, *args: object, **kwargs: object) -> NoReturn: + async def fail_both_hops(*args: object, **kwargs: object) -> NoReturn: logging_obj: Final[object] = kwargs["litellm_logging_obj"] assert isinstance(logging_obj, Logging) assert logging_obj.model_call_details.get("has_logged_async_failure", False) is False @@ -287,6 +286,9 @@ class _TwoFailureFallbackRouter(Router): logging_obj.model_call_details["current_attempt_api_call_start_time"] = _FINAL_HOP_FAILURE_MARKER raise _RateLimitedHop("fallback-dep-2") + router.async_function_with_fallbacks = fail_both_hops + return router + def _two_deployment_router() -> Router: return Router( @@ -302,6 +304,63 @@ def _two_deployment_router() -> Router: ) +def _entra_fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": "fallback-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + "client_secret": "configured-secret", + }, + "model_info": {"id": f"fallback-dep-{index}"}, + } + for index in (1, 2) + ], + num_retries=0, + ) + + +def _raised_from_an_entra_rejection(failed_deployment_id: str) -> litellm.AuthenticationError: + from litellm.llms.azure.common_utils import EntraRejectionError + + reason: Final = "Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided." + try: + raise litellm.AuthenticationError(message=reason, llm_provider="azure", model="gpt-4.1") from ( + EntraRejectionError(status_code=401, message=reason) + ) + except litellm.AuthenticationError as raised: + raised.failed_deployment_id = failed_deployment_id + return raised + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_params", "cooled_down"), + [ + pytest.param({"client_secret": "not-the-credential"}, [], id="caller-secret"), + pytest.param({"client_secret": "configured-secret"}, ["fallback-dep-1"], id="deployment-secret"), + ], +) +async def test_fallback_hop_cools_down_only_a_deployment_entra_rejected_its_own_credential( + call_params: Mapping[str, object], cooled_down: list[str] +) -> None: + router: Final = _entra_fallback_router() + + _trigger_cooldown_for_failed_deployment( + ctx=RoutingContext(router), + kwargs={}, + exception=_raised_from_an_entra_rejection("fallback-dep-1"), + model_call_details={"litellm_params": call_params}, + ) + + assert await async_get_cooldown_deployments(ctx=RoutingContext(router), parent_otel_span=None) == cooled_down + + @pytest.mark.asyncio @pytest.mark.parametrize("failure_already_logged", [False, True]) async def test_run_async_fallback_cools_down_a_failed_hop_once(failure_already_logged: bool) -> None: @@ -340,8 +399,9 @@ async def test_run_async_fallback_cools_down_a_failed_hop_once(failure_already_l @pytest.mark.asyncio +@pytest.mark.usefixtures("python_router_core") # the Rust core walks fallbacks in Rust, so it has no such member async def test_run_async_fallback_cools_down_the_final_deployment_after_retry_failure_logging() -> None: - router: Final = _TwoFailureFallbackRouter() + router: Final = _two_failure_fallback_router() logging_obj: Final = _request_logging_obj() assert logging_obj.model_call_details.get("has_logged_async_failure", False) is False diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index ce8f04b8c54..1500d7433d2 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Mapping from typing import Final, cast import pytest @@ -377,6 +378,32 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content(): assert isinstance(result[2]["content"], list) +def test_prompt_cache_breakpoint_marks_the_same_cacheable_prefix_as_cache_control(): + def _messages(marker: Mapping[str, Mapping[str, str]]) -> list[AllMessageValues]: + return cast( # cast-ok: the text part type does not declare the prompt_cache_breakpoint key + list[AllMessageValues], + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Here is the full text of a complex legal agreement" * 400, **marker} + ], + }, + {"role": "user", "content": "what are the key terms?"}, + ], + ) + + marked_key: Final = PromptCachingCache.get_prompt_caching_cache_key( + _messages({"prompt_cache_breakpoint": {"mode": "explicit"}}), None + ) + + assert marked_key is not None + assert marked_key == PromptCachingCache.get_prompt_caching_cache_key( + _messages({"cache_control": {"type": "ephemeral"}}), None + ) + assert PromptCachingCache.get_prompt_caching_cache_key(_messages({}), None) is None + + @pytest.mark.asyncio async def test_same_cacheable_prefix_routes_to_same_deployment() -> None: from litellm import Router diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index abf0e055e73..9d8b09dd1c4 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -138,3 +138,110 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" assert additional["request-id"] == additional["llm_provider-request-id"] + + +_ADAPTIVE_OPUS: Final = "anthropic/claude-opus-4-7" + + +@pytest.mark.parametrize( + ("model", "expected"), + [("claude-sonnet-4-5", "anthropic"), ("gpt-4o", "openai"), ("not-a-known-model", None)], +) +def test_inferred_provider_is_the_one_the_python_handler_resolves(model: str, expected: str | None) -> None: + assert messages_host.inferred_provider(model, None) == expected + + +@pytest.mark.parametrize( + ("request_effort", "levels", "drop_params", "expected"), + [ + ("max", ["low", "high"], False, "high"), + ("high", ["low"], True, "low"), + ], +) +def test_body_params_round_effort_to_the_levels_the_deployment_declares( + request_effort: str, levels: list[str], drop_params: bool, expected: str +) -> None: + request: Final[dict[str, object]] = { + "model": _ADAPTIVE_OPUS, + "messages": [], + "max_tokens": 8, + "output_config": {"effort": request_effort}, + "drop_params": drop_params, + "model_info": {"id": "deployment-1", "reasoning_effort_levels": levels}, + } + + assert messages_host.body_params(request)["output_config"] == {"effort": expected} + + +def test_body_params_keep_the_requested_effort_without_deployment_levels() -> None: + request: Final[dict[str, object]] = { + "model": _ADAPTIVE_OPUS, + "messages": [], + "max_tokens": 8, + "output_config": {"effort": "max"}, + } + + assert messages_host.body_params(request)["output_config"] == {"effort": "max"} + + +def _marked(phase: str | None = None, *args: object) -> ValueError: + error: Final = ValueError(*args) + if phase is not None: + error.messages_timeout = phase # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + return error + + +def test_a_stalled_stream_body_reaches_the_caller_as_the_transport_timeout() -> None: + import httpx + + mapped: Final = messages_host.map_failure(_marked("Body"), {"model": _ADAPTIVE_OPUS}, "anthropic") + + assert isinstance(mapped, httpx.ReadTimeout) + assert str(mapped) == "Timeout on reading data from socket" + + +def test_a_native_read_timeout_maps_to_the_public_timeout_with_python_text() -> None: + mapped: Final = messages_host.map_failure(_marked("Read"), {"model": _ADAPTIVE_OPUS, "timeout": 1.0}, "anthropic") + + assert isinstance(mapped, litellm.Timeout) + assert "Read timed out. Timeout passed=1.0" in str(mapped) + + +def test_an_upstream_status_maps_through_the_provider_error_class() -> None: + upstream: Final = ValueError(429, "slow down") + upstream.headers = [("retry-after", "3")] # pyright: ignore[reportAttributeAccessIssue] # set by the native host + + mapped: Final = messages_host.map_failure(upstream, {"model": _ADAPTIVE_OPUS}, "anthropic") + + assert isinstance(mapped, litellm.RateLimitError) + assert "slow down" in str(mapped) + assert mapped.__context__ is upstream + + +def test_a_late_rejection_tells_the_logger_the_resolved_call_first() -> None: + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + + logger: Final = Logging( + model=_ADAPTIVE_OPUS, + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="late", + function_id="late", + ) + rejected: Final = ValueError("budget_tokens must be a number") + rejected.messages_late_rejection = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + rejected.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = messages_host.map_failure( + rejected, + {"model": _ADAPTIVE_OPUS, "messages": [], "max_tokens": 8, "litellm_logging_obj": logger}, + "anthropic", + ) + + assert isinstance(mapped, litellm.BadRequestError) + assert logger.model_call_details["custom_llm_provider"] == "anthropic" + assert logger.model_call_details["optional_params"] == {"max_tokens": 8} diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index a40f8a77105..1ba0188e76d 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -5,6 +5,7 @@ from pathlib import Path from types import MappingProxyType from typing import Final +import httpx import pytest from pydantic import TypeAdapter @@ -244,3 +245,56 @@ def test_the_rust_contract_matches_the_shim_signatures() -> None: contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) assert contract == {name: list(inspect.signature(getattr(legacy, name)).parameters) for name in contract} + + +def _messages_logger() -> Logging: + return Logging( + model="claude-sonnet-4-5", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="messages", + function_id="messages", + ) + + +def test_native_provider_response_is_recorded_for_logging_replay() -> None: + logger: Final = _messages_logger() + + legacy.record_response(logger, 200, (("request-id", "req_native"),), '{"id": "msg_native"}') + + recorded: Final[object] = logger.model_call_details["httpx_response"] + assert isinstance(recorded, httpx.Response) + assert recorded.status_code == 200 + assert recorded.content == b'{"id": "msg_native"}' + assert logger.model_call_details["response_headers"]["request-id"] == "req_native" + + +def test_native_provider_exchange_window_is_recorded_like_the_timed_post() -> None: + logger: Final = _messages_logger() + logger.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed"}} + + legacy.record_api_window(logger, 1_700_000_000.0, 1_700_000_000.25) + + assert logger.model_call_details["llm_api_duration_ms"] == 250.0 + metadata: Final = logger.model_call_details["litellm_params"]["litellm_metadata"] + assert metadata["llm_api_timing_windows"] == ((1_700_000_000.0, 1_700_000_000.25),) + + +def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: + logger: Final = _messages_logger() + head: Final[dict[str, object]] = {"additional_headers": {"request-id": "req_native"}} + started: Final = datetime.datetime.now() + + legacy.stamp_stream_head( + head, + logger, + {"model": "anthropic/claude-sonnet-4-5", "litellm_call_id": "messages"}, + started, + started + datetime.timedelta(milliseconds=5), + ) + + assert head["additional_headers"] == {"request-id": "req_native", "llm_provider-request-id": "req_native"} + assert head["litellm_call_id"] == "messages" + assert "_response_ms" in head diff --git a/tests/unit/rust_bridge/messages/test_secrets.py b/tests/unit/rust_bridge/messages/test_secrets.py index 64559d63193..5ee89fa7794 100644 --- a/tests/unit/rust_bridge/messages/test_secrets.py +++ b/tests/unit/rust_bridge/messages/test_secrets.py @@ -11,7 +11,7 @@ import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.llms.anthropic.messages.native.handler import anthropic_messages from litellm.rust_bridge.host import settings -from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES +from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES from litellm.rust_bridge.public_call import NativeCall from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE @@ -72,21 +72,13 @@ async def _python_messages() -> object: return await anthropic_messages(**_public_kwargs()) -async def _rust_messages() -> object: - route: Final = NATIVE_MESSAGES.load() - assert route is not None - return route(_native_request()) - - async def _rust_amessages() -> object: route: Final = NATIVE_AMESSAGES.load() assert route is not None return await route(_native_request()) -@pytest.fixture( - params=(_python_messages, _rust_messages, _rust_amessages), ids=("python-async", "rust-sync", "rust-async") -) +@pytest.fixture(params=(_python_messages, _rust_amessages), ids=("python-async", "rust-async")) def messages(request: pytest.FixtureRequest) -> Messages: return cast(Messages, request.param) diff --git a/tests/unit/secret_managers/test_aws_secret_manager.py b/tests/unit/secret_managers/test_aws_secret_manager.py index d9922b43ec8..e16da7a0844 100644 --- a/tests/unit/secret_managers/test_aws_secret_manager.py +++ b/tests/unit/secret_managers/test_aws_secret_manager.py @@ -1,4 +1,6 @@ import base64 +import re +from typing import Final import pytest from botocore.stub import Stubber @@ -34,3 +36,15 @@ def test_decrypt_value_turns_only_boolean_literals_into_bools(monkeypatch, plain assert decrypted == expected assert type(decrypted) is type(expected) + + +def test_kms_v2_without_a_license_tells_the_user_to_set_one(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + monkeypatch.delenv("LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE", raising=False) + + expected_message: Final = ( + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + AWSKeyManagementService_V2() diff --git a/tests/unit/test_cerebras_llama_model_metadata.py b/tests/unit/test_cerebras_llama_model_metadata.py index c5ec096e553..5b51e9e2f24 100644 --- a/tests/unit/test_cerebras_llama_model_metadata.py +++ b/tests/unit/test_cerebras_llama_model_metadata.py @@ -52,7 +52,6 @@ CEREBRAS_MODELS_EXPECTED = { PAIR_PARITY = [ ("cerebras/llama-3.1-8b", "cerebras/llama3.1-8b"), - ("cerebras/llama-3.1-70b", "cerebras/llama3.1-70b"), ] diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index d7122541416..c6ea0cd936e 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -1,7 +1,10 @@ """Tests for scripts/check_openapi_docs.py, the scan behind scripts/openapi_docs_gate.py.""" +from collections.abc import Mapping from typing import Final +import pytest + import check_openapi_docs as checker _BODY: Final = {"content": {"application/json": {"schema": {"type": "object"}}}} @@ -9,8 +12,26 @@ _RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref": _DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES} -def test_a_fully_documented_post_has_no_findings() -> None: - assert checker.violations({"paths": {"/v1/thing": {"post": _DOCUMENTED}}}) == () +def _post_with_media_types(request_media_type: str, response_media_type: str) -> Mapping[str, object]: + return { + **_DOCUMENTED, + "requestBody": {"content": {request_media_type: {"schema": {"type": "string"}}}}, + "responses": {"200": {"content": {response_media_type: {"schema": {"type": "string"}}}}}, + } + + +@pytest.mark.parametrize("request_type, response_type", [ + ("application/json", "application/json"), ("application/sdp", "application/sdp"), + ("multipart/form-data", "application/json"), +]) +def test_a_fully_documented_post_has_no_findings(request_type: str, response_type: str) -> None: + operation: Final = _post_with_media_types(request_type, response_type) + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () + + +def test_a_schema_under_another_media_type_still_counts_as_undocumented() -> None: + operation: Final = _post_with_media_types("application/octet-stream", "audio/wav") + assert {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": operation}}})} == {"OA001", "OA003"} def test_each_missing_piece_is_its_own_rule() -> None: @@ -19,8 +40,9 @@ def test_each_missing_piece_is_its_own_rule() -> None: assert codes == {"OA001", "OA002", "OA003"} -def test_only_the_missing_piece_is_reported() -> None: - no_body: Final = {**_DOCUMENTED, "requestBody": {}} +@pytest.mark.parametrize("body", [{}, {"content": {"multipart/form-data": {}}}]) +def test_only_the_missing_piece_is_reported(body: Mapping[str, object]) -> None: + no_body: Final = {**_DOCUMENTED, "requestBody": body} blank_description: Final = {**_DOCUMENTED, "description": " "} untyped_200: Final = {**_DOCUMENTED, "responses": {"200": {"content": {"application/json": {"schema": {}}}}}} spec: Final = { diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index aa759f2c4d9..49d51e99790 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1451,6 +1451,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 +@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None: + from litellm.cost_calculator import _get_transcription_usage_duration + + response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds}) + + assert _get_transcription_usage_duration(response) is None + + def test_realtime_transcription_prices_each_clip_at_the_transcription_model_set_when_it_ran(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -4522,6 +4531,103 @@ def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities(): assert prompt_cost == pytest.approx(100 * 1e-7) +def _cached_audio_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=1000, audio_tokens=1000, cached_tokens_details=CachedTokensDetails(audio_tokens=1000) + ), + ) + + +def _cached_text_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=1000), + ) + + +@pytest.mark.parametrize( + ("audio_batch_cache_price", "usage", "expected_rate"), + [ + pytest.param(None, _cached_audio_batch_usage(), 1e-7, id="cached-audio-bills-the-standard-audio-cache-price"), + pytest.param(None, _cached_text_batch_usage(), 3e-8, id="cached-text-bills-the-standard-cache-price"), + pytest.param(8e-8, _cached_audio_batch_usage(), 8e-8, id="an-explicit-audio-batch-cache-price-still-wins"), + ], +) +def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_price_when_the_row_has_no_batch_cache_price( + audio_batch_cache_price: float | None, usage: Usage, expected_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 3e-7, + "input_cost_per_audio_token": 1e-6, + "cache_read_input_token_cost": 3e-8, + "cache_read_input_audio_token_cost": 1e-7, + "input_cost_per_token_batches": 1.5e-7, + "input_cost_per_audio_token_batches": 5e-7, + "cache_read_input_audio_token_cost_batches": audio_batch_cache_price, + } + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gemini-2.5-flash", custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * expected_rate) + + +def test_batch_cost_calculator_bills_long_context_cached_tokens_at_the_standard_long_context_cache_price() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1.25e-6, + "input_cost_per_token_above_200k_tokens": 2.5e-6, + "cache_read_input_token_cost": 1.25e-7, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-7, + "input_cost_per_token_batches": 6.25e-7, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-6, + } + usage: Final = Usage( + prompt_tokens=300_000, + completion_tokens=0, + total_tokens=300_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300_000), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gemini-2.5-pro", custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(300_000 * 2.5e-7) + + +def _batch_and_standard_cached_audio_cost(model: str, provider: str) -> tuple[str, float, float]: + batch_prompt_cost, _ = batch_cost_calculator(_cached_audio_batch_usage(), model, provider) + standard_prompt_cost, _ = litellm.cost_per_token( + model=model, custom_llm_provider=provider, usage_object=_cached_audio_batch_usage() + ) + return model, batch_prompt_cost, standard_prompt_cost + + +def test_batch_cached_audio_never_costs_more_than_standard_on_any_row_with_a_batch_input_price( + _local_model_cost_map: None, +) -> None: + rows: Final = tuple( + (name, entry["litellm_provider"]) + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("input_cost_per_token_batches") is not None + and entry.get("cache_read_input_audio_token_cost") is not None + ) + assert rows, "no cost map row prices both Batch input and Standard cached audio" + + costs: Final = tuple(_batch_and_standard_cached_audio_cost(name, provider) for name, provider in rows) + overpriced: Final = tuple(cost for cost in costs if cost[1] > cost[2]) + + assert overpriced == () + + @pytest.mark.parametrize( ("image_batch_rate", "image_tokens", "expected_completion_cost"), ( @@ -6245,7 +6351,7 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_flat_batch_cached_rate assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1e-7) -def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_without_a_cached_batch_rate( +def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_rate_without_a_cached_batch_rate( _tiered_batch_models: None, ) -> None: from litellm.cost_calculator import batch_cost_calculator @@ -6254,7 +6360,7 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_witho usage=_cached_usage(300_048, 300_045, 11), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" ) - assert prompt_cost == pytest.approx(300_048 * 3e-6) + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 2e-7) def _cache_write_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: @@ -7474,6 +7580,39 @@ def test_transcription_cost_falls_back_to_provider_billed_seconds( assert cost == pytest.approx(0.01 * expected_seconds) +@pytest.mark.parametrize( + ("provider", "measured_seconds", "expected_seconds"), + [("openai", 1.0, 1.0), ("azure", 1.0, 1.0), ("azure", None, 7.0)], + ids=["openai_measured_upload_wins", "azure_measured_upload_wins", "unmeasured_upload_bills_provider_seconds"], +) +def test_transcription_priced_by_provider_duration_still_bills_the_measured_upload( + monkeypatch: pytest.MonkeyPatch, provider: str, measured_seconds: float | None, expected_seconds: float +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "gpt-fake-transcribe", + { + "input_cost_per_second": 0.01, + "litellm_provider": provider, + "mode": "audio_transcription", + "provider_specific_entry": {"transcription_usage_duration": 1}, + }, + ) + measured: Final = {} if measured_seconds is None else {"audio_transcription_duration": measured_seconds} + transcription: Final = convert_to_model_response_object( + response_object={"text": "What is the weather in Paris?", "usage": {"type": "duration", "seconds": 7}}, + model_response_object=TranscriptionResponse(), + hidden_params={"model": "gpt-fake-transcribe", "custom_llm_provider": provider, **measured}, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, model=f"{provider}/gpt-fake-transcribe", call_type="atranscription" + ) + + assert cost == pytest.approx(0.01 * expected_seconds) + + def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem( litellm.model_cost, @@ -7498,6 +7637,77 @@ def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypat assert cost == pytest.approx(0.000075 * 2) +@pytest.mark.parametrize( + "deployment_id,deployment_rates,expected_cost", + [ + ("per-second-only", {"input_cost_per_second": 0.0001}, 30 * 0.0001), + ("zero-token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 0.0}, 0.0), + ("token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 1e-6}, 500 * 1e-6), + ], +) +def test_transcription_deployment_bills_seconds_only_without_a_declared_token_rate( + monkeypatch: pytest.MonkeyPatch, + deployment_id: str, + deployment_rates: Mapping[str, float], + expected_cost: float, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {**deployment_rates, "litellm_provider": "openai", "mode": "audio_transcription"}, + ) + transcription: Final = convert_to_model_response_object( + response_object={ + "text": "What is the weather in Paris?", + "usage": { + "type": "tokens", + "input_tokens": 500, + "output_tokens": 100, + "total_tokens": 600, + "input_token_details": {"audio_tokens": 500, "text_tokens": 0}, + }, + }, + model_response_object=TranscriptionResponse(), + hidden_params={ + "model": "gpt-4o-transcribe", + "custom_llm_provider": "openai", + "audio_transcription_duration": 30.0, + }, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, + model="openai/gpt-4o-transcribe", + call_type="atranscription", + custom_pricing=True, + router_model_id=deployment_id, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"]) +def test_azure_transcription_without_usage_bills_only_tokens_while_openai_bills_the_measured_audio( + _local_model_cost_map: None, model: str +) -> None: + def cost_of_ten_seconds_without_usage(provider: str) -> float: + transcript: Final = TranscriptionResponse(text="Good") + transcript.set_audio_transcription_duration(10.0) + return completion_cost( + completion_response=transcript, + model=f"{provider}/{model}", + custom_llm_provider=provider, + call_type="atranscription", + ) + + assert cost_of_ten_seconds_without_usage("openai") > 0 + assert cost_of_ten_seconds_without_usage("azure") == 0.0, ( + "Azure lists only token prices for this model, https://azure.microsoft.com/en-us/pricing/details/azure-openai/ " + "(per the 2026-10-28 review)" + ) + + def test_gpt_image_2_azure_cost_tracking(): azure_image_generation_response: Final = { "created": 1758585600, @@ -8205,7 +8415,7 @@ def test_cost_openai_prompt_caching(): "model", [ "cohere/rerank-english-v3.0", - "azure_ai/cohere-rerank-v3-english", + "azure_ai/cohere-rerank-v4.0-pro", ], ) def test_completion_cost_azure_ai_rerank(model): @@ -9258,7 +9468,7 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): @pytest.mark.parametrize( "response_model, custom_llm_provider", [ - ("azure_ai/Meta-Llama-3.1-70B-Instruct", "azure_ai"), + ("azure_ai/Llama-3.3-70B-Instruct", "azure_ai"), ("anthropic.claude-3-5-sonnet-20240620-v1:0", "bedrock"), ], ) @@ -9318,7 +9528,7 @@ def test_select_model_name_for_cost_calc(): "completion_response": ModelResponse( id="127f24aed4984b4c9a4c5e32ad3752f3", created=1734406048, - model="azure_ai/mistral-large", + model="azure_ai/mistral-large-latest", object="chat.completion", system_fingerprint=None, choices=[ @@ -9347,7 +9557,7 @@ def test_select_model_name_for_cost_calc(): } return_model = select_model_name_for_cost_calc(**args) - assert return_model == "azure_ai/mistral-large" + assert return_model == "azure_ai/mistral-large-latest" def test_cost_calculator_with_base_model(): @@ -10441,6 +10651,301 @@ def test_responses_ws_logging_object_carries_the_completed_responses_id_tier_and assert ResponsesWebSocketTokenUsageProcessor.create_logging_object([{"type": "error", "error": {}}]).id is None +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 1.5}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["input_cost_per_second"]) + + +def test_realtime_translation_bills_session_audio_once_on_openai_and_azure(_local_model_cost_map) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + one_minute_session: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}} + ] + openai_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="openai", litellm_model_name="gpt-realtime-translate" + ) + azure_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="azure", litellm_model_name="gpt-realtime-translate" + ) + + assert openai_cost == pytest.approx(60 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"]) + assert openai_cost == pytest.approx(azure_cost) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "translation-deployment-priced-both-directions" + litellm.register_model( + model_cost={ + model: { + "litellm_provider": "openai", + "mode": "realtime", + "input_cost_per_second": 0.001, + "output_cost_per_second": 0.002, + } + } + ) + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) + + +def _translation_session_cost( + deployment_id: str, + session_model: str | None = "gpt-realtime-translate", + model: str = "gpt-realtime-translate", + base_model: str | None = None, +) -> float: + session_created: Final[OpenAIRealtimeStreamList] = ( + [{"type": "session.created", "session": {"model": session_model}}] if session_model is not None else [] + ) + one_minute_session: Final = LiteLLMRealtimeStreamLoggingObject( + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + results=[ + *session_created, + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}}, + ], + ) + return completion_cost( + completion_response=one_minute_session, + model=model, + call_type=CallTypes.arealtime.value, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + base_model=base_model, + ) + + +@pytest.mark.parametrize("session_model", ["gpt-realtime-translate", None]) +@pytest.mark.parametrize( + "deployment_model,base_model", + [("gpt-realtime-translate", None), ("my-translator", "gpt-realtime-translate")], +) +@pytest.mark.parametrize( + "deployment_id,deployment_rates,bills_public_input_rate", + [ + ("translate-output-priced", {"output_cost_per_second": 0.001}, True), + ("translate-free-input", {"input_cost_per_second": 0.0, "output_cost_per_second": 0.001}, False), + ], +) +def test_realtime_translation_deployment_per_second_rates_fall_back_to_public_rates_per_key( + _local_model_cost_map: None, + deployment_id: str, + deployment_rates: Mapping[str, float], + bills_public_input_rate: bool, + deployment_model: str, + base_model: str | None, + session_model: str | None, +) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": f"openai/{deployment_model}", + "api_key": "sk-test", + **deployment_rates, + }, + "model_info": {"id": deployment_id, **({"base_model": base_model} if base_model else {})}, + } + ] + ) + public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + + assert public_input_rate > 0 + assert _translation_session_cost( + deployment_id, session_model=session_model, model=deployment_model, base_model=base_model + ) == pytest.approx(60 * 0.001 + (60 * public_input_rate if bills_public_input_rate else 0.0)) + + +def test_realtime_translation_custom_deployment_with_one_rate_bills_the_session_model_for_the_other( + _local_model_cost_map: None, +) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/my-translator", + "api_key": "sk-test", + "output_cost_per_second": 0.001, + }, + "model_info": {"id": "custom-translate-output-priced"}, + } + ] + ) + public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + + assert public_input_rate > 0 + assert _translation_session_cost( + "custom-translate-output-priced", model="my-translator" + ) == pytest.approx(60 * 0.001 + 60 * public_input_rate) + + +def test_realtime_translation_rate_ignores_a_session_model_the_deployment_does_not_run( + _local_model_cost_map: None, +) -> None: + litellm.register_model( + model_cost={ + "translation-session-echo": { + "litellm_provider": "openai", + "mode": "realtime", + "input_cost_per_second": 0.5, + } + } + ) + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "output_cost_per_second": 0.001, + }, + "model_info": {"id": "translate-output-priced"}, + } + ] + ) + + assert _translation_session_cost( + "translate-output-priced", session_model="translation-session-echo" + ) == pytest.approx(_translation_session_cost("translate-output-priced", session_model=None)) + + +def test_realtime_translation_deployment_without_per_second_rates_bills_public_rates(_local_model_cost_map) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "model_info": {"id": "translate-token-priced"}, + } + ] + ) + public_entry: Final = litellm.model_cost["gpt-realtime-translate"] + public_cost: Final = 60 * (public_entry.get("input_cost_per_second") or 0.0) + 60 * ( + public_entry.get("output_cost_per_second") or 0.0 + ) + + assert public_cost > 0 + assert _translation_session_cost("translate-token-priced") == pytest.approx(public_cost) + + +def test_realtime_translation_spend_usage_carries_the_billed_input_seconds(_local_model_cost_map) -> None: + logging_obj: Final = Logging( + model="gpt-realtime-translate", + messages=[], + stream=False, + call_type=CallTypes.arealtime.value, + start_time=datetime.datetime.now(), + litellm_call_id="translation-spend-usage", + function_id="translation-spend-usage", + ) + logging_obj.update_environment_variables( + model="gpt-realtime-translate", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 30.0, "output_seconds": 42.5}} + ] + + logging_obj.success_handler(result=events, start_time=datetime.datetime.now(), end_time=datetime.datetime.now()) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["metadata"]["usage_object"]["seconds"] == pytest.approx(30.0) + assert payload["response_cost"] == pytest.approx( + 30.0 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + ) + + @pytest.mark.parametrize("profile_prefix", ("us", "global")) @pytest.mark.parametrize( "model_name", @@ -11140,3 +11645,68 @@ def test_upstream_cost_header_total_on_an_unmapped_alias_is_split_by_the_served_ input_share: Final = (10 * 0.001) / (10 * 0.001 + 20 * 0.002) assert breakdown["input_cost"] == pytest.approx(0.25 * input_share) assert breakdown["output_cost"] == pytest.approx(0.25 * (1 - input_share)) + + +_PER_CALL_RATES: Final = MappingProxyType({"input_cost_per_token": 2e-06, "output_cost_per_token": 4e-06}) + + +def _per_call_priced_reply(model: str, usage_extras: Mapping[str, str]) -> ModelResponse: + usage: Final = Usage(prompt_tokens=1000, completion_tokens=1000, total_tokens=2000) + for field, value in usage_extras.items(): + setattr(usage, field, value) + return ModelResponse(model=model, choices=[Choices(message=Message(content="hi"))], usage=usage) + + +@pytest.mark.parametrize( + ("row", "call_kwargs", "usage_extras", "multiplier"), + [ + pytest.param( + {"litellm_provider": "openai", "mode": "chat", "regional_processing_uplift_multiplier_eu": 1.25}, + { + "model": "openai/lit-eu-residency-test", + "api_key": "fake-key", + "api_base": "https://eu.api.openai.com/v1", + }, + {}, + 1.25, + id="openai-data-residency", + ), + pytest.param( + {"litellm_provider": "vertex_ai", "mode": "chat", "regional_endpoint_uplift_multiplier": 1.5}, + { + "model": "vertex_ai/lit-regional-endpoint-test", + "vertex_project": "lit-project", + "vertex_location": "us-east5", + "vertex_credentials": "{}", + }, + {}, + 1.5, + id="vertex-regional-endpoint", + ), + pytest.param( + {"litellm_provider": "anthropic", "mode": "chat", "provider_specific_entry": {"fast": 2.0}}, + {"model": "anthropic/lit-fast-mode-test", "api_key": "fake-key"}, + {"speed": "fast"}, + 2.0, + id="anthropic-fast-mode", + ), + ], +) +def test_per_call_custom_prices_keep_the_regional_and_speed_multipliers( + monkeypatch: pytest.MonkeyPatch, + row: Mapping[str, object], + call_kwargs: Mapping[str, str], + usage_extras: Mapping[str, str], + multiplier: float, +) -> None: + monkeypatch.setitem(litellm.model_cost, call_kwargs["model"], {**row, **_PER_CALL_RATES}) + _invalidate_model_cost_lowercase_map() + + response: Final = litellm.completion( + messages=[{"role": "user", "content": "hello"}], + mock_response=_per_call_priced_reply(call_kwargs["model"].split("/", 1)[1], usage_extras), + **call_kwargs, + **_PER_CALL_RATES, + ) + + assert response._hidden_params["response_cost"] == pytest.approx((1000 * 2e-06 + 1000 * 4e-06) * multiplier) diff --git a/tests/unit/test_count_tokens_public_api.py b/tests/unit/test_count_tokens_public_api.py index 4eb8bea3427..581682be474 100644 --- a/tests/unit/test_count_tokens_public_api.py +++ b/tests/unit/test_count_tokens_public_api.py @@ -59,7 +59,7 @@ def test_acount_tokens_fallback_to_local(): """Test that unsupported providers fall back to local tiktoken counting.""" result = asyncio.run( litellm.acount_tokens( - model="together_ai/meta-llama/Llama-3-8b-chat-hf", + model="together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo", messages=[{"role": "user", "content": "Hello"}], ) ) diff --git a/tests/unit/test_get_blog_posts.py b/tests/unit/test_get_blog_posts.py index d1de75a4500..71099dc2326 100644 --- a/tests/unit/test_get_blog_posts.py +++ b/tests/unit/test_get_blog_posts.py @@ -50,11 +50,10 @@ def reset_blog_posts_cache(): GetBlogPosts._last_fetch_time = 0.0 -def test_load_local_blog_posts_returns_posts_without_tags(): +def test_load_local_blog_posts_returns_posts(): posts = GetBlogPosts.load_local_blog_posts() assert len(posts) > 0 assert all(isinstance(post, BlogPost) for post in posts) - assert posts[0].tags == () def test_parse_rss_to_posts(): diff --git a/tests/unit/test_gpt_realtime_mode.py b/tests/unit/test_gpt_realtime_mode.py index 0ea85df84cb..c6f6d68dcc9 100644 --- a/tests/unit/test_gpt_realtime_mode.py +++ b/tests/unit/test_gpt_realtime_mode.py @@ -24,13 +24,10 @@ REALTIME_ONLY_GPT_MODELS = ( REALTIME_ONLY_GPT_MODELS_WITHOUT_ENDPOINTS = ( "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17", - "azure/eu/gpt-4o-realtime-preview-2024-10-01", "azure/eu/gpt-4o-realtime-preview-2024-12-17", "azure/gpt-4o-mini-realtime-preview-2024-12-17", - "azure/gpt-4o-realtime-preview-2024-10-01", "azure/gpt-4o-realtime-preview-2024-12-17", "azure/us/gpt-4o-mini-realtime-preview-2024-12-17", - "azure/us/gpt-4o-realtime-preview-2024-10-01", "azure/us/gpt-4o-realtime-preview-2024-12-17", "gpt-4o-mini-realtime-preview", "gpt-4o-mini-realtime-preview-2024-12-17", diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 3336cce9aaf..ceb9ef38887 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -25,7 +25,7 @@ _REPO: Final = Path(__file__).resolve().parents[2] _REDIS_PRODUCER_ROOTS: Final = ("litellm", "enterprise") # The cache implementations and facades: they emit the service events, their callers declare the family. -_CACHE_LAYER_DIRS: Final = ("litellm/caching", "litellm/_v2/cache") +_CACHE_LAYER_DIRS: Final = ("litellm/caching",) # Helpers that act on a cache handed in by the declaring caller, or forward to the response-cache facade. _CACHE_PARAMETER_HELPERS: Final = frozenset( { diff --git a/tests/unit/test_lint_workflow_diff_gates.py b/tests/unit/test_lint_workflow_diff_gates.py index 816ef6ce707..5e6ae9d285a 100644 --- a/tests/unit/test_lint_workflow_diff_gates.py +++ b/tests/unit/test_lint_workflow_diff_gates.py @@ -383,10 +383,20 @@ def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]: return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD") +def _schema_base_commit_and_its_type_in_the_checkout(run: StepRun) -> tuple[str, str]: + argv: Final = _argv_recorded(run.runner, "uv") + base_commit: Final = argv[argv.index("--base-commit") + 1] + return base_commit, _git(run.runner.checkout, "cat-file", "-t", base_commit).strip() + + def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str: return upstream.merge_base_by_ref[ref] +def _merge_base_as_a_commit(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]: + return upstream.merge_base_by_ref[ref], "commit" + + def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]: merge_base: Final = upstream.merge_base_by_ref[ref] changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head) @@ -402,6 +412,7 @@ CONSUMERS: Final = MappingProxyType( "test-linting.yml:python": Consumer(_gate_base_exported, _merge_base), "test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base), "test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base), + "test-unit.yml:unit": Consumer(_schema_base_commit_and_its_type_in_the_checkout, _merge_base_as_a_commit), "spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), "cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), } diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 95c5958c748..99c6f6c402c 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -6184,6 +6184,79 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR assert response.text == "hello" +def test_transcription_keeps_the_prompt_passed_by_position(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + litellm.transcription( + "groq/whisper-large-v3", GROQ_WAV_FILE, "en", "Four score", base_url=GROQ_INTERNAL_BASE, api_key="fake-key" + ) + + sent: Final = route.calls.last.request.content + assert b'name="language"\r\n\r\nen\r\n' in sent + assert b'name="prompt"\r\n\r\nFour score\r\n' in sent + + +@pytest.mark.parametrize( + "provider_body,expected_upload_measurements", + [({"text": "hello", "duration": 2.5}, 0), ({"text": "hello"}, 1)], + ids=["provider_reports_duration", "provider_omits_duration"], +) +def test_transcription_measures_the_upload_only_when_the_provider_omits_the_duration( + respx_mock: respx.MockRouter, provider_body: Mapping[str, object], expected_upload_measurements: int +): + respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(200, json=provider_body) + ) + + with patch.object( + litellm_main, "calculate_request_duration", wraps=litellm_main.calculate_request_duration + ) as measure_upload: + response: Final = litellm.transcription( + model="groq/whisper-large-v3", file=GROQ_WAV_FILE, base_url=GROQ_INTERNAL_BASE, api_key="fake-key" + ) + + assert response.text == "hello" + assert measure_upload.call_count == expected_upload_measurements + + +@pytest.mark.parametrize("model_name", ("whisper-1", "gpt-4o-transcribe", "gpt-transcribe")) +def test_openai_compatible_transcription_with_an_openai_model_name_reaches_the_server( + respx_mock: respx.MockRouter, model_name: str +): + route: Final = respx_mock.post("https://vllm.internal/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "served by vllm"}) + ) + + response: Final = litellm.transcription( + model=f"hosted_vllm/{model_name}", + file=GROQ_WAV_FILE, + api_base="https://vllm.internal/v1", + api_key="fake-key", + response_format="verbose_json", + ) + + assert route.called + assert response.text == "served by vllm" + + +def test_openai_gpt_transcribe_still_rejects_non_json_response_formats(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "should not be sent"}) + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): + litellm.transcription( + model="openai/gpt-transcribe", + file=GROQ_WAV_FILE, + api_key="fake-key", + response_format="srt", + ) + + assert not route.called + + def test_groq_speech_honors_base_url_alias(respx_mock: respx.MockRouter): route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/speech").mock( return_value=httpx.Response(200, content=b"mp3-bytes") @@ -7630,6 +7703,7 @@ async def test_bedrock_converse_parses_scripted_response( assert json.loads(route.calls[0].request.content) == { "messages": [{"role": "user", "content": [{"text": "hello"}]}], "inferenceConfig": {}, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], } assert response.choices[0].message.content == "Hello from Bedrock." assert response.usage.prompt_tokens == 8 @@ -11474,3 +11548,28 @@ async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_to assert error is not None and "Too many strict tools" in str(error) assert len(sent) == 1 + + +def test_supplied_openai_client_base_url_decides_data_residency(monkeypatch: pytest.MonkeyPatch) -> None: + """A caller-supplied OpenAI client sends the request to its own base_url, so billing must use that + host and not an env-configured FedRAMP or regional base.""" + import openai + + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("OPENAI_BASE_URL", "https://gov.api.openai.com/v1") + captured = {} + real_init = litellm.Logging.__init__ + + def init_spy(self, *args, **kwargs): + real_init(self, *args, **kwargs) + captured["logging_obj"] = self + + with patch.object(litellm.Logging, "__init__", init_spy): + litellm.completion( + model="gpt-4.1", + messages=[{"role": "user", "content": "hi"}], + client=openai.OpenAI(api_key="test-key", base_url="https://api.openai.com/v1"), + mock_response="ok", + ) + + assert captured["logging_obj"].litellm_params.get("data_residency") is None diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 759eee51ba4..e195b970fd5 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -279,6 +279,10 @@ def test_azure_rows_price_web_search_when_their_openai_model_does(prices: dict): PRIORITY_SUFFIX: Final = "_priority" MIN_PRIORITY_MARKUP: Final = 1.0 MAX_PRIORITY_MARKUP: Final = 5.0 +# Vertex pricing, verified 2026-11-03: https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing +VERIFIED_PRIORITY_MARKUP_MAXIMA: Final = MappingProxyType( + {("gemini-2.5-flash", "input_cost_per_audio_token_above_200k_tokens_priority"): 6.0} +) def _rate(value: JsonValue | None) -> float | None: @@ -312,7 +316,7 @@ def test_priority_rates_stay_within_a_premium_of_their_standard_rate(path: Path) overpriced: Final = tuple( f"{name}.{key} is {markup:.1f}x its standard rate" for name, key, markup in markups - if not MIN_PRIORITY_MARKUP <= markup <= MAX_PRIORITY_MARKUP + if not MIN_PRIORITY_MARKUP <= markup <= VERIFIED_PRIORITY_MARKUP_MAXIMA.get((name, key), MAX_PRIORITY_MARKUP) ) assert overpriced == (), ( "a priority rate is a premium on the standard rate, so one below it or far above it was likely copied from " @@ -866,6 +870,67 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert set(openai.get("supported_endpoints", ())) <= set(azure.get("supported_endpoints", ())) + + +_DATED_SNAPSHOT: Final = re.compile(r"(?P.+)-\d{4}-\d{2}-\d{2}") + + +def _azure_realtime_snapshots_with_an_alias(prices: Mapping[str, dict]) -> tuple[tuple[str, str], ...]: + dated: Final = ( + (name, _DATED_SNAPSHOT.fullmatch(name)) + for name, entry in prices.items() + if name.startswith("azure/") and "/v1/realtime" in entry.get("supported_endpoints", ()) + ) + return tuple((name, match["alias"]) for name, match in dated if match is not None and match["alias"] in prices) + + +def _realtime_ga_only(entry: dict) -> object: + return (entry.get("provider_specific_entry") or {}).get("realtime_ga_only") + + +def test_dated_azure_realtime_models_select_the_same_protocol_as_their_alias(prices: dict): + snapshots: Final = _azure_realtime_snapshots_with_an_alias(prices) + disagreeing: Final = tuple( + (snapshot, alias) + for snapshot, alias in snapshots + if _realtime_ga_only(prices[snapshot]) != _realtime_ga_only(prices[alias]) + ) + + assert snapshots + assert disagreeing == () + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices + + def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) return MappingProxyType( diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 3c0aab8a4b4..a3168ad6d36 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -5242,7 +5242,6 @@ def test_public_aliases( resolved_owner: Final = _get_owner(module_path, owner_name) alias_owner: Final = _get_instance(resolved_owner, new_name) if use_instance else resolved_owner old_value, new_value = getattr(alias_owner, old_name), getattr(alias_owner, new_name) - # Router's private aliases wrap the public method so they skip a subclass override of it. if isinstance(old_value, MethodType) and isinstance(new_value, MethodType): assert old_value.__func__ is new_value.__func__ or inspect.unwrap(old_value.__func__) is new_value.__func__ else: diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..afff995aa69 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -17,10 +17,16 @@ from typing import Final, NoReturn import pytest - import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map -from litellm.main import _build_custom_pricing_entry +from litellm.types.utils import ( + Choices, + CompletionTokensDetailsWrapper, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) from litellm.utils import _invalidate_model_cost_lowercase_map @@ -37,131 +43,6 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() -def test_build_custom_pricing_entry_includes_all_kwargs_fields(): - """All CustomPricingLiteLLMParams fields present in kwargs should be - included in the resulting entry dict.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.00025, - "cache_creation_input_token_cost": 0.005, - "output_cost_per_reasoning_token": 0.01, - "input_cost_per_audio_token": 0.003, - "unrelated_kwarg": "should_be_ignored", - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert entry["cache_read_input_token_cost"] == 0.00025 - assert entry["cache_creation_input_token_cost"] == 0.005 - assert entry["output_cost_per_reasoning_token"] == 0.01 - assert entry["input_cost_per_audio_token"] == 0.003 - assert "unrelated_kwarg" not in entry - - -def test_build_custom_pricing_entry_merges_model_info_metadata(): - """Fields from model_info (mode, supports_prompt_caching, max_tokens) - should be merged into the entry when present.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "id": "deployment-123", - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - -def test_build_custom_pricing_entry_setdefault_does_not_override_existing(): - """model_info uses setdefault, so it should not override a key that is - already present in the entry dict. Currently CustomPricingLiteLLMParams - and the model_info keys (mode, supports_prompt_caching, max_tokens) do - not overlap, but if they ever do, setdefault ensures the kwargs-sourced - value wins.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - # Verify setdefault behavior: if a model_info key already exists in - # the entry (e.g. from a future CustomPricingLiteLLMParams addition), - # setdefault must not overwrite it. - entry["mode"] = "embedding" # simulate pre-existing value - # Re-apply setdefault the same way _build_custom_pricing_entry does - entry.setdefault("mode", model_info["mode"]) - assert entry["mode"] == "embedding" # must NOT revert to "chat" - - -def test_build_custom_pricing_entry_skips_none_values(): - """Fields with None values in kwargs should not be included.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": None, # explicitly None - "cache_read_input_token_cost": None, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["input_cost_per_token"] == 0.001 - assert "output_cost_per_token" not in entry - assert "cache_read_input_token_cost" not in entry - - -def test_build_custom_pricing_entry_handles_no_model_info(): - """Should work correctly when model_info is None.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=None, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert "mode" not in entry - - def test_register_model_receives_cache_pricing_fields(): """End-to-end: when register_model is called with a full pricing entry, the cache pricing fields should be present in litellm.model_cost.""" @@ -192,23 +73,6 @@ def test_register_model_receives_cache_pricing_fields(): litellm.model_cost.pop(model_key, None) -def test_build_custom_pricing_entry_time_based(): - """Time-based pricing fields should be included correctly.""" - kwargs = { - "input_cost_per_second": 0.01, - "output_cost_per_second": 0.02, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_second"] == 0.01 - assert entry["output_cost_per_second"] == 0.02 - - def test_register_model_strips_none_litellm_provider(): """``get_model_info`` returns ``litellm_provider: None`` for deployments registered without a provider (e.g. ``Router.add_deployment`` flows). @@ -236,10 +100,7 @@ def test_register_model_strips_none_litellm_provider(): registered = litellm.model_cost.get(model_key) assert registered is not None, f"{model_key} should be in model_cost" # The key may be absent entirely, but if present it must not be None. - assert ( - "litellm_provider" not in registered - or registered["litellm_provider"] is not None - ) + assert "litellm_provider" not in registered or registered["litellm_provider"] is not None # Downstream consumers must accept this entry for any provider, # mirroring what the cost calculator does. assert _check_provider_match(registered, "openai") is True @@ -297,9 +158,7 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp # The strip must have removed the None-valued provider that # ``get_model_info`` returned. The key may be absent entirely, but # it must never be present with value ``None``. - assert "litellm_provider" not in registered or ( - registered["litellm_provider"] is not None - ), ( + assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), ( "register_model failed to strip litellm_provider=None returned " f"by get_model_info, got {registered.get('litellm_provider')!r}" ) @@ -360,14 +219,8 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa ) registered = litellm.model_cost[registered_key] - assert ( - registered.get("cache_creation_input_token_cost") - == builtin["cache_creation_input_token_cost"] - ) - assert ( - registered.get("cache_read_input_token_cost") - == builtin["cache_read_input_token_cost"] - ) + assert registered.get("cache_creation_input_token_cost") == builtin["cache_creation_input_token_cost"] + assert registered.get("cache_read_input_token_cost") == builtin["cache_read_input_token_cost"] assert registered["litellm_provider"] == "bedrock" usage = Usage( @@ -431,8 +284,7 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog): ) assert any( - registered_key in record.message - and "cache_creation_input_token_cost" in record.message + registered_key in record.message and "cache_creation_input_token_cost" in record.message for record in caplog.records ), "expected a warning naming the unmapped key and the cache cost fields" finally: @@ -619,17 +471,14 @@ def test_register_model_router_add_deployment_custom_pricing_applies(): # the deployment's provider. from litellm.utils import _check_provider_match - registered_keys = [ - k for k in (deployment_model, model_key) if k in litellm.model_cost - ] + registered_keys = [k for k in (deployment_model, model_key) if k in litellm.model_cost] assert registered_keys, ( - "Router.add_deployment did not register custom pricing for " - f"{model_key} / {deployment_model}" + f"Router.add_deployment did not register custom pricing for {model_key} / {deployment_model}" ) for k in registered_keys: - assert ( - _check_provider_match(litellm.model_cost[k], "openai") is True - ), f"custom pricing for {k} was dropped by _check_provider_match" + assert _check_provider_match(litellm.model_cost[k], "openai") is True, ( + f"custom pricing for {k} was dropped by _check_provider_match" + ) finally: litellm.model_cost.pop(model_key, None) litellm.model_cost.pop(deployment_model, None) @@ -646,12 +495,8 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): """ shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 try: @@ -666,12 +511,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response=[0.1, 0.2], ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 - assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert deployment_id not in litellm.model_cost sibling_response = litellm.embedding( model=shared_key, @@ -679,31 +522,33 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): api_key="fake-key", mock_response=[0.1, 0.2], ) - sibling_cost = litellm.completion_cost( - completion_response=sibling_response, call_type="embedding" - ) + sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding") assert sibling_cost == pytest.approx(10 * builtin_input_cost) finally: _restore_model_cost_entries(snapshot) def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): - """The request that carries custom pricing must still be costed with that - pricing (via its deployment id entry), while the shared backend key keeps - the built-in rate for siblings. - """ + """A Router registers the deployment's pricing under its id when the deployment is added. The routed request + is costed from that entry, and the shared backend key keeps the built-in rate for siblings.""" shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost != override_input_cost try: + litellm.register_model( + { + deployment_id: { + "input_cost_per_token": override_input_cost, + "output_cost_per_token": override_input_cost * 2, + "litellm_provider": "openai", + "mode": "embedding", + } + } + ) response = litellm.embedding( model=shared_key, input=["hello"], @@ -715,19 +560,10 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): mock_response=[0.1, 0.2], ) - request_cost = litellm.completion_cost( - completion_response=response, - model=shared_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - router_model_id=deployment_id, - ) - assert request_cost == pytest.approx(10 * override_input_cost) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * override_input_cost ) + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -736,12 +572,8 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): """Same isolation as the embedding path, exercised through completion().""" shared_key = "openai/gpt-4o-mini" deployment_id = "lit3991-wildcard-chat-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "gpt-4o-mini", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 try: @@ -756,24 +588,21 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response="hello back", ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert deployment_id not in litellm.model_cost finally: _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ - model_key = "openai/lit3991-direct-sdk-embed-model" - override_input_cost = 3e-05 +def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_writing_the_cost_map(): + """A direct SDK call with its own prices is costed with them and leaves ``litellm.model_cost`` alone, so the + next call for the same model name, and every other deployment of it, is priced from the cost map again.""" + model_key: Final = "openai/lit3991-direct-sdk-embed-model" + override_input_cost: Final = 3e-05 try: - response = litellm.embedding( + response: Final = litellm.embedding( model=model_key, input=["hello"], api_key="fake-key", @@ -782,63 +611,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): mock_response=[0.1, 0.2], ) - assert ( - litellm.model_cost[model_key]["input_cost_per_token"] - == override_input_cost + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * override_input_cost ) - cost = litellm.completion_cost( - completion_response=response, - model=model_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - ) - assert cost == pytest.approx(10 * override_input_cost) + + plain: Final = litellm.embedding(model=model_key, input=["hello"], api_key="fake-key", mock_response=[0.1, 0.2]) + assert plain._hidden_params["response_cost"] != response._hidden_params["response_cost"] finally: litellm.model_cost.pop(model_key, None) _invalidate_model_cost_lowercase_map() -@pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) -def test_register_model_prices_a_geo_alias_from_its_builtin_when_the_lookup_fails( - monkeypatch: pytest.MonkeyPatch, route: str -) -> None: - from litellm.types.utils import ModelResponse, Usage - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - _invalidate_model_cost_lowercase_map() - canonical_key: Final = "au.anthropic.claude-sonnet-5" - alias: Final = f"bedrock/{route}{canonical_key}" - canonical: Final = litellm.model_cost[canonical_key] - - def lookup_tripped_by_a_concurrent_registration(model: str, custom_llm_provider: str | None = None) -> NoReturn: - raise RuntimeError("dictionary changed size during iteration") - - with pytest.MonkeyPatch.context() as lookup_patch: - lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_by_a_concurrent_registration) - litellm.register_model( - {alias: {"litellm_provider": "bedrock", "mode": "chat", "supports_vision": True}}, - persist_across_reloads=False, - ) - - usage: Final = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100) - alias_cost: Final = litellm.completion_cost( - completion_response=ModelResponse(model=alias, usage=usage), model=alias, custom_llm_provider="bedrock" - ) - canonical_cost: Final = litellm.completion_cost( - completion_response=ModelResponse(model=canonical_key, usage=usage), - model=canonical_key, - custom_llm_provider="bedrock", - ) - try: - assert alias_cost == canonical_cost > 0 - assert litellm.get_model_info(alias)["input_cost_per_token"] == canonical["input_cost_per_token"] - assert litellm.get_model_info(alias)["supports_vision"] is True - finally: - _invalidate_model_cost_lowercase_map() - - def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(monkeypatch: pytest.MonkeyPatch) -> None: geo_key: Final = "au.anthropic.lit-5853-geo-model" geo_row: Final = { @@ -1407,10 +1191,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( @@ -1609,12 +1393,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): +def test_completion_prices_cost_per_second_from_the_call_without_writing_the_cost_map(): model_key: Final = "openai/test-cost-per-second-registration" original_entries: Final = _snapshot_model_cost_entries([model_key]) try: - litellm.completion( + response: Final = litellm.completion( model=model_key, messages=[{"role": "user", "content": "hello"}], api_key="fake-key", @@ -1622,7 +1406,11 @@ def test_completion_registers_cost_per_second_pricing(): mock_response="hello back", ) - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] > 0 + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.02) == pytest.approx( + 0.02 * response._response_ms / 1000 + ) finally: _restore_model_cost_entries(original_entries) @@ -1676,31 +1464,238 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): _restore_model_cost_entries(model_cost_entries) -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request +def _audio_reply() -> ModelResponse: + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=1000, + total_tokens=2000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + ) + return ModelResponse(model="gpt-audio-2025-08-28", choices=[Choices(message=Message(content="hi"))], usage=usage) - deployment_id: Final = "baseten/zai-org/glm-5.2" - builtin_key: Final = "baseten/zai-org/GLM-5.2" - shared_key: Final = "openai/zai-org/GLM-5.2" - model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key)) - builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) - openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) + +_AUDIO_RATES: Final = { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, +} + + +def test_router_call_with_audio_rates_is_priced_per_modality_from_its_registered_deployment(): + """A Router registers the deployment's full custom pricing when the deployment is added. The per-call flat + prices must not override that entry, or 1000 audio input plus 1000 audio output tokens would bill at the text + rates ($0.0125) instead of the audio rates ($0.12).""" + deployment_id: Final = "audio-rates-deployment" + snapshot: Final = _snapshot_model_cost_entries([deployment_id]) try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, + litellm.register_model({deployment_id: {**_AUDIO_RATES, "litellm_provider": "openai", "mode": "chat"}}) + response: Final = litellm.completion( + model="openai/gpt-audio-2025-08-28", + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + metadata={"model_info": {"id": deployment_id}}, + mock_response=_audio_reply(), + **_AUDIO_RATES, ) - - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 - assert litellm.model_cost[builtin_key] == builtin_row_before + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) finally: - _restore_model_cost_entries(model_cost_entries) - litellm.open_ai_chat_completion_models.intersection_update(openai_models_before) + _restore_model_cost_entries(snapshot) + + +def test_direct_call_with_flat_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-flat-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 2.5e-06 + 1000 * 1e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_audio_rates_is_priced_per_modality_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-audio-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + input_cost_per_audio_token=4e-05, + output_cost_per_audio_token=8e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_directional_per_second_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-per-second-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_second=0.004, + ) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.004) == pytest.approx( + 0.004 * response._response_ms / 1000 + ) + assert response._hidden_params["response_cost"] > 0 + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_router_call_with_request_prices_bills_them_over_the_deployment_rates(): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "lit-request-override", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "mock_response": "ok", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, + }, + } + ] + ) + overridden: Final = router.completion( + model="lit-request-override", + messages=[{"role": "user", "content": "hello"}], + input_cost_per_token=5e-05, + output_cost_per_token=9e-05, + ) + assert overridden._hidden_params["response_cost"] == pytest.approx( + overridden.usage.prompt_tokens * 5e-05 + overridden.usage.completion_tokens * 9e-05 + ) + plain: Final = router.completion(model="lit-request-override", messages=[{"role": "user", "content": "hello"}]) + assert plain._hidden_params["response_cost"] == pytest.approx( + plain.usage.prompt_tokens * 1e-06 + plain.usage.completion_tokens * 2e-06 + ) + + +def test_direct_call_flat_rates_override_the_catalogs_tiers_per_direction(): + model_key: Final = "openai/lit-tiered-catalog-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07}, + {"range": [128000, 1000000], "input_cost_per_token": 6e-07, "output_cost_per_token": 1.2e-06}, + ], + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + output_cost_per_token=2e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 2e-05 + ) + input_only: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + ) + assert input_only._hidden_params["response_cost"] == pytest.approx( + input_only.usage.prompt_tokens * 1e-05 + input_only.usage.completion_tokens * 6e-07 + ) + assert "input_cost_per_token" not in litellm.model_cost[model_key] + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_only_a_per_second_rate_on_a_mapped_model_is_billed_per_second(): + model_key: Final = "openai/lit-mapped-token-rates-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + cost_per_second=0.02, + ) + token_rate_cost: Final = response.usage.prompt_tokens * 1e-06 + response.usage.completion_tokens * 2e-06 + assert response._hidden_params["response_cost"] != pytest.approx(token_rate_cost) + assert response._hidden_params["response_cost"] == pytest.approx(0.02 * response._response_ms / 1000, rel=0.25) + assert litellm.model_cost[model_key]["input_cost_per_token"] == 1e-06 + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_flat_rate_applies_to_the_callers_own_tiers_not_the_catalogs(): + model_key: Final = "openai/lit-caller-tiers-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 1000000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07} + ], + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + tiered_pricing=[{"range": [0, 1000000], "input_cost_per_token": 5e-06, "output_cost_per_token": 9e-06}], + ) + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 9e-06 + ) + assert litellm.model_cost[model_key]["tiered_pricing"][0]["output_cost_per_token"] == 6e-07 + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_router/core/conftest.py b/tests/unit/test_router/core/conftest.py index dffa938c441..3477cfc0c3d 100644 --- a/tests/unit/test_router/core/conftest.py +++ b/tests/unit/test_router/core/conftest.py @@ -1,4 +1,6 @@ -from collections.abc import Callable +from collections.abc import Callable, Generator +from pathlib import Path +from typing import Final import pytest @@ -15,12 +17,53 @@ def host() -> ScriptedHost: def make_core(request: pytest.FixtureRequest, host: ScriptedHost) -> Callable[..., RouterCore]: """``make_core(**router_arguments)``: the core under test, on the ``host`` fixture. Every test that uses it runs once per core in ``CORE_BUILDERS`` (``-k python`` or ``-k rust`` picks one).""" - builder = CORE_BUILDERS[request.param] + name: Final[str] = str(request.param) # pyright: ignore[reportAny] # the fixture parameter + builder: Final = CORE_BUILDERS[name] def build(**arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments try: return builder(host, **arguments) except CoreDeclined as declined: - pytest.skip(f"{request.param} core declines this configuration: {declined}") + pytest.skip(f"{name} core declines this configuration: {declined}") return build + + +_RUST_PARITY_PENDING: Final = frozenset( + line.strip() + for line in (Path(__file__).parent / "rust_parity_pending.txt").read_text().splitlines() + if line.strip() and not line.startswith("#") +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Characterization tests the Rust core does not match yet are listed in + ``rust_parity_pending.txt`` and expected to fail (strictly: one that starts passing must leave the list).""" + here: Final = Path(__file__).parent + for item in items: + path: Final = Path(str(item.path)) + if not path.is_relative_to(here): + continue # the hook sees every collected item, not only this folder's + name: Final = "::".join((path.relative_to(here).as_posix(), *item.nodeid.split("::")[1:])) + if name in _RUST_PARITY_PENDING: + item.add_marker(pytest.mark.xfail(reason="Rust core parity pending", strict=True)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item: pytest.Item) -> Generator[None, object, None]: + """A test that reaches a member or request the Rust core does not serve yet is skipped with + that member named, so the run separates "not served yet" from a wrong answer.""" + outcome: Final = yield + excinfo: Final[object] = getattr(outcome, "excinfo", None) + if not isinstance(excinfo, tuple): + return + info: Final[tuple[object, ...]] = excinfo # pyright: ignore[reportUnknownVariableType] # pluggy's (type, value, traceback) + if len(info) < 2: + return + from litellm.router_rust.core import RustCoreUnsupported + + error: Final[object] = info[1] + cause: Final[object] = getattr(error, "__cause__", None) + force_exception: Final[object] = getattr(outcome, "force_exception", None) + if (isinstance(error, RustCoreUnsupported) or isinstance(cause, RustCoreUnsupported)) and callable(force_exception): + force_exception(pytest.skip.Exception(f"not served by the Rust core yet: {error}", _use_item_location=True)) diff --git a/tests/unit/test_router/core/construction_precall_helpers.py b/tests/unit/test_router/core/construction_precall_helpers.py new file mode 100644 index 00000000000..c83bd37d692 --- /dev/null +++ b/tests/unit/test_router/core/construction_precall_helpers.py @@ -0,0 +1,98 @@ +""" +Helpers for the construction / pre-call characterization tests. + +``CallbackHost`` is a ``ScriptedHost`` whose callback manager also has +``remove_callback_from_list_by_object`` (litellm's ``LoggingCallbackManager`` has it; ``HostCallbacks`` +does not), so the path that switches optional pre-call checks off at runtime can run. +""" + +import copy +import gc +import uuid +from collections.abc import Generator, Mapping +from typing import Final + +import litellm +from tests.unit.test_router.core.scripted_host import HostCallbacks, ScriptedHost, deployment + + +class RemovingHostCallbacks(HostCallbacks): + def remove_callback_from_list_by_object( + self, + callback_list: object, + obj: object, + require_self: bool = True, + ) -> None: + """``LoggingCallbackManager.remove_callback_from_list_by_object``, on the host's lists.""" + if not isinstance(callback_list, list): + return + if require_self: + doomed = [c for c in callback_list if getattr(c, "__self__", None) is obj] + else: + doomed = [c for c in callback_list if c == obj] + for callback in doomed: + callback_list.remove(callback) + + +class CallbackHost(ScriptedHost): + __slots__ = () + + @property + def callbacks(self) -> RemovingHostCallbacks: # pyright: ignore[reportIncompatibleVariableOverride] + return RemovingHostCallbacks(self) + + +def callback_names(callbacks: list[object]) -> list[str]: # mutable-ok: a fresh list + """Each callback by name: a bound method by its function name, anything else by its class name.""" + names: Final[list[str]] = [] + for callback in callbacks: + function = getattr(callback, "__func__", None) + names.append(function.__name__ if function is not None else type(callback).__name__) + return names + + +def host_callback_names(host: ScriptedHost) -> dict[str, list[str]]: # mutable-ok: a fresh dict + return {name: callback_names(callbacks) for name, callbacks in host.lists.items()} + + +def windowed( + group: str, deployment_id: str, max_input_tokens: int, **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry + """A deployment declaring ``max_input_tokens`` under a model name no other test uses: a deployment's + model_info is written into the process-wide cost map under its model name and id.""" + model_info: Final = {"max_input_tokens": max_input_tokens, **dict(params.pop("model_info", None) or {})} # pyright: ignore[reportArgumentType] + return deployment( + group, + deployment_id, + model=f"openai/cw-{deployment_id}-{uuid.uuid4().hex[:8]}", + model_info=model_info, + **params, + ) + + +def collect_other_routers() -> None: + """Drop unreachable routers from earlier tests: retire() and runtime pre-call-check changes look + at every live router in the process, not only the ones on this host.""" + gc.collect() + + +def restoring_model_cost(*names: str) -> Generator[None, None, None]: + """Undo what routers built meanwhile wrote into the process cost map: keys they added are removed + and the named entries get their old value back.""" + added_before: Final = frozenset(litellm.model_cost) + saved: Final = {name: copy.deepcopy(litellm.model_cost[name]) for name in names if name in litellm.model_cost} + yield + for key in frozenset(litellm.model_cost) - added_before: + litellm.model_cost.pop(key, None) + litellm.model_cost.update(saved) + + +__all__ = [ + "CallbackHost", + "RemovingHostCallbacks", + "callback_names", + "collect_other_routers", + "host_callback_names", + "restoring_model_cost", + "windowed", +] diff --git a/tests/unit/test_router/core/endpoints_helpers.py b/tests/unit/test_router/core/endpoints_helpers.py new file mode 100644 index 00000000000..cc79e05a686 --- /dev/null +++ b/tests/unit/test_router/core/endpoints_helpers.py @@ -0,0 +1,212 @@ +""" +Helpers for the endpoint characterization tests (``test_endpoints_characterization*.py``). + +``EndpointHost`` is a ``ScriptedHost`` that also scripts the host ops ``ScriptedHost`` leaves to +litellm (assistants, skills, realtime, fine-tuning, batches, ...), so every ``host.invoke`` the core +makes is answered and recorded. + +Some endpoints do not go through ``host.invoke``: the Python core binds litellm's own functions for +them when it is built (vector stores, Google GenAI, OCR, search, videos, containers, interactions, +managed agents). ``patch_direct_endpoints`` replaces those litellm functions, before the core is +built, with recording stand-ins that answer from the same host script, under the same ``__name__`` +(the core routes on the handler's name). A call to one of them is recorded with the function's name +as its ``op``. +""" + +import importlib +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +from tests.unit.test_router.core.scripted_host import INFERENCE_OPS, ScriptedHost + +EXTRA_HOST_OPS: Final = frozenset( + { + "aadapter_completion", + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", + "acompact_responses", + "_arealtime", + "acreate_realtime_client_secret", + "arealtime_calls", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", + "_aresponses_websocket", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", + "afile_list", + "aimage_edit", + "allm_passthrough_route", + "acreate_skill", + "alist_skills", + "aget_skill", + "adelete_skill", + "aretrieve_batch", + "acancel_batch", + "alist_batches", + } +) + + +class EndpointHost(ScriptedHost): + """``ScriptedHost`` that scripts every host op the core's endpoints invoke.""" + + __slots__ = () + + def invoke(self, op: str) -> Callable[..., object]: + if op in INFERENCE_OPS or op not in EXTRA_HOST_OPS: + return super().invoke(op) + return recording_endpoint(self, op, is_async=True) + + +def recording_endpoint(host: ScriptedHost, name: str, *, is_async: bool) -> Callable[..., object]: + """A stand-in for litellm's ``name`` that records the call on ``host`` and answers from its script.""" + if is_async: + + async def endpoint(*args: object, **kwargs: object) -> object: # kwargs-ok: the endpoint's request + return await host._answer_async(name, args, kwargs) # pyright: ignore[reportPrivateUsage] # the test host + + endpoint.__name__ = name + endpoint.__qualname__ = name + return endpoint + + def sync_endpoint(*args: object, **kwargs: object) -> object: # kwargs-ok: the endpoint's request + return host._answer_sync(name, args, kwargs) # pyright: ignore[reportPrivateUsage] # the test host + + sync_endpoint.__name__ = name + sync_endpoint.__qualname__ = name + return sync_endpoint + + +# module -> {attribute on the module: the function's own name} +DIRECT_ENDPOINTS: Final[Mapping[str, Mapping[str, str]]] = { + "litellm.vector_stores.main": { + name: name + for name in ( + "asearch", + "search", + "acreate", + "create", + "aretrieve", + "retrieve", + "alist", + "list", + "aupdate", + "update", + "adelete", + "delete", + ) + }, + "litellm.vector_stores": {"acreate": "acreate"}, + "litellm.vector_store_files.main": { + name: name + for name in ( + "acreate", + "create", + "alist", + "list", + "aretrieve", + "retrieve", + "aretrieve_content", + "retrieve_content", + "aupdate", + "update", + "adelete", + "delete", + ) + }, + "litellm.google_genai": { + name: name + for name in ("agenerate_content", "generate_content", "agenerate_content_stream", "generate_content_stream") + }, + "litellm.ocr": {"aocr": "aocr", "ocr": "ocr"}, + "litellm.search": {"asearch": "asearch", "search": "search"}, + "litellm.decisions": {"adecisions": "adecisions", "decisions": "decisions"}, + "litellm.systemone": {"asystemone": "asystemone", "systemone": "systemone"}, + "litellm.extraction": {"aextraction": "aextraction", "extraction": "extraction"}, + "litellm.videos": { + f"{prefix}video_{name}": f"{prefix}video_{name}" + for prefix in ("a", "") + for name in ( + "generation", + "list", + "status", + "content", + "remix", + "create_character", + "get_character", + "edit", + "extension", + ) + }, + "litellm.containers": { + f"{prefix}{name}": f"{prefix}{name}" + for prefix in ("a", "") + for name in ("create_container", "list_containers", "retrieve_container", "delete_container") + }, + "litellm.interactions": { + f"{prefix}{name}": f"{prefix}{name}" for prefix in ("a", "") for name in ("create", "get", "delete", "cancel") + }, + "litellm.interactions.agents": { + f"{prefix}{name}": f"{prefix}{name}" + for prefix in ("a", "") + for name in ("create", "get", "delete", "list", "list_versions") + }, +} + +# Recorded op names for the direct endpoints whose bare names collide across modules. +RECORDED_NAMES: Final[Mapping[tuple[str, str], str]] = { + **{("litellm.vector_stores.main", name): f"vs.{name}" for name in DIRECT_ENDPOINTS["litellm.vector_stores.main"]}, + ("litellm.vector_stores", "acreate"): "vs.acreate", + **{ + ("litellm.vector_store_files.main", name): f"vsf.{name}" + for name in DIRECT_ENDPOINTS["litellm.vector_store_files.main"] + }, + **{("litellm.interactions", name): f"interactions.{name}" for name in DIRECT_ENDPOINTS["litellm.interactions"]}, + **{ + ("litellm.interactions.agents", name): f"agents.{name}" + for name in DIRECT_ENDPOINTS["litellm.interactions.agents"] + }, +} + + +def patch_direct_endpoints(monkeypatch: pytest.MonkeyPatch, host: ScriptedHost) -> None: + """Replace litellm's directly bound endpoint functions with recording stand-ins on ``host``. + + The stand-in keeps the function's ``__name__`` (the core routes on it); the call is recorded + under ``RECORDED_NAMES`` when the bare name is ambiguous, else under the function's name.""" + for module_name, attributes in DIRECT_ENDPOINTS.items(): + module: Final = importlib.import_module(module_name) + for attribute, function_name in attributes.items(): + original: Final[object] = getattr(module, attribute) + is_async: Final = function_name.startswith("a") + recorded: Final = RECORDED_NAMES.get((module_name, attribute), function_name) + stand_in: Final = recording_endpoint(host, recorded, is_async=is_async) + stand_in.__name__ = getattr(original, "__name__", function_name) + monkeypatch.setattr(module, attribute, stand_in) + from litellm.containers import endpoint_factory + + for name in list(endpoint_factory._generated_endpoints): # pyright: ignore[reportPrivateUsage] # litellm's registry + monkeypatch.setitem( + endpoint_factory._generated_endpoints, # pyright: ignore[reportPrivateUsage] # litellm's registry + name, + recording_endpoint(host, name, is_async=name.startswith("a")), + ) + + +__all__ = [ + "DIRECT_ENDPOINTS", + "EXTRA_HOST_OPS", + "EndpointHost", + "patch_direct_endpoints", + "recording_endpoint", +] diff --git a/tests/unit/test_router/core/fallbacks_helpers.py b/tests/unit/test_router/core/fallbacks_helpers.py new file mode 100644 index 00000000000..c88c31a9371 --- /dev/null +++ b/tests/unit/test_router/core/fallbacks_helpers.py @@ -0,0 +1,105 @@ +"""Helpers for the fallbacks characterization tests (``test_fallbacks_characterization*.py``).""" + +from collections.abc import Callable, Mapping +from typing import Final + +from litellm.router_core import RouterCore +from litellm.router_python_host import Endpoint +from tests.unit.test_router.core.scripted_host import Outcome, ScriptedHost, deployment + +MakeCore = Callable[..., RouterCore] + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +GROUPS_ABC: Final = [deployment("a", "a"), deployment("b", "b"), deployment("c", "c")] +GROUP_A_OF_THREE: Final = [deployment("a", "a1"), deployment("a", "a2"), deployment("a", "a3"), deployment("b", "b")] +TIERED_A: Final = [ + deployment("a", "a1", order=1), + deployment("a", "a2", order=2), + deployment("b", "b"), + deployment("c", "c"), +] + + +def build(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(**{"model_list": GROUPS_ABC, "num_retries": 0, **arguments}) + + +async def complete(core: RouterCore, model: str = "a", **request: object) -> object: # kwargs-ok: request kwargs + return await core.acompletion(model=model, messages=MESSAGES, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def complete_sync(core: RouterCore, model: str = "a", **request: object) -> object: # kwargs-ok: request kwargs + return core.completion(model=model, messages=MESSAGES, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def hidden(response: object) -> Mapping[str, object]: + return getattr(response, "_hidden_params") + + +def headers(response: object) -> Mapping[str, object]: + found: Final = hidden(response)["additional_headers"] + assert isinstance(found, Mapping) + return found + + +def metadata_of(kwargs: Mapping[str, object]) -> Mapping[str, object]: + found: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {} + assert isinstance(found, Mapping) + return found + + +def show_router_debug(host: ScriptedHost) -> None: + host.set_dynamic(expose_router_debug_in_errors=True) + + +def fails_once_then_answers(error: BaseException) -> Callable[..., Outcome]: + """One scripted outcome shared by several models: the first call to any of them raises + ``error``, every later call answers the default response.""" + calls: Final = [0] + + def outcome(**_request: object) -> Outcome: # kwargs-ok: the scripted request + calls[0] += 1 + return error if calls[0] == 1 else None + + return outcome + + +class NamedEndpointHost(ScriptedHost): + """A ``ScriptedHost`` whose ``anthropic_messages`` endpoint carries that name, as litellm's + own function does. The core recognises a /v1/messages call (and its safeguard-refusal gate) + by the endpoint's ``__name__``; ``ScriptedHost``'s endpoints are all named ``endpoint``.""" + + __slots__ = () + + def invoke(self, op: str) -> Endpoint: + endpoint: Final = super().invoke(op) + if op == "anthropic_messages": + endpoint.__name__ = op # pyright: ignore[reportFunctionMemberAccess] # a plain function + return endpoint + + +def anthropic_refusal(*, with_stop_details: bool = True) -> dict[str, object]: # mutable-ok: a provider response + response: Final[dict[str, object]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [], + "stop_reason": "refusal", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + if with_stop_details: + response["stop_details"] = {"type": "refusal", "category": None, "explanation": "no"} + return response + + +def anthropic_message(text: str = "fine") -> dict[str, object]: # mutable-ok: a provider response + return { + "id": "msg_2", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } diff --git a/tests/unit/test_router/core/gaps_l11100_helpers.py b/tests/unit/test_router/core/gaps_l11100_helpers.py new file mode 100644 index 00000000000..38fb2006503 --- /dev/null +++ b/tests/unit/test_router/core/gaps_l11100_helpers.py @@ -0,0 +1,112 @@ +"""Shared helpers for ``test_gaps_l11100*.py``: strategy-router marker deployments and a host whose +callback registration can also remove a callback from every list (what the adaptive router's hook +sync asks of litellm's ``LoggingCallbackManager``).""" + +from collections.abc import Callable, Mapping +from typing import Final + +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import HostCallbacks, ScriptedHost + +MakeCore = Callable[..., RouterCore] + +TIERS: Final = {"SIMPLE": "small", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"} + + +class RemovingHostCallbacks(HostCallbacks): + """``HostCallbacks`` plus ``remove_callback_from_all_lists``, as litellm's callback manager has it.""" + + def __init__(self, host: "RemovingHost") -> None: + super().__init__(host) + self._owner: Final = host + + def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: + del require_self + for name, callbacks in self._owner.lists.items(): + self._owner.lists[name] = [callback for callback in callbacks if callback is not obj] + + +class RemovingHost(ScriptedHost): + """A ``ScriptedHost`` whose callbacks can be removed from every list.""" + + __slots__ = () + + @property + def callbacks(self) -> RemovingHostCallbacks: # pyright: ignore[reportIncompatibleVariableOverride] # same methods plus one + return RemovingHostCallbacks(self) + + +def complexity_marker( + name: str = "smart", + deployment_id: str = "marker", + *, + config: Mapping[str, object] | None = None, + **params: object, # kwargs-ok: extra marker litellm_params +) -> dict[str, object]: # mutable-ok: a model_list entry + """An ``auto_router/complexity_router`` marker with the shipped tiers and default model ``big``.""" + return { + "model_name": name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": dict(config) if config is not None else {"tiers": dict(TIERS)}, + "complexity_router_default_model": "big", + **params, + }, + "model_info": {"id": deployment_id}, + } + + +def adaptive_marker( + name: str, deployment_id: str, available_models: list[str], **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry # kwargs-ok: extra marker litellm_params + """An ``auto_router/adaptive_router`` marker over ``available_models``.""" + return { + "model_name": name, + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": {"available_models": available_models}, + **params, + }, + "model_info": {"id": deployment_id}, + } + + +def marker( + name: str, deployment_id: str, model: str, **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry # kwargs-ok: marker litellm_params + """Any strategy-router marker: ``model`` is its ``auto_router/...`` litellm model.""" + return {"model_name": name, "litellm_params": {"model": model, **params}, "model_info": {"id": deployment_id}} + + +def registered_tags(core: RouterCore, kind: str) -> dict[str, list[tuple[str, ...]]]: # mutable-ok: a fresh dict + """``{router name: [tags of each registered strategy]}`` for a pre-routing strategy registry.""" + registry: Final = core.registry().plugins(kind) # pyright: ignore[reportCallIssue, reportArgumentType] # one of the strategy kinds + assert isinstance(registry, dict) + return {name: [entry.tags for entry in entries] for name, entries in registry.items()} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # TaggedPreRoutingStrategy lists + + +def model_ids(core: RouterCore) -> list[str]: # mutable-ok: a fresh list + """The ``model_info.id`` of every deployment on the model list, in order.""" + return [str(entry["model_info"]["id"]) for entry in core.registry().all()] # pyright: ignore[reportAny] # model_list rows + + +class ModelsEndpoint: + """A stand-in for the ``AsyncHTTPHandler`` that model-info discovery reads ``/v1/models`` with: + answers ``body`` (or raises ``error``) and records each request's url and headers.""" + + def __init__(self, body: Mapping[str, object] | None = None, error: BaseException | None = None) -> None: + self.body: Final = body or {"data": []} + self.error: Final = error + self.requests: Final[list[tuple[str, dict[str, str]]]] = [] + + async def get( + self, url: str, headers: dict[str, str] | None = None, **_: object + ) -> object: # kwargs-ok: httpx options + import json + + import httpx + + self.requests.append((url, dict(headers or {}))) + if self.error is not None: + raise self.error + return httpx.Response(200, content=json.dumps(self.body).encode(), request=httpx.Request("GET", url)) diff --git a/tests/unit/test_router/core/registry_helpers.py b/tests/unit/test_router/core/registry_helpers.py new file mode 100644 index 00000000000..aa154454b8d --- /dev/null +++ b/tests/unit/test_router/core/registry_helpers.py @@ -0,0 +1,47 @@ +"""Shared helpers for the registry characterization tests (``test_registry_characterization*.py``).""" + +import uuid +from collections.abc import Callable, Iterable, Mapping + +from litellm.router_core import RouterCore +from litellm.types.router import Deployment +from tests.unit.test_router.core.scripted_host import deployment + +MakeCore = Callable[..., RouterCore] + +MESSAGES = [{"role": "user", "content": "hi"}] + + +def ids(deployments: Iterable[Mapping[str, object]] | None) -> list[str]: # mutable-ok: a fresh list + """The ``model_info.id`` of each deployment dict, in order.""" + return [str(entry["model_info"]["id"]) for entry in deployments or ()] # pyright: ignore[reportIndexIssue] # deployment dicts + + +def names_and_ids( + deployments: Iterable[Mapping[str, object]] | None, +) -> list[tuple[str, str]]: # mutable-ok: a fresh list + """``(model_name, model_info.id)`` of each deployment dict, in order.""" + return [(str(entry["model_name"]), str(entry["model_info"]["id"])) for entry in deployments or ()] # pyright: ignore[reportIndexIssue] # deployment dicts + + +def typed(model_name: str, deployment_id: str, **params: object) -> Deployment: # kwargs-ok: litellm_params + """A ``Deployment`` built from the same entry ``scripted_host.deployment`` returns.""" + return Deployment(**deployment(model_name, deployment_id, **params)) # pyright: ignore[reportArgumentType] # a model_list entry + + +def team_deployment( + model_name: str, deployment_id: str, team_id: str, public_name: str, **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry # kwargs-ok: litellm_params + """A team-owned deployment: ``model_info.team_id`` and ``team_public_model_name`` set.""" + extra_info: dict[str, object] = dict(params.pop("model_info", None) or {}) # pyright: ignore[reportArgumentType] # a model_info dict + return deployment( + model_name, + deployment_id, + model_info={"team_id": team_id, "team_public_model_name": public_name, **extra_info}, + **params, + ) + + +def unique_id(prefix: str = "reg") -> str: + """A deployment id no other test uses, for tests that write into the process-wide ``litellm.model_cost``.""" + return f"{prefix}-{uuid.uuid4().hex}" diff --git a/tests/unit/test_router/core/retries_health_helpers.py b/tests/unit/test_router/core/retries_health_helpers.py new file mode 100644 index 00000000000..eae0b836530 --- /dev/null +++ b/tests/unit/test_router/core/retries_health_helpers.py @@ -0,0 +1,198 @@ +""" +Helpers for the retries / deployment-health characterization tests (``test_retries_health_characterization*.py``). + +Backoff values carry litellm's random jitter (up to ``JITTER`` seconds), so sleeps are asserted inside +``[base, base + JITTER)``. +""" + +import asyncio +import copy +from collections.abc import Callable, Mapping +from typing import Final + +import httpx + +import litellm +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import Call, ScriptedHost + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + +MakeCore = Callable[..., RouterCore] +ErrorBuilder = Callable[[], Exception] + +_REQUEST: Final = httpx.Request("POST", "https://api.openai.com/v1/chat/completions") + + +def fresh(builder: ErrorBuilder) -> Callable[..., Exception]: + """A scripted outcome that raises a new error per call.""" + return lambda **_: builder() + + +def http_response(status: int, headers: Mapping[str, str] | None = None) -> httpx.Response: + return httpx.Response(status, headers=dict(headers or {}), request=_REQUEST) + + +def not_found() -> litellm.NotFoundError: + return litellm.NotFoundError(message="no such model", llm_provider="openai", model="gpt-4o") + + +def permission_denied() -> litellm.PermissionDeniedError: + return litellm.PermissionDeniedError( + message="forbidden", llm_provider="openai", model="gpt-4o", response=http_response(403) + ) + + +def unprocessable() -> litellm.UnprocessableEntityError: + return litellm.UnprocessableEntityError( + message="unprocessable", llm_provider="openai", model="gpt-4o", response=http_response(422) + ) + + +def service_unavailable() -> litellm.ServiceUnavailableError: + return litellm.ServiceUnavailableError(message="unavailable", llm_provider="openai", model="gpt-4o") + + +def bad_gateway() -> litellm.BadGatewayError: + return litellm.BadGatewayError(message="bad gateway", llm_provider="openai", model="gpt-4o") + + +def api_connection() -> litellm.APIConnectionError: + return litellm.APIConnectionError(message="connection reset", llm_provider="openai", model="gpt-4o") + + +def api_status(status: int) -> ErrorBuilder: + return lambda: litellm.APIError( + status_code=status, message=f"status {status}", llm_provider="openai", model="gpt-4o" + ) + + +def plain_error() -> ValueError: + return ValueError("not a provider error") + + +def rate_limited(retry_after: str | None = None) -> litellm.RateLimitError: + headers: Final = {"retry-after": retry_after} if retry_after is not None else {} + return litellm.RateLimitError( + message="slow down", llm_provider="openai", model="gpt-4o", response=httpx.Response(429, headers=headers) + ) + + +async def attempt(core: RouterCore, model: str = "g", **kwargs: object) -> object: # kwargs-ok: request kwargs + """The response, or the error the core raised. A request the Rust core does not serve yet propagates, so the + test is skipped rather than failed.""" + from litellm.router_rust.core import RustCoreUnsupported + + try: + return await core.acompletion(model=model, messages=MESSAGES, **kwargs) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + except RustCoreUnsupported: + raise + except Exception as error: + return error + + +def attempt_sync(core: RouterCore, model: str = "g", **kwargs: object) -> object: # kwargs-ok: request kwargs + from litellm.router_rust.core import RustCoreUnsupported + + try: + return core.completion(model=model, messages=MESSAGES, **kwargs) # pyright: ignore[reportAttributeAccessIssue] # every core serves completion + except RustCoreUnsupported: + raise + except Exception as error: + return error + + +def cooled_down(core: RouterCore, ids: list[str]) -> dict[str, Mapping[str, object]]: # mutable-ok: a fresh dict + return dict(core.health().cooldowns.get_active_cooldowns(ids, None)) + + +def headers_of(response: object) -> Mapping[str, object]: + hidden: Final = getattr(response, "_hidden_params", None) or {} + return hidden.get("additional_headers") or {} + + +class MetadataRecorder: + """A scripted outcome factory that snapshots the request metadata each call carried, then answers.""" + + KEYS: Final = ( + "attempted_retries", + "max_retries", + "model_group", + "model_group_size", + "request_retry_count", + "previous_models", + ) + + def __init__(self) -> None: + self.snapshots: Final[list[dict[str, object]]] = [] + + def then(self, builder: ErrorBuilder | None) -> Callable[..., object]: + def outcome(**kwargs: object) -> object: # kwargs-ok: the request + metadata: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {} + assert isinstance(metadata, Mapping) + self.snapshots.append({key: copy.deepcopy(metadata.get(key)) for key in self.KEYS}) + return builder() if builder is not None else None + + return outcome + + +class BackgroundSuccessLoggingHost(ScriptedHost): + """``ScriptedHost`` whose async success callbacks run in a background task after the response is returned, + as litellm's logging worker runs them (``ScriptedHost`` runs them before returning). Await ``drain()`` + before reading what they wrote.""" + + __slots__ = ("_pending",) + + def __init__(self) -> None: + super().__init__() + self._pending: list[asyncio.Task[None]] = [] + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + call, outcome = self._outcome(op, args, kwargs) + error, response = self._finish(op, call, outcome) + if error is not None: + logged_failure: Final = self._logging_kwargs(call, error, response) + for callback in list(self.lists.get("failure_callback", ())): + if callable(callback): + callback(logged_failure, None, 0.0, 0.0) + for callback in list(self.lists.get("_async_failure_callback", ())): + if callable(callback): + await callback(logged_failure, None, 0.0, 0.0) + raise error + + async def log_success(logged_call: Call) -> None: + logged: Final = self._logging_kwargs(logged_call, None, response) + for callback in list(self.lists.get("_async_success_callback", ())): + if callable(callback): + await callback(logged, response, 0.0, 0.0) + + self._pending.append(asyncio.ensure_future(log_success(call))) + return response + + async def drain(self) -> None: + while self._pending: + await self._pending.pop() + + +__all__ = [ + "MESSAGES", + "BackgroundSuccessLoggingHost", + "ErrorBuilder", + "MakeCore", + "MetadataRecorder", + "api_connection", + "api_status", + "attempt", + "attempt_sync", + "bad_gateway", + "cooled_down", + "fresh", + "headers_of", + "http_response", + "not_found", + "permission_denied", + "plain_error", + "rate_limited", + "service_unavailable", + "unprocessable", +] diff --git a/tests/unit/test_router/core/rust_parity_pending.txt b/tests/unit/test_router/core/rust_parity_pending.txt new file mode 100644 index 00000000000..8cfcebad287 --- /dev/null +++ b/tests/unit/test_router/core/rust_parity_pending.txt @@ -0,0 +1,2 @@ +# Characterization tests the Rust core does not match yet, one node id per line (`file.py::test[rust]`). +# conftest.py marks each one xfail(strict=True), so a test that starts passing must leave the list. diff --git a/tests/unit/test_router/core/scripted_host.py b/tests/unit/test_router/core/scripted_host.py index af28160b620..f673ab98610 100644 --- a/tests/unit/test_router/core/scripted_host.py +++ b/tests/unit/test_router/core/scripted_host.py @@ -18,6 +18,7 @@ from typing import Final import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager from litellm.router_config import RouterDynamicConfig, build_router_config from litellm.router_core import RouterCore @@ -138,6 +139,12 @@ def server_error(model: str = "gpt-4o") -> litellm.InternalServerError: return litellm.InternalServerError(message="boom", llm_provider="openai", model=model) +def agentic_followup_failure(error: Exception) -> Exception: + """``error`` as an agentic follow-up raises it, after the first call was served.""" + mark_agentic_followup_failure(error, ()) + return error + + def timeout(model: str = "gpt-4o") -> litellm.Timeout: return litellm.Timeout(message="timed out", llm_provider="openai", model=model) @@ -324,6 +331,9 @@ class ScriptedHost(PythonHost): return done() + def sleep_sync(self, seconds: float) -> None: + self.sleeps.append(seconds) + def dynamic(self) -> RouterDynamicConfig: return self._dynamic @@ -398,9 +408,9 @@ class ScriptedHost(PythonHost): return outcome, None return None, _default_response(op, call.model) if outcome is None else outcome - async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: - call, outcome = self._outcome(op, args, kwargs) - error, response = self._finish(op, call, outcome) + async def log_async(self, call: Call, error: BaseException | None, response: object) -> None: + """Run the callbacks an async call's logging runs: the failure callbacks for ``error``, else + the async success callbacks for ``response``.""" logged: Final = self._logging_kwargs(call, error, response) if error is not None: for callback in list(self.lists.get("failure_callback", ())): @@ -409,24 +419,50 @@ class ScriptedHost(PythonHost): for callback in list(self.lists.get("_async_failure_callback", ())): if callable(callback): await callback(logged, None, 0.0, 0.0) - raise error + return for callback in list(self.lists.get("_async_success_callback", ())): if callable(callback): await callback(logged, response, 0.0, 0.0) - return response - def _answer_sync(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: - call, outcome = self._outcome(op, args, kwargs) - error, response = self._finish(op, call, outcome) + def log_sync(self, call: Call, error: BaseException | None, response: object) -> None: + """``log_async`` for a sync call: the failure callbacks, else the sync success callbacks.""" logged: Final = self._logging_kwargs(call, error, response) if error is not None: for callback in list(self.lists.get("failure_callback", ())): if callable(callback): callback(logged, None, 0.0, 0.0) - raise error + return for callback in list(self.lists.get("success_callback", ())): if callable(callback): callback(logged, response, 0.0, 0.0) + + def _logs_at_stream_end(self, call: Call, response: object) -> bool: + """A stream whose logging object logs through this host (``streaming_helpers.HostLogging``) + logs when it ends, as litellm's stream logging does, so the call logs nothing when it opens.""" + bind: Final[object] = getattr(getattr(response, "logging_obj", None), "bind_host", None) + if not callable(bind): + return False + bind(self, call) + return True + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + call, outcome = self._outcome(op, args, kwargs) + error, response = self._finish(op, call, outcome) + if error is not None: + await self.log_async(call, error, response) + raise error + if not self._logs_at_stream_end(call, response): + await self.log_async(call, None, response) + return response + + def _answer_sync(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + call, outcome = self._outcome(op, args, kwargs) + error, response = self._finish(op, call, outcome) + if error is not None: + self.log_sync(call, error, response) + raise error + if not self._logs_at_stream_end(call, response): + self.log_sync(call, None, response) return response @@ -452,25 +488,46 @@ def python_core(host: ScriptedHost, **arguments: object) -> PythonCore: # kwarg return PythonCore(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the same arguments Router binds -CoreBuilder = Callable[..., RouterCore] -"""``builder(host, **router_arguments)``: a router core on ``host``.""" - -CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core} -"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a -configuration raises ``CoreDeclined``, and the test is skipped for that core.""" - - class CoreDeclined(Exception): """The core does not serve this configuration (the router would stay on the Python core).""" +CoreBuilder = Callable[..., RouterCore] +"""``builder(host, **router_arguments)``: a router core on ``host``.""" + + +def rust_core(host: ScriptedHost, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + """A ``RustCore`` on ``host``, built the way ``make_core`` builds one when the Rust router is on.""" + from litellm.router_rust.core import RustCore + from litellm.rust_bridge._native import RustBridgeDeclined + + config: Final = build_router_config(arguments) # an invalid config raises here, as for the Python core + try: + return RustCore.build(config, host) + except RustBridgeDeclined as declined: # make_core keeps such a router on the Python core + raise CoreDeclined(str(declined)) from declined + + +CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core, "rust": rust_core} + + +def records_outcomes_through_callbacks(core: RouterCore) -> bool: + """Whether ``core`` learns attempt outcomes from callbacks it registers on its host. The Python + core does; the Rust core records every outcome from the attempt itself and registers none.""" + return isinstance(core, PythonCore) + + +"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a +configuration raises ``CoreDeclined``, and the test is skipped for that core.""" + + __all__ = [ - "Call", - "HostCallbacks", - "INFERENCE_OPS", "CORE_BUILDERS", + "INFERENCE_OPS", + "Call", "CoreBuilder", "CoreDeclined", + "HostCallbacks", "ScriptedHost", "auth_error", "bad_request", @@ -482,6 +539,7 @@ __all__ = [ "embedding_response", "python_core", "rate_limit", + "rust_core", "server_error", "timeout", ] diff --git a/tests/unit/test_router/core/selection_helpers.py b/tests/unit/test_router/core/selection_helpers.py new file mode 100644 index 00000000000..b38cfec1a19 --- /dev/null +++ b/tests/unit/test_router/core/selection_helpers.py @@ -0,0 +1,88 @@ +""" +Helpers for the selection characterization tests. + +``TrafficHost`` is a ``ScriptedHost`` whose async inference calls also feed the routing strategy +loggers the core registered on the host, the way litellm's logging does around a real call: input +callbacks see the call start (``log_pre_api_call``), and the ``CustomLogger`` instances in the +``callbacks`` list see it succeed or fail with real start and end times. A call can be held open +on a gate (an ``asyncio.Event``) or delayed for real, so in-flight counts and latencies are real +for any core. +""" + +import asyncio +import datetime +from collections.abc import Callable, Mapping +from typing import Final + +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import Call, ScriptedHost + +MakeCore = Callable[..., RouterCore] + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +class TrafficHost(ScriptedHost): + """``ScriptedHost`` plus the strategy-logger traffic litellm's logging produces around a call.""" + + def __init__(self) -> None: + super().__init__() + self.gates: dict[str, asyncio.Event] = {} + self.delays: dict[str, float] = {} + self.started: list[str] = [] # mutable-ok: the models whose calls have started, in order + + def _loggers(self, name: str) -> tuple[CustomLogger, ...]: + return tuple(callback for callback in self.lists.get(name, ()) if isinstance(callback, CustomLogger)) + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + model: Final = str(kwargs.get("model") or (args[0] if args else "")) + logged: Final = self._logging_kwargs(Call(op=op, model=model, kwargs=dict(kwargs)), None, None) + for logger in self._loggers("input_callback"): + logger.log_pre_api_call(model=model, messages=kwargs.get("messages"), kwargs=logged) + self.started.append(model) + start: Final = datetime.datetime.now() + gate: Final = self.gates.get(model) + if gate is not None: + await gate.wait() + delay: Final = self.delays.get(model) + if delay: + await asyncio.sleep(delay) + try: + response = await super()._answer_async(op, args, kwargs) + except Exception as error: + failed = {**logged, "exception": error} + for logger in self._loggers("callbacks"): + await logger.async_log_failure_event(failed, None, start, datetime.datetime.now()) + raise + succeeded: Final = self._logging_kwargs(Call(op=op, model=model, kwargs=dict(kwargs)), None, response) + for logger in self._loggers("callbacks"): + await logger.async_log_success_event(succeeded, response, start, datetime.datetime.now()) + return response + + +async def picked_ids( + core: RouterCore, + model: str, + times: int = 20, + request_kwargs: Mapping[str, object] | None = None, + **select_arguments: object, # kwargs-ok: select's keyword options +) -> set[str]: + """The deployment ids ``select`` picks over ``times`` picks, each with a fresh copy of ``request_kwargs``.""" + picked: Final[set[str]] = set() + for _ in range(times): + chosen = await core.select(model, dict(request_kwargs or {}), **select_arguments) # pyright: ignore[reportCallIssue] # PythonCore's select takes keyword options + assert isinstance(chosen, Mapping) + picked.add(str(chosen["model_info"]["id"])) + return picked + + +def ids_of(deployments: object) -> list[str]: # mutable-ok: a fresh list + """The ids of a ``select(kind="healthy")`` result, in order (a single dict gives one id).""" + if isinstance(deployments, Mapping): + return [str(deployments["model_info"]["id"])] + assert isinstance(deployments, list) + return [str(deployment["model_info"]["id"]) for deployment in deployments] + + +__all__ = ["MESSAGES", "MakeCore", "TrafficHost", "ids_of", "picked_ids"] diff --git a/tests/unit/test_router/core/streaming_helpers.py b/tests/unit/test_router/core/streaming_helpers.py new file mode 100644 index 00000000000..3e436103e60 --- /dev/null +++ b/tests/unit/test_router/core/streaming_helpers.py @@ -0,0 +1,474 @@ +""" +Scripted streams for the streaming characterization tests. + +``ScriptedHost`` outcomes may be callables, so each builder here returns a callable that the host +calls with the request kwargs and that answers with a fresh stream: + +- ``chat_stream``: a ``CustomStreamWrapper`` over chat chunks, optionally raising ``error`` after + ``fail_after`` chunks, and ending with a ``finish_reason`` chunk unless ``finish`` is False. +- ``anthropic_stream``: an async iterator of raw ``/v1/messages`` SSE byte frames, optionally raising. +- ``responses_stream``: a ``BaseResponsesAPIStreamingIterator`` over Responses API events, + optionally raising. + +A stream built with ``logged=True`` logs through the ``ScriptedHost`` that served it when it ends +(``HostLogging``), as litellm's stream logging does, rather than when the call returns it. +""" + +import datetime +import json +import sys +import types +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.types.utils import ModelResponseStream + +if TYPE_CHECKING: + from tests.unit.test_router.core.scripted_host import Call, ScriptedHost + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +def install_orjson_stub_if_missing() -> None: + """ + The /v1/messages stream wrapper imports proxy modules that import ``orjson`` and ``soundfile`` at load + time. Where either is not installed (some dev containers), a stand-in is registered so the wrapper can + load (json-backed for ``orjson``, empty for ``soundfile``, which only audio passthrough logging uses); + where they are installed (CI), nothing changes. + """ + try: + import orjson # noqa: F401 # pyright: ignore[reportUnusedImport] # only probing for the module + except ImportError: + stub: Final = types.ModuleType("orjson") + stub.loads = json.loads # pyright: ignore[reportAttributeAccessIssue] # module attribute + stub.dumps = lambda value, **_options: json.dumps(value).encode() # pyright: ignore[reportAttributeAccessIssue] # module attribute + stub.JSONDecodeError = json.JSONDecodeError # pyright: ignore[reportAttributeAccessIssue] # module attribute + sys.modules["orjson"] = stub + try: + import soundfile # noqa: F401 # pyright: ignore[reportUnusedImport, reportMissingImports] # only probing for the module + except (ImportError, OSError): + sys.modules["soundfile"] = types.ModuleType("soundfile") + + +def _logging_obj(model: str, call_type: str = "acompletion", logged: bool = False) -> LiteLLMLogging: + logging_obj: Final = (HostLogging if logged else LiteLLMLogging)( + model=model, + messages=MESSAGES, + stream=True, + call_type=call_type, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id=f"stream-{model}", + function_id=f"stream-{model}", + ) + logging_obj.update_environment_variables(model=model, user=None, optional_params={}, litellm_params={}) + return logging_obj + + +def request_logging(model: str = "a") -> LiteLLMLogging: + """The logging object a request shares across its attempts (``litellm_logging_obj``).""" + return _logging_obj(model) + + +class HostLogging(LiteLLMLogging): + """A stream's logging object that runs the serving ``ScriptedHost``'s callbacks where litellm's + stream logging runs litellm's: once with the assembled response when the stream ends, and once + per failure the stream raises (``dispatch_attempt_failure_handlers``). The host binds itself and + the call when it returns the stream (``ScriptedHost._logs_at_stream_end``).""" + + def bind_host(self, host: "ScriptedHost", call: "Call") -> None: + self._scripted: tuple[ScriptedHost, Call] | None = (host, call) # pyright: ignore[reportUninitializedInstanceVariable] # set when the host binds + self._logged_success = False + + def _bound(self) -> "tuple[ScriptedHost, Call] | None": + return getattr(self, "_scripted", None) + + async def dispatch_success_handlers( # pyright: ignore[reportIncompatibleMethodOverride] # the arguments the stream passes + self, result: object = None, *_args: object, **_kwargs: object + ) -> None: + bound: Final = self._bound() + if bound is None or self._logged_success: + return + self._logged_success = True + await bound[0].log_async(bound[1], None, result) + + def dispatch_attempt_failure_handlers( + self, exception: Exception, traceback_exception: str + ) -> Coroutine[None, None, None]: + bound: Final = self._bound() + + async def dispatch() -> None: + if bound is not None: + await bound[0].log_async(bound[1], exception, None) + + return dispatch() + + def success_handler(self, result: object = None, *_args: object, **_kwargs: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # see dispatch_success_handlers + bound: Final = self._bound() + if bound is None or self._logged_success or not getattr(result, "usage", None): + return + self._logged_success = True + bound[0].log_sync(bound[1], None, result) + + def failure_handler(self, exception: Exception, *_args: object, **_kwargs: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # see dispatch_success_handlers + bound: Final = self._bound() + if bound is not None: + bound[0].log_sync(bound[1], exception, None) + + +class _EndLogged: + """What a hand-written scripted stream logs once when it ends: its failure, or a response carrying + ``total_tokens`` (the usage the provider's own stream logging would assemble).""" + + def __init__(self, model: str, call_type: str, logged: bool, total_tokens: int) -> None: + self.logging_obj: Final[LiteLLMLogging | None] = _logging_obj(model, call_type, True) if logged else None + self._total_tokens: Final = total_tokens + self._logged = False + + async def log_end(self, error: BaseException | None) -> None: + if not isinstance(self.logging_obj, HostLogging) or self._logged: + return + self._logged = True + if error is not None: + # The provider's own error, as a stream logs it before wrapping it for the router. + original: Final = getattr(error, "original_exception", None) + logged: Final = original if isinstance(original, Exception) else error + await self.logging_obj.dispatch_attempt_failure_handlers( + logged if isinstance(logged, Exception) else Exception(logged), "" + ) + return + usage: Final = types.SimpleNamespace(total_tokens=self._total_tokens) + await self.logging_obj.dispatch_success_handlers(types.SimpleNamespace(usage=usage)) + + +def text_chunk(text: str) -> ModelResponseStream: + return ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) + + +def role_chunk() -> ModelResponseStream: + return ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": ""}}]) + + +def finish_chunk(reason: str = "stop", usage: Mapping[str, int] | None = None) -> ModelResponseStream: + chunk: Final = ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": reason}]) + if usage is not None: + setattr(chunk, "usage", dict(usage)) + return chunk + + +def chat_stream( + *texts: str, + error: BaseException | None = None, + fail_after: int | None = None, + finish: bool = True, + leading_role_chunk: bool = False, + model: str = "d1", + hidden_params: Mapping[str, object] | None = None, + sync: bool = False, + usage: Mapping[str, int] | None = None, + logged: bool = False, +) -> Callable[..., CustomStreamWrapper]: + """ + A stream of ``texts`` (one chunk each). With ``error``, raises it after ``fail_after`` chunks + (default: after every text chunk, before the finish chunk). ``leading_role_chunk`` sends a + role-only, empty-content chunk first, which counts as the first chunk but carries no content. + ``usage`` rides on the finish chunk; ``logged`` logs through the host when the stream ends. + """ + + def answer(**request: object) -> CustomStreamWrapper: # kwargs-ok: the host passes the request kwargs + produced: list[ModelResponseStream] = [] + if leading_role_chunk: + produced.append(role_chunk()) + produced.extend(text_chunk(text) for text in texts) + cut_at: Final = len(produced) if fail_after is None else fail_after + + def items() -> Iterator[ModelResponseStream]: + for index, chunk in enumerate(produced): + if error is not None and index == cut_at: + raise error + yield chunk + if error is not None: + raise error + if finish: + yield finish_chunk(usage=usage) + + async def achunks() -> AsyncIterator[ModelResponseStream]: + for chunk in items(): + yield chunk + + wrapper: Final = CustomStreamWrapper( + completion_stream=items() if sync else achunks(), + model=model, + custom_llm_provider="openai", + logging_obj=_logging_obj(model, "completion" if sync else "acompletion", logged), + ) + if hidden_params is not None: + wrapper._hidden_params.update(hidden_params) # pyright: ignore[reportPrivateUsage] # test seam + return wrapper + + return answer + + +async def collect_texts(stream: object) -> str: + """The text content of every chunk of an async chat stream, joined.""" + texts: Final[list[str]] = [] + async for chunk in stream: # pyright: ignore[reportGeneralTypeIssues] # an async stream + choices = getattr(chunk, "choices", None) + if choices: + content = choices[0].delta.content + if content: + texts.append(content) + return "".join(texts) + + +def collect_texts_sync(stream: object) -> str: + texts: Final[list[str]] = [] + for chunk in stream: # pyright: ignore[reportGeneralTypeIssues] # a sync stream + choices = getattr(chunk, "choices", None) + if choices: + content = choices[0].delta.content + if content: + texts.append(content) + return "".join(texts) + + +# --- /v1/messages ------------------------------------------------------------------------------ + + +def sse(event: str, data: Mapping[str, object]) -> bytes: + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + +MESSAGE_START: Final = sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "content": []}}, +) +BLOCK_START: Final = sse( + "content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}} +) +PING: Final = sse("ping", {"type": "ping"}) +MESSAGE_STOP: Final = sse("message_stop", {"type": "message_stop"}) + + +def text_delta(text: str) -> bytes: + return sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + ) + + +def error_frame(error_type: str = "overloaded_error", message: str = "Overloaded") -> bytes: + return sse("error", {"type": "error", "error": {"type": error_type, "message": message}}) + + +def refusal_delta(stop_details: Mapping[str, object] | None = None) -> bytes: + delta: Final[dict[str, object]] = {"stop_reason": "refusal"} + if stop_details is not None: + delta["stop_details"] = dict(stop_details) + return sse("message_delta", {"type": "message_delta", "delta": delta, "usage": {"output_tokens": 1}}) + + +class ByteStream: + """An async iterator of SSE frames that records whether it was closed. ``logged`` logs through the host + when it ends, with ``total_tokens`` (``_EndLogged``).""" + + def __init__( + self, + frames: Sequence[bytes], + error: BaseException | None, + hidden_params: Mapping[str, object] | None, + logged: bool = False, + total_tokens: int = 0, + ) -> None: + self._frames: Final = list(frames) + self._error: Final = error + self._index = 0 + self.closed = False + self._end: Final = _EndLogged("d1", "anthropic_messages", logged, total_tokens) + if logged: + self.logging_obj = self._end.logging_obj + if hidden_params is not None: + self._hidden_params = dict(hidden_params) + + def __aiter__(self) -> "ByteStream": + return self + + async def __anext__(self) -> bytes: + if self._index < len(self._frames): + self._index += 1 + return self._frames[self._index - 1] + await self._end.log_end(self._error) + if self._error is not None: + raise self._error + raise StopAsyncIteration + + async def aclose(self) -> None: + self.closed = True + + +def anthropic_stream( + *frames: bytes, + error: BaseException | None = None, + hidden_params: Mapping[str, object] | None = None, + opened: list[ByteStream] | None = None, + logged: bool = False, + total_tokens: int = 0, +) -> Callable[..., ByteStream]: + def answer(**_request: object) -> ByteStream: # kwargs-ok: the host passes the request kwargs + stream: Final = ByteStream(frames, error, hidden_params, logged, total_tokens) + if opened is not None: + opened.append(stream) + return stream + + return answer + + +async def collect_bytes(stream: object) -> list[bytes]: + return [frame async for frame in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + + +# --- Responses API ----------------------------------------------------------------------------- + + +class Event: + """A Responses API stream event: only ``type`` (and optionally ``response``) is read.""" + + def __init__(self, event_type: str, response: object = None, delta: str | None = None) -> None: + self.type = event_type + self.response = response + self.delta = delta + self._hidden_params: dict[str, object] = {} + + def __repr__(self) -> str: + return f"Event({self.type!r})" + + +class BackgroundResponse: + background = True + id = "resp_bg" + + +class ScriptedResponsesStream(BaseResponsesAPIStreamingIterator): + """A Responses stream over ``events`` that bypasses the provider plumbing of the base class.""" + + def __init__( # pyright: ignore[reportMissingSuperCall] # the base needs an httpx response + self, + events: Sequence[object], + error: BaseException | None, + model: str, + custom_llm_provider: str, + hidden_params: Mapping[str, object] | None, + started_tool_call: bool = False, + completed_response: object = None, + logged: bool = False, + total_tokens: int = 0, + ) -> None: + self._end: Final = _EndLogged(model, "aresponses", logged, total_tokens) + self._events: Final = list(events) + self._started_tool_call = started_tool_call + self._error: Final = error + self._index = 0 + self._hidden_params = dict(hidden_params or {}) + self.model = model + self.custom_llm_provider = custom_llm_provider + self.logging_obj = self._end.logging_obj or _logging_obj(model, "aresponses") + self.litellm_metadata = None + self.responses_api_provider_config = None + self.finished = False + self.completed_response = completed_response + self.response = None + self.start_time = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + self.request_data = {} + self.call_type = None + self.closed = False + + def __aiter__(self) -> "ScriptedResponsesStream": + return self + + async def __anext__(self) -> object: + if self._index < len(self._events): + self._index += 1 + return self._events[self._index - 1] + await self._end.log_end(self._error) + if self._error is not None: + raise self._error + raise StopAsyncIteration + + async def aclose(self) -> None: + self.closed = True + + +def responses_stream( + *events: object, + error: BaseException | None = None, + model: str = "d1", + custom_llm_provider: str = "anthropic", + hidden_params: Mapping[str, object] | None = None, + opened: list[ScriptedResponsesStream] | None = None, + started_tool_call: bool = False, + completed_response: object = None, + logged: bool = False, + total_tokens: int = 0, +) -> Callable[..., ScriptedResponsesStream]: + """``completed_response`` is the terminal event the stream latched before failing, if any. ``logged`` + logs through the host when the stream ends, with ``total_tokens``.""" + + def answer(**_request: object) -> ScriptedResponsesStream: # kwargs-ok: the host passes the request kwargs + stream: Final = ScriptedResponsesStream( + events, + error, + model, + custom_llm_provider, + hidden_params, + started_tool_call, + completed_response, + logged, + total_tokens, + ) + if opened is not None: + opened.append(stream) + return stream + + return answer + + +async def drain_into(stream: object, seen: list[object]) -> None: + """Append every item of an async stream to ``seen``, letting an error the stream raises propagate.""" + async for item in stream: # pyright: ignore[reportGeneralTypeIssues] # an async stream + seen.append(item) + + +async def collect_event_types(stream: object) -> list[str]: + return [getattr(event, "type", repr(event)) async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + + +__all__ = [ + "BLOCK_START", + "MESSAGES", + "MESSAGE_START", + "MESSAGE_STOP", + "PING", + "BackgroundResponse", + "ByteStream", + "Event", + "HostLogging", + "ScriptedResponsesStream", + "anthropic_stream", + "chat_stream", + "collect_bytes", + "collect_event_types", + "collect_texts", + "collect_texts_sync", + "drain_into", + "error_frame", + "finish_chunk", + "install_orjson_stub_if_missing", + "refusal_delta", + "responses_stream", + "role_chunk", + "sse", + "text_chunk", + "text_delta", +] diff --git a/tests/unit/test_router/core/test_construction_precall_characterization.py b/tests/unit/test_router/core/test_construction_precall_characterization.py new file mode 100644 index 00000000000..a56cd78ccac --- /dev/null +++ b/tests/unit/test_router/core/test_construction_precall_characterization.py @@ -0,0 +1,1302 @@ +""" +Characterization of a router core's construction and lifecycle: what each constructor argument sets +(and the process settings it falls back to), the validation errors, the caches and callbacks it +creates, alerting, optional pre-call check registration, ``observe``, ``register_plugin``, +``retire``, ``flush_cache``/``reset``, attribute writes, ``get_valid_args`` and ``print_deployment``. + +Each test asserts what the Python core does today; oddities are named in the test's docstring. +""" + +import logging +from collections.abc import Callable, Generator, Mapping +from typing import Final + +import pytest + +import litellm +from litellm._logging import verbose_router_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCache +from litellm.router_api import ChangeSet, router_init_args +from litellm.router_core import RouterCore +from litellm.router_python_core import PythonCore +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck +from litellm.types.router import ( + AlertingConfig, + AllowedFailsPolicy, + CustomRoutingStrategyBase, + RetryPolicy, + RoutingStrategy, +) +from tests.unit.test_router.core.construction_precall_helpers import ( + CallbackHost, + collect_other_routers, + host_callback_names, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, records_outcomes_through_callbacks + +MakeCore = Callable[..., RouterCore] +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +@pytest.fixture +def host() -> ScriptedHost: + return CallbackHost() + + +def _one(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: constructor arguments + return make_core(model_list=[deployment("g", "a")], **arguments) + + +# --- defaults -------------------------------------------------------------------------------- + +SETTING_DEFAULTS: Final = { + "routing_strategy": "simple-shuffle", + "fallbacks": None, + "context_window_fallbacks": None, + "content_policy_fallbacks": None, + "default_deployment": None, + "default_max_parallel_requests": None, + "cooldown_time": 5, + "allowed_fails": 3, + "allowed_fails_policy": None, + "disable_cooldowns": None, + "enable_pre_call_checks": False, + "enable_health_check_routing": False, + "health_check_ignore_transient_errors": False, + "background_health_check_model_groups": None, + "enable_tag_filtering": False, + "tag_filtering_match_any": True, + "tag_routing_prefix": "", + "provider_budget_config": None, + "provider_default_deployment_ids": [], + "fallback_access_check": None, + "fallback_budget_check": None, + "search_tools": [], + "guardrail_list": [], + "cache_responses": False, +} + + +@pytest.mark.parametrize(("field", "expected"), sorted(SETTING_DEFAULTS.items())) +def test_settings_default_of_a_router_built_with_only_a_model_list( + make_core: MakeCore, field: str, expected: object +) -> None: + """Note: ``fallbacks`` defaults to ``[]`` in the signature, yet reads ``None``: an empty list falls + through ``fallbacks or ``.""" + assert getattr(_one(make_core).settings(), field) == expected + + +def test_default_litellm_params_default_to_no_timeout_no_client_retries_and_no_caching_groups( + make_core: MakeCore, +) -> None: + assert _one(make_core).settings().default_litellm_params == { + "timeout": None, + "max_retries": 0, + "metadata": {"caching_groups": None}, + } + + +def test_router_general_settings_default_to_async_only_and_pass_through_off(make_core: MakeCore) -> None: + general: Final = _one(make_core).settings().router_general_settings + assert (general.async_only_mode, general.pass_through_all_models) == (False, False) + + +def test_settings_as_dict_of_a_default_router(make_core: MakeCore) -> None: + assert dict(_one(make_core).settings().as_dict()) == { + "routing_strategy_args": {}, + "routing_strategy": "simple-shuffle", + "allowed_fails": 3, + "cooldown_time": 5, + "num_retries": 2, + "timeout": 6000.0, + "retry_after": 0, + "fallbacks": None, + "context_window_fallbacks": None, + "model_group_retry_policy": {}, + "retry_policy": None, + "model_group_alias": {}, + "enable_weighted_failover": False, + "enable_mid_stream_fallback_continuation": False, + "enable_tag_filtering": False, + "tag_routing_prefix": "", + "fallback_on_unsupported_params": True, + "routing_groups": [], + } + + +ATTRIBUTE_DEFAULTS: Final = { + "num_retries": 2, + "max_fallbacks": 5, + "timeout": 6000.0, + "stream_timeout": None, + "request_timeout": None, + "retry_after": 0, + "client_ttl": 3600, + "set_verbose": False, + "debug_level": "INFO", + "retry_policy": None, + "model_group_retry_policy": {}, + "alerting_config": None, + "routing_strategy_args": {}, + "deployment_affinity_ttl_seconds": 3600, + "default_priority": None, + "optional_callbacks": None, + "router_budget_logger": None, + "fallback_on_unsupported_params": True, + "enable_weighted_failover": False, + "enable_mid_stream_fallback_continuation": False, + "explicit_allowed_fails": None, + "ignore_invalid_deployments": False, + "auto_router_capability_limit": None, + "assistants_config": None, + "session_affinity_repin_on_failover": False, + "model_group_affinity_config": None, + "access_groups": None, + "leastbusy_logger": None, + "lowesttpm_logger": None, +} + + +@pytest.mark.parametrize(("name", "expected"), sorted(ATTRIBUTE_DEFAULTS.items())) +def test_attribute_default_of_a_router_built_with_only_a_model_list( + make_core: MakeCore, name: str, expected: object +) -> None: + assert getattr(_one(make_core), name) == expected + + +def test_a_router_without_a_model_list_has_no_deployments(make_core: MakeCore) -> None: + core: Final = make_core() + assert list(core.registry().all()) == [] + + +def test_construction_turns_off_the_host_debug_info(make_core: MakeCore, host: ScriptedHost) -> None: + _one(make_core) + assert host.suppressed is True + + +def test_model_group_alias_none_reads_as_an_empty_alias_map(make_core: MakeCore) -> None: + core: Final = _one(make_core, model_group_alias=None) + assert dict(core.settings().as_dict()["model_group_alias"]) == {} # pyright: ignore[reportArgumentType] + + +# --- process settings each argument falls back to ----------------------------------------- + + +@pytest.mark.parametrize(("argument", "host_value", "expected"), [(5, 4, 5), (0, 4, 0), (None, 4, 4), (None, None, 2)]) +def test_num_retries_comes_from_the_argument_then_the_process_then_the_openai_default( + make_core: MakeCore, host: ScriptedHost, argument: int | None, host_value: int | None, expected: int +) -> None: + host.set_dynamic(num_retries=host_value) + core: Final = _one(make_core, **({"num_retries": argument} if argument is not None else {})) + assert core.settings().as_dict()["num_retries"] == expected + + +@pytest.mark.asyncio +async def test_num_retries_from_the_process_settings_governs_the_attempts( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(num_retries=1) + host.answer("openai/a", litellm.InternalServerError(message="boom", llm_provider="openai", model="a")) + core: Final = _one(make_core) + with pytest.raises(litellm.InternalServerError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert len(host.calls) == 2 + + +@pytest.mark.parametrize( + ("argument", "host_max", "host_router_max", "expected"), + [(3, 9, 11, 3), (0, 9, 11, 0), (None, 9, 11, 9), (None, None, 11, 11)], +) +def test_max_fallbacks_comes_from_the_argument_then_litellm_max_fallbacks_then_router_max_fallbacks( + make_core: MakeCore, + host: ScriptedHost, + argument: int | None, + host_max: int | None, + host_router_max: int, + expected: int, +) -> None: + host.set_dynamic(max_fallbacks=host_max, router_max_fallbacks=host_router_max) + core: Final = _one(make_core, **({"max_fallbacks": argument} if argument is not None else {})) + assert core.max_fallbacks == expected # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize(("argument", "expected", "explicit"), [(7, 7, 7), (0, 0, 0), (None, 8, None)]) +def test_allowed_fails_comes_from_the_argument_then_the_process_and_only_an_argument_counts_as_explicit( + make_core: MakeCore, host: ScriptedHost, argument: int | None, expected: int, explicit: int | None +) -> None: + host.set_dynamic(allowed_fails=8) + core: Final = _one(make_core, **({"allowed_fails": argument} if argument is not None else {})) + assert core.settings().allowed_fails == expected + assert core.explicit_allowed_fails == explicit # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize(("argument", "expected"), [(12.5, 12.5), (None, 33.0), (0, 33.0)]) +def test_timeout_comes_from_the_argument_else_the_process_request_timeout( + make_core: MakeCore, host: ScriptedHost, argument: float | None, expected: float +) -> None: + """Oddity: ``timeout=0`` is falsy and reads as the process ``request_timeout``.""" + host.set_dynamic(request_timeout=33.0) + core: Final = _one(make_core, **({"timeout": argument} if argument is not None else {})) + assert core.settings().as_dict()["timeout"] == expected + + +def test_timeout_zero_still_reaches_default_litellm_params_as_zero(make_core: MakeCore) -> None: + core: Final = _one(make_core, timeout=0) + assert core.settings().default_litellm_params["timeout"] == 0 + + +@pytest.mark.parametrize(("argument", "expected"), [(30, 30), (0.5, 0.5), (0, 5), (None, 5)]) +def test_cooldown_time_defaults_to_five_seconds_and_zero_reads_as_the_default( + make_core: MakeCore, argument: float | None, expected: float +) -> None: + """Oddity: ``cooldown_time=0`` is replaced by the 5 second default (``cooldown_time or DEFAULT``).""" + core: Final = _one(make_core, **({"cooldown_time": argument} if argument is not None else {})) + assert core.settings().cooldown_time == expected + + +def test_request_timeout_is_unset_without_a_router_timeout( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "request_timeout", 77.0) + assert _one(make_core).request_timeout is None # pyright: ignore[reportAttributeAccessIssue] + + +def test_request_timeout_reads_litellm_request_timeout_directly_when_a_router_timeout_is_set( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + """Oddity: this one setting is read from ``litellm.request_timeout`` itself, not from the host's + process settings (which say 33 here).""" + host.set_dynamic(request_timeout=33.0) + monkeypatch.setattr(litellm, "request_timeout", 77.0) + assert _one(make_core, timeout=10).request_timeout == 77.0 # pyright: ignore[reportAttributeAccessIssue] + + +def test_request_timeout_is_unset_when_litellm_request_timeout_is_its_untouched_default( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "request_timeout", 6000) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False, raising=False) + assert _one(make_core, timeout=10).request_timeout is None # pyright: ignore[reportAttributeAccessIssue] + + +def test_stream_timeout_is_kept_as_given(make_core: MakeCore) -> None: + assert _one(make_core, stream_timeout=3).stream_timeout == 3 # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize("field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) +def test_fallback_lists_come_from_the_process_when_the_argument_is_empty( + make_core: MakeCore, host: ScriptedHost, field: str +) -> None: + host.set_dynamic(**{field: [{"g": ["process"]}]}) + assert getattr(_one(make_core).settings(), field) == [{"g": ["process"]}] + + +@pytest.mark.parametrize("field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) +def test_fallback_list_arguments_win_over_the_process_lists( + make_core: MakeCore, host: ScriptedHost, field: str +) -> None: + host.set_dynamic(**{field: [{"g": ["process"]}]}) + assert getattr(_one(make_core, **{field: [{"g": ["argument"]}]}).settings(), field) == [{"g": ["argument"]}] + + +def test_default_fallbacks_are_appended_to_the_fallbacks_as_a_star_entry(make_core: MakeCore) -> None: + core: Final = _one(make_core, fallbacks=[{"g": ["b"]}], default_fallbacks=["c"]) + assert core.settings().fallbacks == [{"g": ["b"]}, {"*": ["c"]}] + + +def test_default_fallbacks_alone_become_the_only_star_entry(make_core: MakeCore) -> None: + assert _one(make_core, default_fallbacks=["c"]).settings().fallbacks == [{"*": ["c"]}] + + +def test_default_fallbacks_come_from_the_process_when_not_given(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(default_fallbacks=["d"]) + assert _one(make_core).settings().fallbacks == [{"*": ["d"]}] + + +def test_default_fallbacks_given_as_a_string_raise_type_error(make_core: MakeCore) -> None: + with pytest.raises(TypeError, match=r"^default_fallbacks must be a list of model names, got 'b'\. Use \['b'\]$"): + _one(make_core, default_fallbacks="b") + + +def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_itself( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: with no ``fallbacks`` argument the router adopts the process list object and appends the + star entry to it, so the process setting is changed too.""" + process_fallbacks: Final[list[object]] = [{"g": ["e"]}] + host.set_dynamic(fallbacks=process_fallbacks) + core: Final = _one(make_core, default_fallbacks=["c"]) + assert core.settings().fallbacks == [{"g": ["e"]}, {"*": ["c"]}] + assert process_fallbacks == [{"g": ["e"]}, {"*": ["c"]}] + + +# --- validation -------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("field", ["fallbacks", "content_policy_fallbacks"]) +def test_a_fallback_entry_that_is_not_a_dict_raises_value_error(make_core: MakeCore, field: str) -> None: + with pytest.raises(ValueError, match=r"^Item 'x' is not a dictionary\.$"): + _one(make_core, **{field: ["x"]}) + + +@pytest.mark.parametrize("field", ["fallbacks", "content_policy_fallbacks"]) +def test_a_fallback_entry_with_two_keys_raises_value_error(make_core: MakeCore, field: str) -> None: + with pytest.raises(ValueError, match=r"must have exactly one key, but has 2 keys\.$"): + _one(make_core, **{field: [{"a": ["b"], "c": ["d"]}]}) + + +@pytest.mark.parametrize("field", ["fallbacks", "content_policy_fallbacks"]) +def test_a_fallback_target_given_as_a_string_raises_type_error(make_core: MakeCore, field: str) -> None: + with pytest.raises(TypeError) as error: + _one(make_core, **{field: [{"a": "b"}]}) + assert str(error.value) == "Fallbacks for 'a' must be a list of model names, got 'b'. Use {'a': ['b']}" + + +@pytest.mark.parametrize("targets", [[], [{"model": "b"}], ["b", "c"]]) +def test_fallback_targets_as_a_list_of_names_or_dicts_are_accepted(make_core: MakeCore, targets: list[object]) -> None: + assert _one(make_core, fallbacks=[{"g": targets}]).settings().fallbacks == [{"g": targets}] + + +@pytest.mark.parametrize("value", [["x"], [{"a": "b"}]]) +def test_context_window_fallbacks_are_not_validated(make_core: MakeCore, value: list[object]) -> None: + """Oddity: unlike ``fallbacks`` and ``content_policy_fallbacks``, malformed context window + fallbacks are accepted as given.""" + assert _one(make_core, context_window_fallbacks=value).settings().context_window_fallbacks == value + + +def test_malformed_process_fallbacks_raise_at_construction(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(fallbacks=[{"a": "e"}]) + with pytest.raises(TypeError, match=r"^Fallbacks for 'a' must be a list of model names"): + _one(make_core) + + +def test_validate_fallbacks_accepts_none(make_core: MakeCore) -> None: + assert _one(make_core).validate_fallbacks(fallback_param=None) is None # pyright: ignore[reportAttributeAccessIssue] + + +def test_an_unknown_routing_strategy_raises_value_error(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match=r"^Invalid routing_strategy: 'bogus'\.") as error: + _one(make_core, routing_strategy="bogus") + assert str(error.value).startswith("Invalid routing_strategy: 'bogus'. Valid options: [") + assert str(error.value).endswith( + "Check 'router_settings.routing_strategy' in your config.yaml " + "or the 'routing_strategy' parameter if using the Router SDK directly." + ) + + +def test_a_routing_strategy_enum_is_stored_as_its_string_value(make_core: MakeCore) -> None: + assert _one(make_core, routing_strategy=RoutingStrategy.LEAST_BUSY).settings().routing_strategy == "least-busy" + + +@pytest.mark.parametrize( + "strategy", + [ + "simple-shuffle", + "least-busy", + "usage-based-routing", + "usage-based-routing-v2", + "latency-based-routing", + "cost-based-routing", + "lar1", + ], +) +def test_each_routing_strategy_is_reported_in_settings(make_core: MakeCore, strategy: str) -> None: + assert _one(make_core, routing_strategy=strategy).settings().routing_strategy == strategy + + +# --- default_litellm_params ---------------------------------------------------------------- + + +def test_default_litellm_params_keep_given_values_and_gain_max_retries_and_caching_groups(make_core: MakeCore) -> None: + core: Final = _one( + make_core, + timeout=12.5, + default_litellm_params={"timeout": 99, "temperature": 0.1, "metadata": {"x": 1}}, + caching_groups=[("a", "b")], + ) + assert core.settings().default_litellm_params == { + "timeout": 99, + "temperature": 0.1, + "metadata": {"x": 1, "caching_groups": [("a", "b")]}, + "max_retries": 0, + } + + +def test_default_litellm_params_timeout_defaults_to_the_router_timeout_argument(make_core: MakeCore) -> None: + assert _one(make_core, timeout=12.5).settings().default_litellm_params["timeout"] == 12.5 + + +def test_default_litellm_params_argument_is_not_changed_by_construction(make_core: MakeCore) -> None: + given: Final = {"temperature": 0.1} + _one(make_core, default_litellm_params=given) + assert given == {"temperature": 0.1} + + +# --- policies, priority, scheduler ------------------------------------------------------- + + +def test_retry_policy_given_as_a_dict_becomes_a_retry_policy(make_core: MakeCore) -> None: + policy: Final = _one(make_core, retry_policy={"RateLimitErrorRetries": 3}).retry_policy # pyright: ignore[reportAttributeAccessIssue] + assert isinstance(policy, RetryPolicy) + assert policy.RateLimitErrorRetries == 3 + + +def test_retry_policy_given_as_a_retry_policy_is_kept(make_core: MakeCore) -> None: + policy: Final = RetryPolicy(TimeoutErrorRetries=1) + assert _one(make_core, retry_policy=policy).retry_policy == policy # pyright: ignore[reportAttributeAccessIssue] + + +def test_retry_policy_of_another_type_is_ignored(make_core: MakeCore) -> None: + assert _one(make_core, retry_policy="nope").retry_policy is None # pyright: ignore[reportAttributeAccessIssue] + + +def test_allowed_fails_policy_given_as_a_dict_becomes_an_allowed_fails_policy(make_core: MakeCore) -> None: + policy: Final = ( + _one(make_core, allowed_fails_policy={"RateLimitErrorAllowedFails": 7}).settings().allowed_fails_policy + ) + assert isinstance(policy, AllowedFailsPolicy) + assert policy.RateLimitErrorAllowedFails == 7 + + +def test_allowed_fails_policy_of_another_type_is_ignored(make_core: MakeCore) -> None: + assert _one(make_core, allowed_fails_policy="nope").settings().allowed_fails_policy is None + + +def test_model_group_retry_policy_is_kept_as_given(make_core: MakeCore) -> None: + policy: Final = {"g": RetryPolicy(TimeoutErrorRetries=4)} + assert _one(make_core, model_group_retry_policy=policy).model_group_retry_policy == policy # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize( + ("given", "expected"), [(1, 1), ("2", 2), (-1, -1), (None, None), ("x", None), (1.5, None), (True, None)] +) +def test_default_priority_keeps_an_integer_and_ignores_anything_else( + make_core: MakeCore, given: object, expected: int | None +) -> None: + assert _one(make_core, default_priority=given).default_priority == expected # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize(("given", "expected"), [(None, 0.03), (0.5, 0.5)]) +def test_scheduler_polling_interval_defaults_to_three_hundredths_of_a_second( + make_core: MakeCore, given: float | None, expected: float +) -> None: + core: Final = _one(make_core, **({"polling_interval": given} if given is not None else {})) + assert core.scheduler.polling_interval == expected # pyright: ignore[reportAttributeAccessIssue] + + +def test_client_ttl_is_kept_as_given(make_core: MakeCore) -> None: + assert _one(make_core, client_ttl=10).client_ttl == 10 # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("routing_strategy_args", {"ttl": 5}), + ("deployment_affinity_ttl_seconds", 60), + ("session_affinity_repin_on_failover", True), + ("enable_weighted_failover", True), + ("fallback_on_unsupported_params", False), + ("ignore_invalid_deployments", True), + ("retry_after", 4), + ], +) +def test_plain_arguments_are_kept_as_attributes(make_core: MakeCore, name: str, value: object) -> None: + assert getattr(_one(make_core, **{name: value}), name) == value + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("enable_tag_filtering", True), + ("tag_filtering_match_any", False), + ("tag_routing_prefix", "team-"), + ("enable_pre_call_checks", True), + ("disable_cooldowns", True), + ("default_max_parallel_requests", 4), + ("enable_health_check_routing", True), + ("health_check_ignore_transient_errors", True), + ], +) +def test_setting_arguments_are_reported_in_settings(make_core: MakeCore, name: str, value: object) -> None: + assert getattr(_one(make_core, **{name: value}).settings(), name) == value + + +def test_background_health_check_model_groups_become_a_frozenset(make_core: MakeCore) -> None: + core: Final = _one(make_core, background_health_check_model_groups=["g", "g", "h"]) + assert core.settings().background_health_check_model_groups == frozenset({"g", "h"}) + + +@pytest.mark.parametrize(("given", "expected"), [(True, True), ("true", True), ("false", False), (False, False)]) +def test_mid_stream_fallback_continuation_accepts_a_bool_or_a_bool_string( + make_core: MakeCore, given: object, expected: bool +) -> None: + assert ( + _one(make_core, enable_mid_stream_fallback_continuation=given).enable_mid_stream_fallback_continuation + is expected + ) # pyright: ignore[reportAttributeAccessIssue] + + +# --- logging level ------------------------------------------------------------------------- + + +@pytest.fixture +def router_log_level() -> Generator[int, None, None]: + level: Final = verbose_router_logger.level + yield level + verbose_router_logger.setLevel(level) + + +@pytest.mark.parametrize( + ("set_verbose", "debug_level", "expected"), + [(True, "DEBUG", logging.DEBUG), (True, "INFO", logging.INFO)], +) +def test_set_verbose_sets_the_router_logger_to_the_debug_level( + make_core: MakeCore, router_log_level: object, set_verbose: bool, debug_level: str, expected: int +) -> None: + verbose_router_logger.setLevel(logging.WARNING) + _one(make_core, set_verbose=set_verbose, debug_level=debug_level) + assert verbose_router_logger.level == expected + + +def test_debug_level_without_set_verbose_leaves_the_router_logger_alone( + make_core: MakeCore, router_log_level: object +) -> None: + verbose_router_logger.setLevel(logging.WARNING) + _one(make_core, debug_level="DEBUG") + assert verbose_router_logger.level == logging.WARNING + + +# --- redis and response caching ---------------------------------------------------------- + + +def test_redis_host_and_port_give_the_routing_state_a_redis_cache(make_core: MakeCore) -> None: + """The port is passed on as a string.""" + core: Final = _one( + make_core, redis_host="localhost", redis_port=6399, redis_password="pw", cache_kwargs={"ssl": False} + ) + redis_cache: Final = core.health().store.redis_cache + assert isinstance(redis_cache, RedisCache) + assert {k: v for k, v in redis_cache.redis_kwargs.items() if k != "socket_timeout"} == { + "host": "localhost", + "port": "6399", + "password": "pw", + "ssl": False, + } + + +def test_redis_url_gives_the_routing_state_a_redis_cache(make_core: MakeCore) -> None: + redis_cache: Final = _one(make_core, redis_url="redis://localhost:6399").health().store.redis_cache + assert isinstance(redis_cache, RedisCache) + assert redis_cache.redis_kwargs["url"] == "redis://localhost:6399" + + +def test_redis_db_is_passed_on_as_a_string(make_core: MakeCore) -> None: + redis_cache: Final = _one(make_core, redis_host="localhost", redis_port=6399, redis_db=3).health().store.redis_cache + assert isinstance(redis_cache, RedisCache) + assert redis_cache.redis_kwargs["db"] == "3" + + +def test_redis_host_without_a_port_creates_no_redis_cache(make_core: MakeCore) -> None: + assert _one(make_core, redis_host="localhost").health().store.redis_cache is None + + +def test_without_redis_arguments_the_routing_state_is_in_memory_only(make_core: MakeCore) -> None: + assert _one(make_core).health().store.redis_cache is None + + +def test_the_routing_state_and_prompt_caching_pins_share_one_redis_cache(make_core: MakeCore) -> None: + core: Final = _one(make_core, redis_host="localhost", redis_port=6399) + assert core.registry().plugins("prompt_caching_pins").redis_cache is core.health().store.redis_cache + + +def test_applying_a_redis_cache_attaches_it_to_the_routing_state_and_the_pins(make_core: MakeCore) -> None: + core: Final = _one(make_core) + redis_cache: Final = RedisCache(host="localhost", port=6399) + core.apply(ChangeSet(redis_cache=redis_cache)) + assert core.health().store.redis_cache is redis_cache + assert core.registry().plugins("prompt_caching_pins").redis_cache is redis_cache + + +def test_cache_responses_installs_a_local_response_cache_on_the_host(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one(make_core, cache_responses=True) + assert isinstance(host.cache, litellm.Cache) + assert host.cache.type == "local" + assert core.settings().cache_responses is True + + +def test_cache_responses_with_redis_installs_a_redis_response_cache(make_core: MakeCore, host: ScriptedHost) -> None: + _one(make_core, cache_responses=True, redis_host="localhost", redis_port=6399) + assert isinstance(host.cache, litellm.Cache) + assert host.cache.type == "redis" + + +def test_cache_responses_keeps_a_response_cache_the_host_already_has(make_core: MakeCore, host: ScriptedHost) -> None: + existing: Final = object() + host.cache = existing + _one(make_core, cache_responses=True) + assert host.cache is existing + + +def test_without_cache_responses_the_host_response_cache_is_untouched(make_core: MakeCore, host: ScriptedHost) -> None: + _one(make_core, redis_host="localhost", redis_port=6399) + assert host.cache is None + + +# --- callback registration ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("strategy", "callbacks", "input_callbacks"), + [ + ("simple-shuffle", [], []), + ("lar1", [], []), + ("least-busy", ["LeastBusyLoggingHandler"], ["LeastBusyLoggingHandler"]), + ("usage-based-routing", ["LowestTPMLoggingHandler"], []), + ("usage-based-routing-v2", ["LowestTPMLoggingHandler_v2"], []), + ("latency-based-routing", ["LowestLatencyLoggingHandler"], []), + ("cost-based-routing", ["LowestCostLoggingHandler"], []), + ], +) +def test_each_routing_strategy_registers_its_selector_on_the_host( + make_core: MakeCore, host: ScriptedHost, strategy: str, callbacks: list[str], input_callbacks: list[str] +) -> None: + _one(make_core, routing_strategy=strategy) + names: Final = host_callback_names(host) + assert (names["callbacks"], names["input_callback"]) == (callbacks, input_callbacks) + + +def test_the_least_busy_selector_is_the_router_leastbusy_logger(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one(make_core, routing_strategy="least-busy") + assert host.lists["callbacks"] == [core.leastbusy_logger] # pyright: ignore[reportAttributeAccessIssue] + + +# --- alerting ---------------------------------------------------------------------------------- + + +def test_alerting_config_registers_slack_alerting_and_its_slow_response_callback( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one( + make_core, alerting_config=AlertingConfig(webhook_url="https://example.invalid/hook", alerting_threshold=10) + ) + names: Final = host_callback_names(host) + assert names["callbacks"] == ["SlackAlerting"] + assert "response_taking_too_long_callback" in names["success_callback"] + logger: Final = core.observe().slack_alerting_logger + assert host.lists["callbacks"] == [logger] + assert logger is not None + assert (logger.alerting_threshold, logger.alerting, logger.default_webhook_url) == ( + 10.0, + ["slack"], + "https://example.invalid/hook", + ) + + +def test_the_python_core_registers_its_sync_outcome_callback_before_the_slow_response_callback( + host: ScriptedHost, +) -> None: + """Python core only: it learns sync outcomes from a success callback on its host. The Rust core + records outcomes from the attempt itself and registers no outcome callbacks.""" + from tests.unit.test_router.core.scripted_host import python_core + + python_core( + host, + model_list=[deployment("g", "a")], + alerting_config=AlertingConfig(webhook_url="https://example.invalid/hook", alerting_threshold=10), + ) + assert host_callback_names(host)["success_callback"] == [ + "sync_deployment_callback_on_success", + "response_taking_too_long_callback", + ] + + +def test_observe_reports_no_slack_alerting_without_an_alerting_config(make_core: MakeCore) -> None: + assert _one(make_core).observe().slack_alerting_logger is None + + +# --- optional pre-call checks: registration ----------------------------------------------- + + +@pytest.mark.parametrize( + ("checks", "expected"), + [ + (["prompt_caching"], ["PromptCachingDeploymentCheck"]), + (["enforce_model_rate_limits"], ["ModelRateLimitingCheck"]), + (["deployment_affinity"], ["DeploymentAffinityCheck"]), + (["responses_api_deployment_check"], ["DeploymentAffinityCheck"]), + (["session_affinity"], ["DeploymentAffinityCheck"]), + (["deployment_affinity", "responses_api_deployment_check", "session_affinity"], ["DeploymentAffinityCheck"]), + (["encrypted_content_affinity"], ["EncryptedContentAffinityCheck"]), + (["prompt_caching", "prompt_caching"], ["PromptCachingDeploymentCheck"]), + (["prompt_caching", "enforce_model_rate_limits"], ["PromptCachingDeploymentCheck", "ModelRateLimitingCheck"]), + ], +) +def test_each_optional_pre_call_check_registers_one_callback_on_the_router_and_the_host( + make_core: MakeCore, host: ScriptedHost, checks: list[str], expected: list[str] +) -> None: + core: Final = _one(make_core, optional_pre_call_checks=checks) + assert [type(callback).__name__ for callback in core.optional_callbacks] == expected # pyright: ignore[reportAttributeAccessIssue] + assert host.lists["callbacks"] == core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.parametrize( + ("checks", "flags"), + [ + (["deployment_affinity"], (True, False, False)), + (["responses_api_deployment_check"], (False, True, False)), + (["session_affinity"], (False, False, True)), + (["deployment_affinity", "session_affinity"], (True, False, True)), + ], +) +def test_affinity_checks_share_one_callback_with_one_flag_each( + make_core: MakeCore, checks: list[str], flags: tuple[bool, bool, bool] +) -> None: + (callback,) = _one( + make_core, optional_pre_call_checks=checks, deployment_affinity_ttl_seconds=60 + ).optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + assert ( + callback.enable_user_key_affinity, + callback.enable_responses_api_affinity, + callback.enable_session_id_affinity, + ) == flags + assert callback.ttl_seconds == 60 + + +def test_encrypted_content_affinity_is_placed_before_deployment_affinity( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core, optional_pre_call_checks=["deployment_affinity", "encrypted_content_affinity"]) + expected: Final = ["EncryptedContentAffinityCheck", "DeploymentAffinityCheck"] + assert [type(callback).__name__ for callback in core.optional_callbacks] == expected # pyright: ignore[reportAttributeAccessIssue] + assert host_callback_names(host)["callbacks"] == expected + + +def test_forward_client_headers_by_model_group_registers_nothing(make_core: MakeCore, host: ScriptedHost) -> None: + """Oddity: a listed optional pre-call check that the router ignores.""" + core: Final = _one(make_core, optional_pre_call_checks=["forward_client_headers_by_model_group"]) + assert core.optional_callbacks is None # pyright: ignore[reportAttributeAccessIssue] + assert host.lists["callbacks"] == [] + + +def test_an_empty_optional_pre_call_check_list_registers_nothing(make_core: MakeCore, host: ScriptedHost) -> None: + assert _one(make_core, optional_pre_call_checks=[]).optional_callbacks is None # pyright: ignore[reportAttributeAccessIssue] + assert host.lists["callbacks"] == [] + + +def test_router_budget_limiting_cannot_be_built_outside_an_event_loop(make_core: MakeCore) -> None: + """Oddity: the budget limiter starts a background task in its constructor, so a router that needs + one cannot be built from synchronous code with no running loop.""" + with pytest.raises(RuntimeError, match="no running event loop"): + _one(make_core, optional_pre_call_checks=["router_budget_limiting"]) + + +@pytest.mark.asyncio +async def test_router_budget_limiting_registers_the_budget_limiter(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one(make_core, optional_pre_call_checks=["router_budget_limiting"]) + limiter: Final = core.router_budget_logger # pyright: ignore[reportAttributeAccessIssue] + assert isinstance(limiter, RouterBudgetLimiting) + assert core.registry().plugins("budget_limiter") is limiter + assert host.lists["callbacks"] == [limiter] + core.retire() + + +@pytest.mark.asyncio +async def test_a_provider_budget_config_adds_the_budget_limiter_by_itself(make_core: MakeCore) -> None: + core: Final = _one(make_core, provider_budget_config={"openai": {"budget_limit": 1, "time_period": "1d"}}) + assert [type(callback).__name__ for callback in core.optional_callbacks] == ["RouterBudgetLimiting"] # pyright: ignore[reportAttributeAccessIssue] + core.retire() + + +@pytest.mark.asyncio +async def test_a_deployment_budget_adds_the_budget_limiter_by_itself(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", max_budget=1, budget_duration="1d")]) + assert isinstance(core.registry().plugins("budget_limiter"), RouterBudgetLimiting) + core.retire() + + +def test_model_group_affinity_config_adds_an_affinity_callback_with_every_global_flag_off(make_core: MakeCore) -> None: + (callback,) = _one(make_core, model_group_affinity_config={"g": ["deployment_affinity"]}).optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + assert type(callback).__name__ == "DeploymentAffinityCheck" + assert ( + callback.enable_user_key_affinity, + callback.enable_responses_api_affinity, + callback.enable_session_id_affinity, + ) == (False, False, False) + assert callback.model_group_affinity_config == {"g": ["deployment_affinity"]} + + +def test_model_group_encrypted_content_affinity_adds_both_affinity_callbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + _one(make_core, model_group_affinity_config={"g": ["encrypted_content_affinity"]}) + assert host_callback_names(host)["callbacks"] == ["EncryptedContentAffinityCheck", "DeploymentAffinityCheck"] + + +def test_mid_stream_fallback_continuation_registers_a_global_prefill_check( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core, enable_mid_stream_fallback_continuation=True) + assert host_callback_names(host)["callbacks"] == ["ContinuationPrefillDeploymentCheck"] + assert core.optional_callbacks is None # pyright: ignore[reportAttributeAccessIssue] + + +def test_add_optional_pre_call_checks_after_construction_registers_the_new_check( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core.add_optional_pre_call_checks(["enforce_model_rate_limits", "prompt_caching"]) # pyright: ignore[reportAttributeAccessIssue] + assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck", "ModelRateLimitingCheck"] + + +def test_add_optional_pre_call_checks_with_none_and_apply_default_settings_change_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + before: Final = list(host.lists["callbacks"]) + core.add_optional_pre_call_checks(None) # pyright: ignore[reportAttributeAccessIssue] + core.apply_default_settings() # pyright: ignore[reportAttributeAccessIssue] + assert host.lists["callbacks"] == before + + +def test_runtime_optional_pre_call_checks_drop_unlisted_toggleable_checks_but_keep_affinity( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching", "deployment_affinity"]) + core.apply(ChangeSet(settings={"optional_pre_call_checks": ["enforce_model_rate_limits"]})) + expected: Final = ["DeploymentAffinityCheck", "ModelRateLimitingCheck"] + assert [type(callback).__name__ for callback in core.optional_callbacks] == expected # pyright: ignore[reportAttributeAccessIssue] + assert host_callback_names(host)["callbacks"] == expected + + +def test_runtime_optional_pre_call_checks_none_changes_nothing(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core.apply(ChangeSet(settings={"optional_pre_call_checks": None})) + assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"] + + +def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_that_check( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the check is left in this host's list because some other live router (here on another + host) still has one of that type.""" + collect_other_routers() + other_host: Final = CallbackHost() + from tests.unit.test_router.core.scripted_host import python_core + + other: Final = python_core( + other_host, model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"] + ) + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core.apply(ChangeSet(settings={"optional_pre_call_checks": []})) + assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue] + assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"] + other.retire() + + +def test_enable_pre_call_checks_is_not_a_runtime_updatable_setting(make_core: MakeCore) -> None: + core: Final = _one(make_core) + core.apply(ChangeSet(settings={"enable_pre_call_checks": True})) + assert core.settings().enable_pre_call_checks is False + + +def test_enable_pre_call_checks_can_be_assigned(make_core: MakeCore) -> None: + core: Final = _one(make_core) + core.apply(ChangeSet(assign={"enable_pre_call_checks": True})) + assert core.settings().enable_pre_call_checks is True + + +# --- retire ------------------------------------------------------------------------------ + + +def test_retire_removes_selectors_optional_checks_and_alerting_but_keeps_foreign_callbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + host.lists["callbacks"].append("foreign") + host.lists["success_callback"].append("foreign-success") + core: Final = _one( + make_core, + routing_strategy="least-busy", + optional_pre_call_checks=["enforce_model_rate_limits"], + alerting_config=AlertingConfig(webhook_url="https://example.invalid/hook"), + ) + core.retire() + assert {name: callbacks for name, callbacks in host.lists.items() if callbacks} == { + "callbacks": ["foreign"], + "success_callback": ["foreign-success"], + } + + +def test_retire_twice_is_harmless(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one(make_core) + core.retire() + core.retire() + assert {name: callbacks for name, callbacks in host.lists.items() if callbacks} == {} + + +def test_retiring_one_router_keeps_the_callbacks_of_another_on_the_same_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + first: Final = _one(make_core, routing_strategy="least-busy") + second: Final = _one(make_core, routing_strategy="least-busy") + first.retire() + assert host.lists["callbacks"] == [second.leastbusy_logger] # pyright: ignore[reportAttributeAccessIssue] + assert host.lists["input_callback"] == [second.leastbusy_logger] # pyright: ignore[reportAttributeAccessIssue] + + +def test_retiring_one_python_core_keeps_the_outcome_callback_of_another_on_the_same_host( + host: ScriptedHost, +) -> None: + """Python core only: each one registers an async outcome callback bound to itself; the Rust core + records outcomes from the attempt itself and registers none.""" + from tests.unit.test_router.core.scripted_host import python_core + + collect_other_routers() + first: Final = python_core(host, model_list=[deployment("g", "a")], routing_strategy="least-busy") + second: Final = python_core(host, model_list=[deployment("g", "a")], routing_strategy="least-busy") + first.retire() + assert host_callback_names(host)["_async_success_callback"] == ["deployment_callback_on_success"] + assert all(callback.__self__ is second for callback in host.lists["_async_success_callback"]) # pyright: ignore[reportAttributeAccessIssue] + + +def test_retire_hands_a_vacated_check_to_another_live_router_on_a_different_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: retire re-registers the same-keyed check of any other live router in the process into + this host's list, even one built on a different host.""" + collect_other_routers() + from tests.unit.test_router.core.scripted_host import python_core + + other: Final = python_core( + CallbackHost(), model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"] + ) + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core.retire() + assert host.lists["callbacks"] == other.optional_callbacks + other.retire() + + +# --- observe ----------------------------------------------------------------------------- + + +class RecordingLogging: + def __init__(self) -> None: + self.calls: list[tuple[str, object]] = [] + + def add_dynamic_function_callbacks(self, **callbacks: object) -> None: # kwargs-ok: the logging object's API + self.calls.append(("functions", sorted(callbacks))) + + def replace_global_callback(self, callback: object) -> None: + self.calls.append(("replace", type(callback).__name__)) + + def add_dynamic_callback(self, callback: object) -> None: + self.calls.append(("add", type(callback).__name__)) + + +def _bound_outcome_callbacks(core: RouterCore) -> list[tuple[str, object]]: # mutable-ok: a fresh list + return ( + [("functions", ["async_failure", "async_success", "failure"])] + if records_outcomes_through_callbacks(core) + else [] + ) + + +def test_observe_request_scoped_removes_every_callback_the_router_registered_globally( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + core: Final = _one(make_core, routing_strategy="least-busy", optional_pre_call_checks=["prompt_caching"]) + observers: Final = core.observe(request_scoped=True) + assert observers.slack_alerting_logger is None + assert {name: callbacks for name, callbacks in host.lists.items() if callbacks} == {} + + +def test_observe_with_a_logging_object_binds_the_router_outcome_callbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The Python core binds its three outcome callbacks; the Rust core records every attempt's + outcome itself and binds none, as it registers none on the host.""" + core: Final = _one(make_core) + before: Final = host_callback_names(host) + logging_obj: Final = RecordingLogging() + core.observe(logging_obj) # pyright: ignore[reportArgumentType] + assert logging_obj.calls == _bound_outcome_callbacks(core) + assert host_callback_names(host) == before + + +def test_observe_request_scoped_then_binding_hands_selectors_and_policies_to_the_request(make_core: MakeCore) -> None: + collect_other_routers() + core: Final = _one(make_core, routing_strategy="least-busy", optional_pre_call_checks=["prompt_caching"]) + core.observe(request_scoped=True) + logging_obj: Final = RecordingLogging() + core.observe(logging_obj) # pyright: ignore[reportArgumentType] + assert logging_obj.calls == [ + *_bound_outcome_callbacks(core), + ("replace", "LeastBusyLoggingHandler"), + ("add", "PromptCachingDeploymentCheck"), + ] + + +def test_binding_without_request_scoping_hands_the_request_no_selector_or_policy(make_core: MakeCore) -> None: + """Without ``observe(request_scoped=True)`` the selectors and checks stay on the host lists, so a + bound request gets none of them.""" + collect_other_routers() + core: Final = _one(make_core, routing_strategy="least-busy", optional_pre_call_checks=["prompt_caching"]) + logging_obj: Final = RecordingLogging() + core.observe(logging_obj) # pyright: ignore[reportCallIssue] # observe takes a logging object on every core + assert logging_obj.calls == _bound_outcome_callbacks(core) + + +def test_binding_skips_a_policy_the_host_already_runs(make_core: MakeCore, host: ScriptedHost) -> None: + """A request-scoped policy whose key is back on the host's ``callbacks`` list (here re-added by + hand) is not added to the request again; the selector still replaces the host's.""" + collect_other_routers() + core: Final = _one(make_core, routing_strategy="least-busy", optional_pre_call_checks=["prompt_caching"]) + core.observe(request_scoped=True) # pyright: ignore[reportCallIssue] # see above + host.lists["callbacks"].append(PromptCachingDeploymentCheck(cache=DualCache())) + logging_obj: Final = RecordingLogging() + core.observe(logging_obj) # pyright: ignore[reportCallIssue] # observe takes a logging object on every core + assert logging_obj.calls == [*_bound_outcome_callbacks(core), ("replace", "LeastBusyLoggingHandler")] + + +# --- register_plugin --------------------------------------------------------------------- + + +class PickB(CustomRoutingStrategyBase): + def __init__(self) -> None: + self.asked: list[str] = [] + + @staticmethod + def _b() -> dict[str, object]: + return deployment("g", "b") + + async def async_get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool = False, + request_kwargs: object = None, + ) -> dict[str, object]: + self.asked.append(model) + return self._b() + + def get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: bool = False, + request_kwargs: object = None, + ) -> dict[str, object]: + self.asked.append(model) + return self._b() + + +@pytest.mark.asyncio +async def test_a_registered_routing_plugin_picks_for_select(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + plugin: Final = PickB() + core.register_plugin(plugin) + picked: Final = {(await core.select("g"))["model_info"]["id"] for _ in range(5)} # pyright: ignore[reportIndexIssue] + assert picked == {"b"} + assert plugin.asked == ["g"] * 5 + + +@pytest.mark.asyncio +async def test_a_registered_routing_plugin_picks_for_async_and_sync_endpoints( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + core.register_plugin(PickB()) + for _ in range(3): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) == {"b"} + + +@pytest.mark.asyncio +async def test_changing_the_routing_strategy_drops_a_registered_routing_plugin(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + plugin: Final = PickB() + core.register_plugin(plugin) + core.apply(ChangeSet(settings={"routing_strategy": "least-busy"})) + for _ in range(5): + await core.select("g") + assert plugin.asked == [] + + +# --- flush_cache / reset ------------------------------------------------------------------ + + +def test_flush_cache_drops_the_response_cache_routing_state_and_prompt_caching_pins( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core) + host.cache = object() + core.health().store.set_cache("k", 1) + core.registry().plugins("prompt_caching_pins").set_cache("p", 1) + before: Final = host_callback_names(host) + core.flush_cache() # pyright: ignore[reportAttributeAccessIssue] + assert host.cache is None + assert core.health().store.get_cache("k") is None + assert core.registry().plugins("prompt_caching_pins").get_cache("p") is None + assert host_callback_names(host) == before + + +def test_reset_empties_every_success_and_failure_list_and_the_retry_policy( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Note: reset clears the host's success/failure lists outright, foreign callbacks included, but + leaves ``callbacks`` and ``input_callback`` alone.""" + core: Final = _one(make_core, retry_policy={"RateLimitErrorRetries": 3}, routing_strategy="least-busy") + host.lists["success_callback"].append("foreign") + core.health().store.set_cache("k", 1) + core.reset() # pyright: ignore[reportAttributeAccessIssue] + assert {name: callbacks for name, callbacks in host_callback_names(host).items() if callbacks} == { + "callbacks": ["LeastBusyLoggingHandler"], + "input_callback": ["LeastBusyLoggingHandler"], + } + assert core.retry_policy is None # pyright: ignore[reportAttributeAccessIssue] + assert core.health().store.get_cache("k") is None + + +# --- attribute writes ---------------------------------------------------------------------- + + +def test_assigning_num_retries_after_construction_is_reported_by_settings(make_core: MakeCore) -> None: + core: Final = _one(make_core) + core.num_retries = 7 # pyright: ignore[reportAttributeAccessIssue] + assert core.settings().as_dict()["num_retries"] == 7 + + +def test_assigning_a_setting_attribute_replaces_the_settings_snapshot(make_core: MakeCore) -> None: + core: Final = _one(make_core) + before: Final = core.settings() + core.enable_pre_call_checks = True # pyright: ignore[reportAttributeAccessIssue] + assert core.settings() is not before + assert core.settings().enable_pre_call_checks is True + assert before.enable_pre_call_checks is False + + +def test_assigning_a_plain_setting_keeps_the_settings_snapshot(make_core: MakeCore) -> None: + core: Final = _one(make_core) + before: Final = core.settings() + core.num_retries = 7 # pyright: ignore[reportAttributeAccessIssue] + assert core.settings() is before + + +def test_assigning_an_unrelated_attribute_keeps_the_settings_snapshot(make_core: MakeCore) -> None: + core: Final = _one(make_core) + before: Final = core.settings() + core.some_custom_attribute = 1 # pyright: ignore[reportAttributeAccessIssue] + assert core.settings() is before + assert core.some_custom_attribute == 1 # pyright: ignore[reportAttributeAccessIssue] + + +def test_assigning_a_new_model_list_advances_the_registry_version_once(make_core: MakeCore) -> None: + core: Final = _one(make_core) + version: Final = core.registry().version + core.model_list = [deployment("g", "z")] # pyright: ignore[reportAttributeAccessIssue] + assert core.registry().version == version + 1 + assert [d["model_info"]["id"] for d in core.registry().all()] == ["z"] + + +def test_deleting_a_setting_attribute_makes_it_unset_in_settings(make_core: MakeCore) -> None: + core: Final = _one(make_core) + del core.routing_strategy # pyright: ignore[reportAttributeAccessIssue] + assert core.settings().routing_strategy is None + assert core.settings().is_set("routing_strategy") is False + + +def test_registry_settings_and_health_views_are_built_once(make_core: MakeCore) -> None: + core: Final = _one(make_core) + assert core.registry() is core.registry() + assert core.settings() is core.settings() + assert core.health() is core.health() + + +# --- get_valid_args ----------------------------------------------------------------------- + + +def test_get_valid_args_lists_every_constructor_argument_but_self_and_the_host() -> None: + names: Final = PythonCore.get_valid_args() + assert names[0] == "model_list" + assert "self" not in names + assert "_host" not in names + assert {"routing_strategy", "optional_pre_call_checks", "enable_pre_call_checks", "redis_url"} <= set(names) + assert len(names) == len(set(names)) + + +def test_get_valid_args_matches_the_router_init_args() -> None: + assert tuple(PythonCore.get_valid_args()) == tuple(router_init_args()) + + +def test_get_valid_args_on_a_built_router(make_core: MakeCore) -> None: + assert _one(make_core).get_valid_args() == PythonCore.get_valid_args() # pyright: ignore[reportAttributeAccessIssue] + + +# --- print_deployment ---------------------------------------------------------------------- + +SECRET_DEPLOYMENT: Final = { + "model_name": "g", + "litellm_params": { + "model": "openai/x", + "api_key": "sk-abcdef", + "api_base": "https://h", + "aws_secret_access_key": "secretvalue", + }, + "model_info": {"id": "x"}, +} + + +def test_print_deployment_shows_two_characters_of_the_api_key_then_ten_stars(make_core: MakeCore) -> None: + printed: Final = _one(make_core).print_deployment(SECRET_DEPLOYMENT) # pyright: ignore[reportAttributeAccessIssue] + assert printed["litellm_params"] == { + "model": "openai/x", + "api_key": "sk**********", + "api_base": "https://h", + "aws_secret_access_key": "secretvalue", + } + assert printed["model_info"] == {"id": "x"} + + +def test_print_deployment_leaves_the_given_deployment_unchanged(make_core: MakeCore) -> None: + _one(make_core).print_deployment(SECRET_DEPLOYMENT) # pyright: ignore[reportAttributeAccessIssue] + assert SECRET_DEPLOYMENT["litellm_params"]["api_key"] == "sk-abcdef" # pyright: ignore[reportIndexIssue] + + +def test_print_deployment_without_an_api_key_returns_an_equal_copy(make_core: MakeCore) -> None: + given: Final = {"litellm_params": {"model": "m"}} + printed: Final = _one(make_core).print_deployment(given) # pyright: ignore[reportAttributeAccessIssue] + assert printed == given + assert printed is not given + + +def test_print_deployment_without_litellm_params_raises_key_error(make_core: MakeCore) -> None: + with pytest.raises(KeyError, match="litellm_params"): + _one(make_core).print_deployment({"model_name": "x"}) # pyright: ignore[reportAttributeAccessIssue] + + +def test_print_deployment_with_redaction_masks_every_secret_keeping_its_length( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one(make_core) + host.set_dynamic(redact_user_api_key_info=True) + printed: Final = core.print_deployment(SECRET_DEPLOYMENT) # pyright: ignore[reportAttributeAccessIssue] + assert printed["litellm_params"] == { + "model": "openai/x", + "api_key": "sk*******", + "api_base": "https://h", + "aws_secret_access_key": "se*********", + } + + +def test_settings_snapshot_reads_constructor_mappings_as_given(make_core: MakeCore) -> None: + alias: Final[Mapping[str, object]] = {"alias": "g"} + core: Final = _one(make_core, model_group_alias=dict(alias)) + assert core.registry().aliases() == alias diff --git a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py new file mode 100644 index 00000000000..96d4d1879ce --- /dev/null +++ b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py @@ -0,0 +1,707 @@ +""" +Characterization of the pre-call checks on a router core: with ``enable_pre_call_checks`` the +context window (token counting over messages, Responses input, instructions, tools and an Anthropic +system block), the allowed region and unsupported params (``response_format``) narrow a group's +deployments before one is picked; image requests skip text-only deployments; budget limits drop +deployments over budget. Covers the errors raised when nothing is left (type and message), which +deployment ``select`` and the endpoints reach, and that no provider call is made then. + +Each test asserts what the Python core does today; oddities are named in the test's docstring. +""" + +import base64 +import re +from collections.abc import Callable, Generator, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.router import RouterRateLimitError +from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, restoring_model_cost, windowed +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, python_core + +MakeCore = Callable[..., RouterCore] +PICKS: Final = 60 # three-way shuffles miss a deployment in 20 picks about once in a thousand runs +SHORT: Final = [{"role": "user", "content": "hi"}] +LONG_TEXT: Final = " ".join(["word"] * 100) +LONG: Final = [{"role": "user", "content": LONG_TEXT}] +EMBEDDING_IMAGE: Final = f"data:image/png;base64,{base64.b64encode(bytes(range(256)) * 1024).decode()}" +IMAGE: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ], + } +] +UNCOUNTABLE: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"detail": "not-a-real-detail-value"}}]} +] +JSON_MODE: Final = {"response_format": {"type": "json_object"}} +NO_JSON_MODE_MODEL: Final = "openai/gpt-3.5-turbo-16k" +UNMAPPED_MODEL: Final = "openai/a-model-the-catalog-does-not-map" +JSON_MODE_MODEL: Final = "openai/gpt-4o" +CONTEXT_WINDOW_PREFIX: Final = ( + "litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded " + "for given call. No models have context window large enough for this call.\n" +) +UNSUPPORTED_MESSAGE: Final = ( + "litellm.UnsupportedParamsError: no deployment of model=g supports ['response_format']. To drop these, set " + "`litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, or send " + "allowed_openai_params=['response_format'] to pass them through" +) + + +@pytest.fixture +def host() -> ScriptedHost: + return CallbackHost() + + +@pytest.fixture(autouse=True) +def isolated_model_cost() -> Generator[None, None, None]: + """Routers write their deployments' model info into the process cost map; undo it after each test.""" + yield from restoring_model_cost("gpt-3.5-turbo-16k") + + +async def _picked( + core: RouterCore, model: str = "g", times: int = PICKS, **select_arguments: object +) -> set[str]: # kwargs-ok: select's arguments + picked: Final[set[str]] = set() + for _ in range(times): + chosen = await core.select(model, **select_arguments) # pyright: ignore[reportCallIssue] + assert isinstance(chosen, Mapping) + picked.add(str(chosen["model_info"]["id"])) + return picked + + +def _small_and_big( + make_core: MakeCore, small: int = 5, **arguments: object +) -> RouterCore: # kwargs-ok: constructor arguments + return make_core( + model_list=[windowed("g", "small", small), windowed("g", "big", 10000)], + **{"enable_pre_call_checks": True, **arguments}, + ) + + +def _json_mode_group(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: constructor arguments + return make_core( + model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL), deployment("g", "new", model=JSON_MODE_MODEL)], + **{"enable_pre_call_checks": True, **arguments}, + ) + + +# --- context window ---------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_without_enable_pre_call_checks_a_too_small_context_window_is_still_picked(make_core: MakeCore) -> None: + core: Final = _small_and_big(make_core, enable_pre_call_checks=False) + assert await _picked(core, messages=LONG) == {"small", "big"} + + +@pytest.mark.asyncio +async def test_without_enable_pre_call_checks_a_too_long_prompt_reaches_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)]) + await core.acompletion(model="g", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["small"] + + +@pytest.mark.asyncio +async def test_select_skips_a_deployment_whose_context_window_the_prompt_exceeds(make_core: MakeCore) -> None: + assert await _picked(_small_and_big(make_core), messages=LONG) == {"big"} + + +@pytest.mark.asyncio +async def test_a_short_prompt_may_pick_either_deployment(make_core: MakeCore) -> None: + assert await _picked(_small_and_big(make_core, small=50), messages=SHORT) == {"small", "big"} + + +@pytest.mark.asyncio +async def test_async_completion_reaches_only_a_deployment_whose_context_window_fits( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _small_and_big(make_core) + for _ in range(8): + await core.acompletion(model="g", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) == {"big"} + + +def test_sync_completion_reaches_only_a_deployment_whose_context_window_fits( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _small_and_big(make_core) + for _ in range(8): + core.completion(model="g", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) == {"big"} + + +@pytest.mark.asyncio +async def test_pass_through_selection_skips_a_deployment_whose_context_window_the_prompt_exceeds( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[ + windowed("g", "small", 5, use_in_pass_through=True), + windowed("g", "big", 10000, use_in_pass_through=True), + ], + enable_pre_call_checks=True, + ) + assert await _picked(core, kind="pass_through", messages=LONG) == {"big"} + + +@pytest.mark.asyncio +async def test_select_without_messages_or_input_runs_no_context_window_check(make_core: MakeCore) -> None: + assert await _picked(_small_and_big(make_core)) == {"small", "big"} + + +@pytest.mark.asyncio +async def test_when_no_context_window_fits_select_raises_context_window_exceeded(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True) + with pytest.raises(litellm.ContextWindowExceededError) as error: + await core.select("g", messages=LONG) + message: Final = str(error.value) + assert message.startswith(CONTEXT_WINDOW_PREFIX) + assert re.fullmatch( + r"Model=openai/cw-small-[0-9a-f]{8}, Max Input Tokens=5, Got=\d+", message.removeprefix(CONTEXT_WINDOW_PREFIX) + ) + assert error.value.model == "g" + assert error.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_when_no_context_window_fits_async_completion_raises_without_calling_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True) + with pytest.raises(litellm.ContextWindowExceededError, match="Context Window exceeded for given call"): + await core.acompletion(model="g", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + assert host.sleeps == [] + + +def test_when_no_context_window_fits_sync_completion_raises_without_calling_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True) + with pytest.raises(litellm.ContextWindowExceededError): + core.completion(model="g", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_the_context_window_error_lists_each_deployment_back_to_back(make_core: MakeCore) -> None: + """Oddity: the per-deployment entries are concatenated with no separator between them.""" + core: Final = make_core(model_list=[windowed("g", "one", 5), windowed("g", "two", 6)], enable_pre_call_checks=True) + with pytest.raises(litellm.ContextWindowExceededError) as error: + await core.select("g", messages=LONG) + assert re.fullmatch( + r"Model=openai/cw-one-[0-9a-f]{8}, Max Input Tokens=5, Got=(\d+)Model=openai/cw-two-[0-9a-f]{8}, Max Input Tokens=6, Got=\1", + str(error.value).removeprefix(CONTEXT_WINDOW_PREFIX), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("where", "expected"), + [("model_info", "azure/gpt-4-mi-base"), ("litellm_params", "azure/gpt-4-lp-base")], +) +async def test_the_context_window_error_names_a_deployment_by_its_base_model( + make_core: MakeCore, where: str, expected: str +) -> None: + extra: Final[dict[str, object]] = ( + {"model_info": {"base_model": expected}} if where == "model_info" else {"base_model": expected} + ) + core: Final = make_core(model_list=[windowed("g", f"base-{where}", 5, **extra)], enable_pre_call_checks=True) + with pytest.raises(litellm.ContextWindowExceededError) as error: + await core.select("g", messages=LONG) + assert f"Model={expected}, Max Input Tokens=5, Got=" in str(error.value) + + +@pytest.mark.asyncio +async def test_when_no_context_window_fits_the_context_window_fallback_group_answers( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[windowed("small-group", "small", 5), deployment("big-group", "big")], + enable_pre_call_checks=True, + context_window_fallbacks=[{"small-group": ["big-group"]}], + num_retries=0, + ) + await core.acompletion(model="small-group", messages=LONG) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["big"] + + +@pytest.mark.asyncio +async def test_a_deployment_without_a_known_context_window_is_kept_for_any_prompt(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[windowed("g", "small", 5), deployment("g", "unknown-window-model")], enable_pre_call_checks=True + ) + assert await _picked(core, messages=LONG) == {"unknown-window-model"} + + +@pytest.mark.asyncio +async def test_the_context_window_comes_from_the_cost_map_when_the_deployment_declares_none( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "gpt4", model="openai/gpt-4"), deployment("g", "gpt4o", model="openai/gpt-4o")], + enable_pre_call_checks=True, + ) + assert await _picked(core, times=10, messages=[{"role": "user", "content": "word " * 10000}]) == {"gpt4o"} + + +@pytest.mark.asyncio +async def test_an_uncountable_prompt_skips_the_context_window_check(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[windowed("g", "tiny", 1)], enable_pre_call_checks=True) + assert await _picked(core, times=2, messages=UNCOUNTABLE) == {"tiny"} + + +@pytest.mark.asyncio +async def test_responses_input_text_is_counted_against_the_context_window(make_core: MakeCore) -> None: + assert await _picked(_small_and_big(make_core), times=10, input=LONG_TEXT) == {"big"} + + +@pytest.mark.asyncio +async def test_an_embedding_image_stays_out_of_the_context_window_on_a_backend_that_reads_it_as_media( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "image", model="vertex_ai/gemini-embedding-2", model_info={"max_input_tokens": 8192}) + ], + enable_pre_call_checks=True, + ) + assert await _picked(core, times=2, input=EMBEDDING_IMAGE) == {"image"} + + +@pytest.mark.asyncio +async def test_an_embedding_image_counts_as_text_on_a_text_only_backend(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "text", model="openai/text-embedding-3-small", model_info={"max_input_tokens": 10}) + ], + enable_pre_call_checks=True, + ) + with pytest.raises(litellm.ContextWindowExceededError): + await _picked(core, times=1, input=EMBEDDING_IMAGE) + + +@pytest.mark.asyncio +async def test_a_short_responses_input_may_pick_either_deployment(make_core: MakeCore) -> None: + assert await _picked(_small_and_big(make_core, small=50), input="hi") == {"small", "big"} + + +@pytest.mark.asyncio +async def test_responses_instructions_are_counted_against_the_context_window(make_core: MakeCore) -> None: + core: Final = _small_and_big(make_core, small=50) + assert await _picked(core, request_kwargs={"instructions": LONG_TEXT}, input="hi") == {"big"} + + +@pytest.mark.asyncio +async def test_tool_definitions_are_counted_against_the_context_window(make_core: MakeCore) -> None: + tool: Final = { + "type": "function", + "function": {"name": "f", "description": LONG_TEXT, "parameters": {"type": "object", "properties": {}}}, + } + core: Final = _small_and_big(make_core, small=50) + assert await _picked(core, request_kwargs={"tools": [tool]}, messages=SHORT) == {"big"} + + +@pytest.mark.asyncio +async def test_an_anthropic_system_block_is_counted_against_the_context_window(make_core: MakeCore) -> None: + core: Final = _small_and_big(make_core, small=50) + assert await _picked(core, request_kwargs={"system": LONG_TEXT}, messages=SHORT) == {"big"} + + +@pytest.mark.asyncio +async def test_a_deployment_context_window_is_written_into_the_process_cost_map(make_core: MakeCore) -> None: + """Oddity: for a model the catalog does not map, a deployment's declared ``max_input_tokens`` lands on + the model's process-wide entry, so it outlives the router and narrows another router's deployment.""" + first: Final = make_core( + model_list=[deployment("g", "declares-five", model=UNMAPPED_MODEL, model_info={"max_input_tokens": 5})] + ) + first.retire() + other: Final = python_core( + CallbackHost(), + model_list=[deployment("g", "declares-nothing", model=UNMAPPED_MODEL)], + enable_pre_call_checks=True, + ) + with pytest.raises(litellm.ContextWindowExceededError, match="Max Input Tokens=5"): + await other.select("g", messages=SHORT) + other.retire() + + +@pytest.mark.asyncio +async def test_a_deployment_context_window_stays_off_a_mapped_models_process_entry(make_core: MakeCore) -> None: + """The catalog's token limits win on the shared entry: the declared limit binds its own deployment.""" + first: Final = make_core( + model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})] + ) + first.retire() + other: Final = python_core( + CallbackHost(), + model_list=[deployment("g", "declares-nothing", model=NO_JSON_MODE_MODEL)], + enable_pre_call_checks=True, + ) + assert await _picked(other, messages=SHORT) == {"declares-nothing"} + other.retire() + + +# --- region ------------------------------------------------------------------------------ + + +def _regional(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: constructor arguments + return make_core( + model_list=[ + deployment("g", "eu", region_name="eu"), + deployment("g", "us", region_name="us"), + deployment("g", "none"), + ], + **{"enable_pre_call_checks": True, **arguments}, + ) + + +@pytest.mark.asyncio +async def test_allowed_model_region_keeps_only_deployments_in_that_region(make_core: MakeCore) -> None: + core: Final = _regional(make_core) + assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}, messages=SHORT) == {"eu"} + + +@pytest.mark.asyncio +async def test_allowed_model_region_is_ignored_without_enable_pre_call_checks(make_core: MakeCore) -> None: + core: Final = _regional(make_core, enable_pre_call_checks=False) + assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}, messages=SHORT) == {"eu", "us", "none"} + + +@pytest.mark.asyncio +async def test_allowed_model_region_is_ignored_without_messages_or_input(make_core: MakeCore) -> None: + core: Final = _regional(make_core) + assert await _picked(core, request_kwargs={"allowed_model_region": "eu"}) == {"eu", "us", "none"} + + +@pytest.mark.asyncio +async def test_no_deployment_in_the_allowed_region_raises_router_rate_limit_error(make_core: MakeCore) -> None: + core: Final = _regional(make_core) + with pytest.raises(RouterRateLimitError) as error: + await core.select("g", {"allowed_model_region": "ap"}, messages=SHORT) + assert str(error.value) == ( + "No deployments available for selected model, Try again in 5 seconds. Passed model=g. " + "pre-call-checks=True, cooldown_list=[]" + ) + + +@pytest.mark.asyncio +async def test_no_deployment_in_the_allowed_region_never_calls_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _regional(make_core) + with pytest.raises(RouterRateLimitError): + await core.acompletion(model="g", messages=SHORT, allowed_model_region="ap") # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_async_completion_with_an_allowed_region_reaches_only_that_region( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _regional(make_core) + for _ in range(6): + await core.acompletion(model="g", messages=SHORT, allowed_model_region="us") # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) == {"us"} + + +# --- unsupported params ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_with_drop_params_off_a_deployment_that_lacks_response_format_is_skipped( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE), messages=SHORT) == {"new"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [None, True]) +async def test_unless_drop_params_is_exactly_false_unsupported_params_are_not_checked( + make_core: MakeCore, host: ScriptedHost, drop_params: bool | None +) -> None: + """Oddity: the unset process default (``None``) skips the check as ``drop_params=True`` does.""" + host.set_dynamic(drop_params=drop_params) + assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old", "new"} + + +@pytest.mark.asyncio +async def test_unsupported_params_are_not_checked_without_enable_pre_call_checks( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = _json_mode_group(make_core, enable_pre_call_checks=False) + assert await _picked(core, request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old", "new"} + + +@pytest.mark.asyncio +async def test_unsupported_params_are_not_checked_without_messages_or_input( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + assert await _picked(_json_mode_group(make_core), request_kwargs=dict(JSON_MODE)) == {"old", "new"} + + +@pytest.mark.asyncio +async def test_a_string_response_format_is_not_checked(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(drop_params=False) + core: Final = _json_mode_group(make_core) + assert await _picked(core, request_kwargs={"response_format": "b64_json"}, messages=SHORT) == {"old", "new"} + + +@pytest.mark.asyncio +async def test_when_no_deployment_supports_the_param_select_raises_unsupported_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True) + with pytest.raises(litellm.UnsupportedParamsError) as error: + await core.select("g", dict(JSON_MODE), messages=SHORT) + assert str(error.value) == UNSUPPORTED_MESSAGE + assert error.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_when_no_deployment_supports_the_param_async_completion_raises_without_calling_or_retrying( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True, num_retries=2 + ) + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"): + await core.acompletion(model="g", messages=SHORT, **JSON_MODE) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + assert host.sleeps == [] + + +def test_when_no_deployment_supports_the_param_sync_completion_raises_without_calling( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True) + with pytest.raises(litellm.UnsupportedParamsError): + core.completion(model="g", messages=SHORT, **JSON_MODE) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + + +REQUEST_ESCAPES: Final = [ + {"allowed_openai_params": ["response_format"]}, + {"drop_params": True}, + {"additional_drop_params": ["response_format"]}, +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"]) +async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deployment( + make_core: MakeCore, host: ScriptedHost, escape: dict[str, object] +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core(model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True) + assert await _picked(core, times=2, request_kwargs={**JSON_MODE, **escape}, messages=SHORT) == {"old"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"]) +async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it( + make_core: MakeCore, host: ScriptedHost, escape: dict[str, object] +) -> None: + host.set_dynamic(drop_params=False) + core: Final = _json_mode_group(make_core) + assert await _picked(core, request_kwargs={**JSON_MODE, **escape}, messages=SHORT) == {"new"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deployment_escape", + [{"drop_params": True}, {"allowed_openai_params": ["response_format"]}], + ids=["drop", "allowlist"], +) +async def test_a_deployment_that_allows_or_drops_the_param_is_kept( + make_core: MakeCore, host: ScriptedHost, deployment_escape: dict[str, object] +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL, **deployment_escape)], enable_pre_call_checks=True + ) + assert await _picked(core, times=2, request_kwargs=dict(JSON_MODE), messages=SHORT) == {"old"} + + +@pytest.mark.asyncio +async def test_a_request_drop_params_false_overrides_the_deployment_drop_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL, drop_params=True)], enable_pre_call_checks=True + ) + with pytest.raises(litellm.UnsupportedParamsError): + await core.select("g", {**JSON_MODE, "drop_params": False}, messages=SHORT) + + +@pytest.mark.asyncio +async def test_a_context_window_error_wins_over_an_unsupported_param(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[deployment("g", "small-no-json", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})], + enable_pre_call_checks=True, + ) + with pytest.raises(litellm.ContextWindowExceededError): + await core.select("g", dict(JSON_MODE), messages=LONG) + core.retire() + + +@pytest.mark.asyncio +async def test_a_capable_deployment_outside_the_region_gives_no_deployments_rather_than_a_400( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[ + deployment("g", "old", model=NO_JSON_MODE_MODEL, region_name="eu"), + deployment("g", "new", model=JSON_MODE_MODEL, region_name="us"), + ], + enable_pre_call_checks=True, + ) + with pytest.raises(RouterRateLimitError, match=r"^No deployments available for selected model"): + await core.select("g", {**JSON_MODE, "allowed_model_region": "eu"}, messages=SHORT) + + +@pytest.mark.asyncio +async def test_dropping_the_param_keeps_the_in_region_deployment_that_lacks_it( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = make_core( + model_list=[ + deployment("g", "old", model=NO_JSON_MODE_MODEL, region_name="eu"), + deployment("g", "new", model=JSON_MODE_MODEL, region_name="us"), + ], + enable_pre_call_checks=True, + ) + request: Final = {**JSON_MODE, "allowed_model_region": "eu", "drop_params": True} + assert await _picked(core, times=5, request_kwargs=request, messages=SHORT) == {"old"} + + +@pytest.mark.asyncio +async def test_a_capable_deployment_in_cooldown_gives_no_deployments_rather_than_a_400( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(drop_params=False) + core: Final = _json_mode_group(make_core) + core.health().cooldowns.add_deployment_to_cooldown( + "new", original_exception=Exception("down"), exception_status=429, cooldown_time=60 + ) + with pytest.raises(RouterRateLimitError) as error: + await core.select("g", dict(JSON_MODE), messages=SHORT) + assert str(error.value).startswith( + "No deployments available for selected model, Try again in 60 seconds. Passed model=g. pre-call-checks=True" + ) + + +# --- images -------------------------------------------------------------------------------- + + +def _vision_group(make_core: MakeCore) -> RouterCore: + return make_core( + model_list=[ + deployment("g", "text", model_info={"supports_vision": False}), + deployment("g", "vision", model_info={"supports_vision": True}), + ] + ) + + +@pytest.mark.asyncio +async def test_an_image_request_skips_a_declared_text_only_deployment_even_without_pre_call_checks( + make_core: MakeCore, +) -> None: + assert await _picked(_vision_group(make_core), messages=IMAGE) == {"vision"} + + +@pytest.mark.asyncio +async def test_an_image_in_the_request_messages_counts_as_an_image_request(make_core: MakeCore) -> None: + assert await _picked(_vision_group(make_core), request_kwargs={"messages": IMAGE}) == {"vision"} + + +@pytest.mark.asyncio +async def test_a_text_request_may_pick_a_text_only_deployment(make_core: MakeCore) -> None: + assert await _picked(_vision_group(make_core), messages=SHORT) == {"text", "vision"} + + +@pytest.mark.asyncio +async def test_an_image_request_keeps_every_deployment_when_all_are_text_only(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "text", model_info={"supports_vision": False}), + deployment("g", "text2", model_info={"supports_vision": False}), + ] + ) + assert await _picked(core, messages=IMAGE) == {"text", "text2"} + + +@pytest.mark.asyncio +async def test_async_completion_with_an_image_reaches_the_vision_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _vision_group(make_core) + for _ in range(6): + await core.acompletion(model="g", messages=IMAGE) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) == {"vision"} + + +# --- budgets ------------------------------------------------------------------------------- + +PROVIDER_BUDGET: Final = {"openai": {"budget_limit": 1, "time_period": "1d"}} + + +@pytest.mark.asyncio +async def test_a_provider_over_budget_leaves_no_deployment_and_select_raises_value_error(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET) + await core.health().store.async_set_cache("provider_spend:openai:1d", 5.0) + with pytest.raises(ValueError, match=r"^No deployments available - crossed budget: ") as error: + await core.select("g") + assert ( + str(error.value) + == "No deployments available - crossed budget: Exceeded budget for provider openai: 5.0 >= 1.0\n" + ) + core.retire() + + +@pytest.mark.asyncio +async def test_a_provider_over_budget_raises_from_async_completion_without_calling_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET) + await core.health().store.async_set_cache("provider_spend:openai:1d", 5.0) + with pytest.raises(ValueError, match=r"^No deployments available - crossed budget"): + await core.acompletion(model="g", messages=SHORT) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + core.retire() + + +@pytest.mark.asyncio +async def test_a_provider_under_budget_is_picked(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET) + await core.health().store.async_set_cache("provider_spend:openai:1d", 0.5) + assert await _picked(core, times=2) == {"a"} + core.retire() + + +@pytest.mark.asyncio +async def test_a_deployment_over_its_own_budget_is_skipped(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", max_budget=1, budget_duration="1d"), deployment("g", "b")]) + await core.health().store.async_set_cache("deployment_spend:a:1d", 5.0) + assert await _picked(core, times=10) == {"b"} + core.retire() diff --git a/tests/unit/test_router/core/test_deprecated_members.py b/tests/unit/test_router/core/test_deprecated_members.py new file mode 100644 index 00000000000..b73dda78ecd --- /dev/null +++ b/tests/unit/test_router/core/test_deprecated_members.py @@ -0,0 +1,665 @@ +""" +The deprecated Router members (``DeprecatedRouterMembers``, litellm/router_deprecated.py) on a Router +over each core: every old attribute reads what the Python core's ``_core_`` holds, through the +entry point named in its DeprecationWarning, and the old writes have the Python core's effect. +""" + +from collections.abc import Callable +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router import Router +from litellm.router_api import SETTING_FIELDS, ChangeSet +from litellm.router_core import RouterCore +from litellm.router_utils.cooldown_cache import CooldownCache +from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery +from litellm.types.router import AllowedFailsPolicy, Deployment, RouterGeneralSettings, RoutingContext +from tests.unit.test_router.core.registry_helpers import MESSAGES, MakeCore, ids, team_deployment +from tests.unit.test_router.core.scripted_host import deployment + + +def as_router(core: RouterCore) -> Router: + """A ``litellm.Router`` facade over ``core``, as ``Router.from_config`` attaches the core it builds.""" + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it + return router + + +def picked_id(picked: object) -> object: + """The ``model_info.id`` of a picked deployment dict.""" + return (as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {}).get("id") + + +def read(router: Router, name: str) -> object: + with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"): + return getattr(router, name) # pyright: ignore[reportAny] # each old member's own type + + +def write(router: Router, name: str, value: object) -> None: + with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"): + setattr(router, name, value) + + +_CONFIGURED: Final[dict[str, object]] = { # mutable-ok: constructor arguments, copied per router + "model_list": [ + deployment("gpt", "a"), + deployment("gpt", "b"), + deployment("claude", "c", model="anthropic/claude-x"), + team_deployment("int-t", "t1", "TA", "pub", model="openai/team-model"), + deployment("openai/*", "w", model="openai/*"), + ], + "fallbacks": [{"gpt": ["claude"]}], + "context_window_fallbacks": [{"gpt": ["claude"]}], + "content_policy_fallbacks": [{"claude": ["gpt"]}], + "cooldown_time": 7, + "allowed_fails": 2, + "allowed_fails_policy": AllowedFailsPolicy(RateLimitErrorAllowedFails=4), + "disable_cooldowns": True, + "enable_pre_call_checks": True, + "enable_tag_filtering": True, + "tag_filtering_match_any": False, + "tag_routing_prefix": "x-", + "default_litellm_params": {"metadata": {"k": "v"}}, + "routing_strategy": "least-busy", + "default_max_parallel_requests": 5, + "guardrail_list": [{"guardrail_name": "g", "litellm_params": {"guardrail": "x"}}], + "search_tools": [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}], + "health_check_ignore_transient_errors": True, + "enable_health_check_routing": True, + "background_health_check_model_groups": ["gpt"], + "model_group_alias": {"fast": "gpt"}, + "cache_responses": True, +} + +# The Python core's values for _CONFIGURED, by setting (the objects it builds are compared by type). +_CONFIGURED_SETTINGS: Final[dict[str, object]] = { # mutable-ok: expected values + "routing_strategy": "least-busy", + "fallbacks": [{"gpt": ["claude"]}], + "context_window_fallbacks": [{"gpt": ["claude"]}], + "content_policy_fallbacks": [{"claude": ["gpt"]}], + "default_deployment": None, + "default_litellm_params": {"timeout": None, "max_retries": 0, "metadata": {"k": "v", "caching_groups": None}}, + "default_max_parallel_requests": 5, + "cooldown_time": 7, + "allowed_fails": 2, + "allowed_fails_policy": AllowedFailsPolicy(RateLimitErrorAllowedFails=4), + "disable_cooldowns": True, + "enable_pre_call_checks": True, + "enable_health_check_routing": True, + "health_check_ignore_transient_errors": True, + "background_health_check_model_groups": frozenset({"gpt"}), + "enable_tag_filtering": True, + "tag_filtering_match_any": False, + "tag_routing_prefix": "x-", + "router_general_settings": RouterGeneralSettings(), + "provider_budget_config": None, + "provider_default_deployment_ids": ["w"], + "fallback_access_check": None, + "fallback_budget_check": None, + "search_tools": [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}], + "guardrail_list": [{"guardrail_name": "g", "litellm_params": {"guardrail": "x"}}], + "cache_responses": True, +} + + +def _configured(make_core: MakeCore) -> Router: + return as_router(make_core(**{name: _copy(value) for name, value in _CONFIGURED.items()})) + + +def _copy(value: object) -> object: + import copy + + return copy.deepcopy(value) + + +# -- settings ------------------------------------------------------------------------------------ + + +@pytest.mark.parametrize("name", sorted(set(SETTING_FIELDS) - {"kubernetes_pod_discovery"})) +def test_a_setting_attribute_reads_the_configured_setting(make_core: MakeCore, name: str) -> None: + router: Final = _configured(make_core) + value: Final = read(router, name) + assert value == _CONFIGURED_SETTINGS[name] + assert value is getattr(router.settings(), name) + + +def test_the_pod_discovery_setting_is_the_routers_pod_lookup(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + discovery: Final = read(router, "kubernetes_pod_discovery") + assert isinstance(discovery, KubernetesPodDiscovery) + assert discovery is router.settings().kubernetes_pod_discovery + + +def test_default_settings_read_as_the_python_cores_defaults(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + assert read(router, "routing_strategy") == "simple-shuffle" + assert read(router, "cooldown_time") == 5 + assert read(router, "allowed_fails") == 3 + assert read(router, "fallbacks") is None + assert read(router, "default_litellm_params") == { + "timeout": None, + "max_retries": 0, + "metadata": {"caching_groups": None}, + } + assert read(router, "tag_filtering_match_any") is True + assert read(router, "tag_routing_prefix") == "" + assert read(router, "cache_responses") is False + assert read(router, "search_tools") == [] + assert read(router, "guardrail_list") == [] + + +def test_a_setting_attribute_write_is_a_changeset(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + write(router, "cooldown_time", 9) + write(router, "fallbacks", [{"gpt": ["gpt"]}]) + assert read(router, "cooldown_time") == 9 + assert router.settings().cooldown_time == 9 + assert read(router, "fallbacks") == [{"gpt": ["gpt"]}] + + +def test_a_deleted_setting_attribute_no_longer_reads(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with pytest.warns(DeprecationWarning, match=r"Router\.fallbacks is deprecated"): + del router.fallbacks + assert not router.settings().is_set("fallbacks") + with pytest.raises(AttributeError, match="'fallbacks'"): + read(router, "fallbacks") + + +# -- registry state ------------------------------------------------------------------------------ + +_REGISTRY_READS: Final[dict[str, Callable[[Router], object]]] = { + "model_list": lambda r: r.registry().all(), + "model_names": lambda r: r.registry().names(kind="group_set"), + "deployment_names": lambda r: r.registry().names(kind="deployment_names"), + "team_public_model_names": lambda r: r.registry().names(kind="team_public"), + "model_name_to_deployment_indices": lambda r: r.registry().index("model_name"), + "team_model_to_deployment_indices": lambda r: r.registry().index("team_model"), + "cached_deployment_model_info": lambda r: r.registry().deployment_model_info_cache(), + "model_group_alias": lambda r: r.registry().aliases(), + "pattern_router": lambda r: r.registry().patterns(), + "team_pattern_routers": lambda r: r.registry().patterns(team=True), + "routing_groups": lambda r: r.registry().routing_groups(), + "auto_routers": lambda r: r.registry().plugins("auto"), + "complexity_routers": lambda r: r.registry().plugins("complexity"), + "quality_routers": lambda r: r.registry().plugins("quality"), + "adaptive_routers": lambda r: r.registry().plugins("adaptive"), + "routing_plugins": lambda r: r.registry().plugins("routing"), + "prompt_caching_pin_cache": lambda r: r.registry().plugins("prompt_caching_pins"), + "cache": lambda r: r.health().store, + "cooldown_cache": lambda r: r.health().cooldowns, + "health_state_cache": lambda r: r.health().health_checks, + "deployment_latency_map": lambda r: r.health().latency, +} + + +@pytest.mark.parametrize("name", sorted(_REGISTRY_READS)) +def test_a_state_attribute_is_the_live_object_its_entry_point_returns(make_core: MakeCore, name: str) -> None: + router: Final = _configured(make_core) + assert read(router, name) is _REGISTRY_READS[name](router) + + +def test_the_registry_index_attributes_hold_the_python_cores_values(make_core: MakeCore) -> None: + router: Final = _configured(make_core) + assert read(router, "model_names") == {"gpt", "claude", "int-t", "openai/*"} + assert read(router, "deployment_names") == [ + "openai/a", + "openai/b", + "anthropic/claude-x", + "openai/team-model", + "openai/*", + ] + assert read(router, "team_public_model_names") == frozenset({"pub"}) + assert read(router, "model_name_to_deployment_indices") == { + "gpt": [0, 1], + "claude": [2], + "int-t": [3], + "openai/*": [4], + } + assert read(router, "team_model_to_deployment_indices") == {("TA", "pub"): [3]} + assert read(router, "model_group_alias") == {"fast": "gpt"} + assert read(router, "routing_groups") == {} + with pytest.warns(DeprecationWarning, match=r"Router\.routing_groups is deprecated"): + assert router._routing_groups == {} # pyright: ignore[reportPrivateUsage] # the old private alias + assert read(router, "routing_plugins") == [] + assert read(router, "deployment_latency_map") == { + "openai/a": 0, + "openai/b": 0, + "anthropic/claude-x": 0, + "openai/team-model": 0, + "openai/*": 0, + } + assert ids(read(router, "model_list")) == ["a", "b", "c", "t1", "w"] # pyright: ignore[reportArgumentType] # the live model list + + +def test_the_cached_model_info_attribute_is_the_lru_cache(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with pytest.warns(DeprecationWarning, match=r"Router\.cached_deployment_model_info is deprecated"): + cached: Final = router.cached_deployment_model_info + first: Final = cached("a", "gpt") + assert cached("a", "gpt") is first + assert cached.cache_info().hits == 1 + assert first == router.registry().deployment_model_info("a", "gpt") + + +def test_a_configured_routing_plugin_is_listed(make_core: MakeCore) -> None: + class Keep: + async def run(self, context: RoutingContext) -> RoutingContext: + return context + + plugin: Final = Keep() + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], plugins=[plugin])) + assert read(router, "routing_plugins") == [plugin] + + +def test_a_routing_plugin_appended_to_the_live_list_runs(make_core: MakeCore) -> None: + class Narrow: + async def run(self, context: RoutingContext) -> RoutingContext: + context.candidate_models = [m for m in context.candidate_models if m == "openai/b"] + return context + + class Keep: + async def run(self, context: RoutingContext) -> RoutingContext: + return context + + router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")], plugins=[Keep()])) + with pytest.warns(DeprecationWarning, match=r"Router\.routing_plugins is deprecated"): + plugins: Final = router.routing_plugins + plugins.append(Narrow()) + + import asyncio + + for _ in range(4): + picked = asyncio.run(router.select("g", {}, messages=MESSAGES)) # pyright: ignore[reportCallIssue] + assert picked_id(picked) == "b" + + +# -- state writes -------------------------------------------------------------------------------- + + +def test_a_registry_state_write_is_what_the_registry_reads(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + write(router, "model_names", {"x"}) + write(router, "deployment_names", ["openai/x"]) + write(router, "model_name_to_deployment_indices", {"x": [0]}) + write(router, "auto_routers", {}) + assert read(router, "model_names") == {"x"} + assert router.registry().names(kind="group_set") == {"x"} + assert router.registry().names(kind="deployment_names") == ["openai/x"] + assert router.registry().index("model_name") == {"x": [0]} + assert router.registry().plugins("auto") == {} + + +def test_a_routing_state_write_is_what_health_reads(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + store: Final = DualCache() + cooldowns: Final = CooldownCache(cache=store, default_cooldown_time=1) + latency: Final[dict[str, float]] = {"openai/a": 0.5} + write(router, "cache", store) + write(router, "cooldown_cache", cooldowns) + write(router, "deployment_latency_map", latency) + assert read(router, "cache") is store + assert router.health().store is store + assert router.health().cooldowns is cooldowns + assert router.health().latency is latency + + +def test_an_assigned_slack_alerting_logger_is_the_routers_observer(make_core: MakeCore) -> None: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with ( + pytest.raises(AttributeError, match="slack_alerting_logger"), + pytest.warns(DeprecationWarning, match=r"Router\.slack_alerting_logger is deprecated"), + ): + _ = router.slack_alerting_logger + logger: Final = SlackAlerting(alerting=["slack"], default_webhook_url="https://example.invalid/hook") + write(router, "slack_alerting_logger", logger) + assert read(router, "slack_alerting_logger") is logger + assert router.observe().slack_alerting_logger is logger + + +@pytest.mark.asyncio +async def test_an_assigned_async_pick_answers_select(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")])) + calls: Final[list[str]] = [] # mutable-ok: the models the pick was asked for + + async def pick( + model: str, + request_kwargs: dict[str, object], # mutable-ok: the old pick's signature + messages: object = None, + input: object = None, + specific_deployment: object = False, + ) -> dict[str, object]: # mutable-ok: a deployment dict + calls.append(model) + return deployment("gpt", "b") + + write(router, "async_get_available_deployment", pick) + assert read(router, "async_get_available_deployment") is pick + picked: Final = await router.select("gpt", {}) + assert picked_id(picked) == "b" + assert calls == ["gpt"] + + +@pytest.mark.asyncio +async def test_the_async_pick_attribute_picks_like_select(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with pytest.warns(DeprecationWarning, match=r"Router\.async_get_available_deployment is deprecated"): + pick: Final = router.async_get_available_deployment + picked: Final = await pick(model="gpt", request_kwargs={}) + assert picked_id(picked) == "a" + + +# -- methods ------------------------------------------------------------------------------------- + + +def _call(router: Router, name: str, *args: object, **kwargs: object) -> object: # kwargs-ok: the old member's + with pytest.warns(DeprecationWarning, match=rf"Router\.{name} is deprecated"): + return getattr(router, name)(*args, **kwargs) # pyright: ignore[reportAny] # each old member's own type + + +def test_registry_methods_answer_through_the_registry(make_core: MakeCore) -> None: + router: Final = _configured(make_core) + registry: Final = router.registry() + found: Final = _call(router, "get_deployment", "a") + assert isinstance(found, Deployment) + assert found.model_info.id == "a" + assert _call(router, "has_model_id", "a") is True + assert _call(router, "get_model_info", "zzz") is None + assert ids(_call(router, "get_model_list", "gpt")) == ["a", "b"] # pyright: ignore[reportArgumentType] + assert ids(_call(router, "get_all_deployments", "gpt")) == ["a", "b"] # pyright: ignore[reportArgumentType] + assert ids(_call(router, "get_model_list_from_model_alias", "fast")) == ["a", "b"] # pyright: ignore[reportArgumentType] + assert _call(router, "get_model_names") == registry.names() + assert _call(router, "get_model_ids", "gpt") == ["a", "b"] + assert _call(router, "get_model_from_alias", "fast") == "gpt" + assert _call(router, "routable_model_group", "fast") == "gpt" + assert _call(router, "is_recognized_model", "claude") is True + assert _call(router, "get_model_group", "a") == registry.resolve("a", kind="group_of_id") + assert _call(router, "map_team_model", "pub", "TA") == "pub" + assert _call(router, "get_model_access_groups") == registry.access_groups() + assert _call(router, "get_deployment_credentials", "a") == {"api_key": "sk-test"} + assert _call(router, "get_router_deployment_budget_limiter") is None + assert _call(router, "should_include_deployment", "gpt", deployment("gpt", "a")) is True + assert _call(router, "get_router_model_info", None, "gpt", id="a") == registry.router_model_info( + None, "gpt", id="a" + ) + + +def test_settings_methods_answer_through_settings(make_core: MakeCore) -> None: + router: Final = _configured(make_core) + assert _call(router, "get_settings") == dict(router.settings().as_dict()) + assert _call(router, "get_available_guardrail", "g") == { + "guardrail_name": "g", + "litellm_params": {"guardrail": "x"}, + } + with ( + pytest.raises(ValueError, match="No guardrail found with name: nope"), + pytest.warns(DeprecationWarning, match=r"Router\.get_available_guardrail is deprecated"), + ): + router.get_available_guardrail("nope") + + +@pytest.mark.asyncio +async def test_health_and_selection_methods_answer_through_their_entry_points(make_core: MakeCore) -> None: + router: Final = as_router( + make_core(model_list=[deployment("gpt", "a"), deployment("pt", "p", use_in_pass_through=True)]) + ) + assert _call(router, "get_fully_blocked_model_names") == set() + with pytest.warns(DeprecationWarning, match=r"Router\.async_get_fully_unhealthy_model_names is deprecated"): + unhealthy: Final = await router.async_get_fully_unhealthy_model_names() + assert unhealthy == set() + with pytest.warns(DeprecationWarning, match=r"Router\.get_model_group_usage is deprecated"): + usage: Final = await router.get_model_group_usage("gpt") + assert usage == (None, None) + with pytest.warns(DeprecationWarning, match=r"Router\.get_remaining_model_group_usage is deprecated"): + remaining: Final = await router.get_remaining_model_group_usage("gpt") + assert remaining == {} + with pytest.warns(DeprecationWarning, match=r"Router\.async_get_healthy_deployments is deprecated"): + healthy = await router.async_get_healthy_deployments("gpt", {}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the old untyped dicts + assert isinstance(healthy, list) + assert [picked_id(entry) for entry in healthy] == ["a"] # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # see above + assert picked_id(_call(router, "get_available_deployment_for_pass_through", "pt")) == "p" + + +def test_write_methods_go_through_apply(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + _call(router, "update_settings", allowed_fails=6) + assert router.settings().allowed_fails == 6 + router.apply(ChangeSet()) + assert _call(router, "delete_deployment", "a") is not None + assert router.registry().all() == [] + + +def test_the_priority_split_answers_through_health(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], default_priority=2)) + assert _call(router, "split_request_priority", "gpt", {"priority": 3, "x": 1}) == (3, {"x": 1}) + assert _call(router, "split_request_priority", "gpt", {"x": 1}) == (2, {"x": 1}) + assert router.health().priority_header_fields("gpt", {}, "5") == {"priority": 5} + assert router.health().priority_header_fields("gpt", {}, None) == {} + with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'bad'"): + router.health().split_priority("gpt", {"priority": "bad"}) + + +# -- plugin context, request logging ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_pre_routing_hook_answers_through_the_routing_context(make_core: MakeCore) -> None: + router: Final = as_router( + make_core(model_list=[deployment("small", "s"), deployment("big", "b"), _complexity_marker()]) + ) + request: Final[dict[str, object]] = {} + with pytest.warns(DeprecationWarning, match=r"Router\.async_pre_routing_hook is deprecated"): + response = await router.async_pre_routing_hook("smart", request, messages=MESSAGES) # pyright: ignore[reportUnknownMemberType] # the old signature's bare ``list`` + assert response is not None + assert response.model == "small" + decision: Final = as_str_mapping((as_str_mapping(request.get("metadata")) or {}).get("routing_decision")) or {} + assert decision.get("router_model_name") == "smart" + assert decision.get("routed_model") == "small" + plain: Final[dict[str, object]] = {} + with pytest.warns(DeprecationWarning, match="async_pre_routing_hook"): + hooked: Final = await router.async_pre_routing_hook("small", plain) # pyright: ignore[reportUnknownMemberType] # see above + assert hooked is None + assert plain == {} + + +def _complexity_marker() -> dict[str, object]: # mutable-ok: a model_list entry + return { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "small", "MEDIUM": "big", "COMPLEX": "big"}}, + "complexity_router_default_model": "big", + }, + "model_info": {"id": "marker"}, + } + + +def test_log_retry_records_the_failure_on_the_request_metadata(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a", base_model="bm")])) + kwargs: Final[dict[str, object]] = { + "model": "gpt", + "metadata": {"model_info": {"id": "a"}, "deployment": "openai/a", "attempted_retries": 1}, + } + assert _call(router, "log_retry", kwargs, ValueError("boom")) is kwargs + assert kwargs["metadata"] == { + "model_info": {"id": "a"}, + "deployment": "openai/a", + "attempted_retries": 1, + "previous_models": ( + { + "model_group": "gpt", + "deployment_id": "a", + "model": "openai/a", + "base_model": "bm", + "exception_type": "ValueError", + "exception_string": "boom", + "attempted_retries": 1, + }, + ), + "request_retry_count": 1, + } + _call(router, "log_retry", kwargs, KeyError("again")) + metadata: Final = as_str_mapping(kwargs["metadata"]) or {} + assert metadata["request_retry_count"] == 2 + previous: Final = metadata["previous_models"] + assert isinstance(previous, tuple) + assert len(previous) == 2 # pyright: ignore[reportUnknownArgumentType] # an untyped tuple of breadcrumbs + + +def test_arming_the_read_prefetch_without_a_request_batch_does_nothing(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], routing_strategy="usage-based-routing-v2")) + assert _call(router, "arm_routing_read_prefetch", "gpt", {}) is None + assert _call(router, "arm_routing_read_prefetch", "unknown-model") is None + + +def test_binding_to_request_logging_binds_through_observe(make_core: MakeCore) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + logging_obj: Final = Logging( + model="gpt", + messages=MESSAGES, + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + ) + assert _call(router, "bind_to_request_logging", logging_obj) is None + with ( + pytest.raises(AttributeError, match="add_dynamic_function_callbacks"), + pytest.warns(DeprecationWarning, match=r"Router\.bind_to_request_logging is deprecated"), + ): + router.bind_to_request_logging(None) # pyright: ignore[reportArgumentType] + + +# -- writes with a routing effect ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_assigned_routing_groups_route_from_the_next_call(make_core: MakeCore) -> None: + from litellm.types.router import RoutingGroup + + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("claude", "c")])) + with pytest.raises(Exception, match="grp"): + await router.select("grp", {}) + write( + router, + "routing_groups", + {"grp": RoutingGroup(group_name="grp", models=["claude"], routing_strategy="simple-shuffle")}, + ) + assert picked_id(await router.select("grp", {})) == "c" + assert router.registry().routing_groups().keys() == {"grp"} + + +class _NarrowTo: + def __init__(self, keep: str) -> None: + self.keep: Final = keep + + async def run(self, context: RoutingContext) -> RoutingContext: + context.candidate_models = [m for m in context.candidate_models if m == self.keep] + return context + + +@pytest.mark.asyncio +async def test_a_plugin_added_to_an_empty_plugin_list_runs(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")])) + with pytest.warns(DeprecationWarning, match=r"Router\.routing_plugins is deprecated"): + plugins: Final = router.routing_plugins + plugins.append(_NarrowTo("openai/b")) + for _ in range(4): + assert picked_id(await router.select("g", {}, messages=MESSAGES)) == "b" # pyright: ignore[reportCallIssue] + plugins.clear() + assert {picked_id(await router.select("g", {}, messages=MESSAGES)) for _ in range(40)} == {"a", "b"} # pyright: ignore[reportCallIssue] + + +@pytest.mark.asyncio +async def test_an_assigned_plugin_list_runs(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("g", "a"), deployment("g", "b")])) + write(router, "routing_plugins", [_NarrowTo("openai/a")]) + for _ in range(4): + assert picked_id(await router.select("g", {}, messages=MESSAGES)) == "a" # pyright: ignore[reportCallIssue] + + +def test_an_assigned_prompt_caching_pin_cache_is_the_one_flushed(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + pins: Final = DualCache() + write(router, "prompt_caching_pin_cache", pins) + assert read(router, "prompt_caching_pin_cache") is pins + pins.in_memory_cache.set_cache("k", "v") # pyright: ignore[reportUnknownMemberType] # the in-memory cache is untyped + router.flush_cache() # pyright: ignore[reportAttributeAccessIssue] + assert pins.in_memory_cache.get_cache("k") is None # pyright: ignore[reportUnknownMemberType] # see above + + +# -- pod resolution and cooldown_time -------------------------------------------------------------- + + +class _TaggingDiscovery: + """Stands in for KubernetesPodDiscovery: stamps which discovery resolved the pick.""" + + def __init__(self, name: str) -> None: + self.name: Final = name + + def resolve_deployment(self, deployment: object, request_kwargs: object) -> object: + return {**(as_str_mapping(deployment) or {}), "resolved_by": self.name} + + async def async_resolve_deployment(self, deployment: object, request_kwargs: object) -> object: + return self.resolve_deployment(deployment, request_kwargs) + + +@pytest.mark.asyncio +async def test_a_custom_strategy_resolves_pods_through_the_discovery_bound_when_registered( + make_core: MakeCore, +) -> None: + from litellm.types.router import CustomRoutingStrategyBase + + class Pick(CustomRoutingStrategyBase): + async def async_get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: object = False, + request_kwargs: object = None, + ) -> object: + return deployment("gpt", "b") + + def get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] + self, + model: str, + messages: object = None, + input: object = None, + specific_deployment: object = False, + request_kwargs: object = None, + ) -> object: + return deployment("gpt", "b") + + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")])) + router.apply(ChangeSet(assign={"kubernetes_pod_discovery": _TaggingDiscovery("first")})) + assert (as_str_mapping(await router.select("gpt", {})) or {}).get("resolved_by") == "first" + router.register_plugin(Pick()) + router.apply(ChangeSet(assign={"kubernetes_pod_discovery": _TaggingDiscovery("second")})) + picked: Final = as_str_mapping(await router.select("gpt", {})) or {} + assert picked_id(picked) == "b" + assert picked.get("resolved_by") == "first" + + +def test_cooldown_time_reads_back_as_given(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], cooldown_time=7)) + value: Final = read(router, "cooldown_time") + assert value == 7 and type(value) is int + assert router.settings().as_dict()["cooldown_time"] == 7 + assert type(router.settings().as_dict()["cooldown_time"]) is int + write(router, "cooldown_time", 2.5) + assert router.settings().cooldown_time == 2.5 + _call(router, "update_settings", cooldown_time="3") + updated: Final = router.settings().cooldown_time + assert updated == 3 and type(updated) is int + default: Final = as_router(make_core(model_list=[deployment("gpt", "a")])).settings().cooldown_time + assert default == 5 and type(default) is int diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index c0c4c8cf6dd..ce56ae403ba 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -22,6 +22,7 @@ from tests.unit.test_router.core.scripted_host import ( completion_response, deployment, embedding_response, + records_outcomes_through_callbacks, ) MESSAGES: Final = [{"role": "user", "content": "hi"}] @@ -45,11 +46,16 @@ ENDPOINTS: Final[dict[str, tuple[str, EndpointCall]]] = { "anthropic_messages", lambda core: core.aanthropic_messages(model="g", messages=MESSAGES, max_tokens=5), ), + # The proxy's /v1/messages route calls the router by this name (route_request). + "anthropic_messages": ( + "anthropic_messages", + lambda core: core.anthropic_messages(model="g", messages=MESSAGES, max_tokens=5), + ), "call-acompletion": ("acompletion", lambda core: core.call("acompletion", {"model": "g", "messages": MESSAGES})), "call-completion": ("completion", lambda core: core.call("completion", {"model": "g", "messages": MESSAGES})), } SYNC_ENDPOINTS: Final = frozenset({"completion", "embedding", "image_generation"}) -LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages"}) +LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages", "anthropic_messages"}) EMBEDDING_ENDPOINTS: Final = frozenset({"aembedding", "embedding"}) @@ -276,14 +282,15 @@ async def test_request_level_api_key_replaces_the_deployment_key_on_the_host_cal async def test_construction_registers_one_success_and_one_failure_callback_per_mode( make_core: MakeCore, host: ScriptedHost ): - _one_group(make_core) + core: Final = _one_group(make_core) registered: Final = {name: len(callbacks) for name, callbacks in host.lists.items()} + one: Final = 1 if records_outcomes_through_callbacks(core) else 0 assert registered == { "callbacks": 0, - "success_callback": 1, - "_async_success_callback": 1, - "failure_callback": 1, - "_async_failure_callback": 1, + "success_callback": one, + "_async_success_callback": one, + "failure_callback": one, + "_async_failure_callback": one, "input_callback": 0, } assert all(callable(callbacks[0]) for callbacks in host.lists.values() if callbacks) @@ -416,7 +423,6 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]: yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) # A stream that ends without a finish reason fails since #47665. yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) - yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) logging_obj: Final = LiteLLMLogging( model="d1", diff --git a/tests/unit/test_router/core/test_endpoints_characterization_batch.py b/tests/unit/test_router/core/test_endpoints_characterization_batch.py new file mode 100644 index 00000000000..6ba298accfe --- /dev/null +++ b/tests/unit/test_router/core/test_endpoints_characterization_batch.py @@ -0,0 +1,525 @@ +""" +Characterization of the fan-out endpoints and request shapers: the files API (one file per +deployment), batches (create, retrieve across deployments, cancel, list), batch completions, +fastest-response completion, the scheduler (priority) and prompt-management model groups. +""" + +import re +import datetime +import io +import json +from collections.abc import Callable, Mapping +from types import SimpleNamespace +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router_core import RouterCore +from litellm.types.utils import LiteLLMBatch, ModelResponse +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.scripted_host import ScriptedHost, bad_request, deployment + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] +TWO_DEPLOYMENTS: Final = [deployment("g", "d1"), deployment("g", "d2")] + + +@pytest.fixture +def host() -> EndpointHost: + return EndpointHost() + + +def _bucket(kwargs: Mapping[str, object], name: str) -> Mapping[str, object]: + bucket: Final = kwargs[name] + assert isinstance(bucket, Mapping) + return bucket + + +def _batch(batch_id: str = "b1") -> LiteLLMBatch: + return LiteLLMBatch( + id=batch_id, + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="f", + object="batch", + status="completed", + ) + + +def _jsonl() -> io.BytesIO: + return io.BytesIO(b'{"custom_id": "1", "body": {"model": "g", "messages": []}}\n') + + +# --- files ---------------------------------------------------------------------------------- + + +async def test_acreate_file_creates_the_file_on_every_deployment_of_the_group(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch") + assert sorted(host.deployments_called("acreate_file")) == ["d1", "d2"] + + +async def test_acreate_file_returns_the_first_response_with_the_deployment_file_id_mapping( + make_core: MakeCore, host: EndpointHost +): + first: Final = ModelResponse(id="file-1") + second: Final = ModelResponse(id="file-2") + host.answer("openai/d1", first) + host.answer("openai/d2", second) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + response: Final = await core.acreate_file(model="g", file=_jsonl(), purpose="batch") + assert response in (first, second) + assert response._hidden_params["model_file_id_mapping"] == {"d1": "file-1", "d2": "file-2"} + assert "model_id" not in response._hidden_params + + +async def test_acreate_file_rewrites_the_jsonl_model_to_each_deployment_model(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch") + models: Final = {} + for call in host.calls: + sent_file: Final = call.kwargs["file"] + assert isinstance(sent_file, io.BytesIO) + models[call.deployment_id] = json.loads(sent_file.getvalue().decode().strip())["body"]["model"] + assert models == {"d1": "d1", "d2": "d2"} + + +async def test_acreate_file_keeps_a_non_batch_file_as_sent(make_core: MakeCore, host: EndpointHost): + upload: Final = _jsonl() + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acreate_file(model="g", file=upload, purpose="assistants") + assert host.calls[0].kwargs["file"] is not None + sent: Final = host.calls[0].kwargs["file"] + assert isinstance(sent, io.BytesIO) + assert json.loads(sent.getvalue().decode().strip())["body"]["model"] == "g" + + +async def test_acreate_file_puts_the_model_group_in_metadata_and_the_deployment_in_litellm_metadata( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch") + sent: Final = host.calls[0].kwargs + assert _bucket(sent, "metadata")["model_group"] == "g" + assert "deployment" not in _bucket(sent, "metadata") + assert _bucket(sent, "litellm_metadata")["deployment"] == "openai/d1" + assert "model_group" not in _bucket(sent, "litellm_metadata") + assert sent["custom_llm_provider"] == "openai" + + +async def test_acreate_file_without_a_purpose_raises_before_calling_the_host(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(Exception, match="file and file_purpose are required for create_file"): + await core.acreate_file(model="g", file=_jsonl()) + assert host.calls == [] + + +async def test_acreate_file_sends_target_model_names_to_a_litellm_proxy_deployment( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="litellm_proxy/remote-model")]) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch", extra_body={"keep": 1}) + assert dict(host.calls[0].kwargs["extra_body"]) == {"keep": 1, "target_model_names": "remote-model"} # pyright: ignore[reportArgumentType] # a mapping + + +async def test_acreate_file_sends_the_deployment_gcs_bucket_in_litellm_metadata( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", gcs_bucket_name="bucket-1")]) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch") + assert _bucket(host.calls[0].kwargs, "litellm_metadata")["gcs_bucket_name"] == "bucket-1" + + +# --- batches -------------------------------------------------------------------------------- + + +async def test_acreate_batch_routes_with_litellm_metadata_only(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + response: Final = await core.acreate_batch( + model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h" + ) + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"]) == ( + "acreate_batch", + "openai/d1", + "d1", + "openai", + ) + assert "metadata" not in call.kwargs + assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "g" + assert response._hidden_params["model_id"] == "d1" + + +async def test_acreate_batch_is_retried(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"), _batch()) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1) + await core.acreate_batch(model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h") + assert len(host.calls_to("acreate_batch")) == 2 + + +async def test_acreate_batch_never_fails_over_to_another_deployment(make_core: MakeCore, host: EndpointHost): + # The batch belongs to the deployment that created it, so its attempt stamps no failed deployment. + host.answer("*", litellm.InternalServerError(message="boom", llm_provider="openai", model="d")) + core: Final = make_core(model_list=TWO_DEPLOYMENTS, num_retries=0, enable_weighted_failover=True) + with pytest.raises(litellm.InternalServerError): + await core.acreate_batch(model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h") + assert len(host.calls_to("acreate_batch")) == 1 + + +async def test_aretrieve_batch_asks_every_deployment_and_returns_the_batch_found( + make_core: MakeCore, host: EndpointHost +): + found: Final = _batch() + host.answer("openai/d1", bad_request("d1", "no such batch")) + host.answer("openai/d2", found) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + assert await core.aretrieve_batch(model="g", batch_id="b1") is found + assert sorted(host.deployments_called("aretrieve_batch")) == ["d1", "d2"] + for call in host.calls: + assert (call.kwargs["batch_id"], call.kwargs["custom_llm_provider"], call.model_group) == ("b1", "openai", "g") + assert "metadata" not in call.kwargs + + +async def test_aretrieve_batch_raises_the_first_error_when_no_deployment_has_the_batch( + make_core: MakeCore, host: EndpointHost +): + host.answer("openai/d1", bad_request("d1", "first missing")) + host.answer("openai/d2", bad_request("d2", "second missing")) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + with pytest.raises(litellm.BadRequestError, match="missing"): + await core.aretrieve_batch(model="g", batch_id="b1") + assert len(host.calls) == 2 + + +async def test_aretrieve_batch_ignores_a_non_batch_answer_and_raises_a_generic_error( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + with pytest.raises(Exception, match="Unable to find batch in any model"): + await core.aretrieve_batch(model="g", batch_id="b1") + + +async def test_aretrieve_batch_without_a_model_asks_every_deployment_with_its_own_group( + make_core: MakeCore, host: EndpointHost +): + host.answer("*", bad_request(message="missing")) + core: Final = make_core(model_list=[*TWO_DEPLOYMENTS, deployment("h", "d3")]) + with pytest.raises(litellm.BadRequestError): + await core.aretrieve_batch(batch_id="b1") + assert sorted((call.deployment_id, call.model_group) for call in host.calls) == [ + ("d1", "g"), + ("d2", "g"), + ("d3", "h"), + ] + + +async def test_aretrieve_batch_is_not_retried(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) + with pytest.raises(litellm.InternalServerError): + await core.aretrieve_batch(model="g", batch_id="b1") + assert len(host.calls) == 1 + + +async def test_acancel_batch_routes_with_litellm_metadata(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", _batch()) + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + response: Final = await core.acancel_batch(model="g", batch_id="b1") + call: Final = host.calls[0] + assert (call.op, call.deployment_id, call.kwargs["batch_id"], call.kwargs["api_base"]) == ( + "acancel_batch", + "d1", + "b1", + "https://d1.example", + ) + assert "metadata" not in call.kwargs + assert response._hidden_params["model_id"] == "d1" + + +async def test_alist_batches_merges_the_lists_of_every_deployment(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", SimpleNamespace(first_id="a", last_id="b", data=[1, 2], has_more=False)) + host.answer("openai/d2", SimpleNamespace(first_id="c", last_id="d", data=[3], has_more=True)) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + merged: Final = await core.alist_batches(model="g", limit=2) + assert sorted(merged["data"]) == [1, 2, 3] + assert (merged["object"], merged["has_more"]) == ("list", True) + assert merged["first_id"] in ("a", "c") + assert merged["last_id"] in ("b", "d") + + +async def test_alist_batches_sends_the_deployment_params_and_request_without_routing_metadata( + make_core: MakeCore, host: EndpointHost +): + host.answer("*", SimpleNamespace(first_id=None, last_id=None, data=[], has_more=False)) + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + await core.alist_batches(model="g", limit=2, api_base="https://request.example") + call: Final = host.calls[0] + assert (call.model, call.kwargs["limit"], call.kwargs["api_base"], call.deployment_id) == ( + "openai/d1", + 2, + "https://request.example", + None, + ) + assert "metadata" not in call.kwargs and "litellm_metadata" not in call.kwargs + + +async def test_alist_batches_drops_failing_deployments(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", bad_request("d1")) + host.answer("openai/d2", SimpleNamespace(first_id="c", last_id="d", data=[3], has_more=False)) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + merged: Final = await core.alist_batches(model="g") + assert merged == {"object": "list", "data": [3], "first_id": "c", "last_id": "d", "has_more": False} + + +async def test_alist_batches_for_an_unknown_group_returns_an_empty_list(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + assert await core.alist_batches(model="nope") == { + "object": "list", + "data": [], + "first_id": None, + "last_id": None, + "has_more": False, + } + assert host.calls == [] + + +# --- batch completions ------------------------------------------------------------------------ + + +async def test_abatch_completion_sends_one_request_to_each_model_and_returns_errors_in_place( + make_core: MakeCore, host: EndpointHost +): + error: Final = bad_request("d1", "a failed") + host.answer("openai/d1", error) + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0) + results: Final = await core.abatch_completion(models=["a", "b"], messages=MESSAGES) + assert isinstance(results[0], litellm.BadRequestError) + assert isinstance(results[1], ModelResponse) and results[1]._hidden_params["model_id"] == "d2" + + +async def test_abatch_completion_with_message_lists_groups_results_per_request(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")]) + second: Final = [{"role": "user", "content": "second"}] + results: Final = await core.abatch_completion(models=["a", "b"], messages=[MESSAGES, second]) + assert [[response._hidden_params["model_id"] for response in row] for row in results] == [ + ["d1", "d2"], + ["d1", "d2"], + ] + assert sorted(str(call.kwargs["messages"]) for call in host.calls) == sorted( + [str(MESSAGES), str(MESSAGES), str(second), str(second)] + ) + + +async def test_abatch_completion_forwards_extra_kwargs_to_every_request(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")]) + await core.abatch_completion(models=["a", "b"], messages=MESSAGES, max_tokens=15) + assert [call.kwargs["max_tokens"] for call in host.calls] == [15, 15] + + +async def test_abatch_completion_one_model_multiple_requests_returns_one_result_per_request( + make_core: MakeCore, host: EndpointHost +): + host.answer("openai/d1", ModelResponse(id="r1"), bad_request("d1"), ModelResponse(id="r3")) + core: Final = make_core(model_list=[deployment("a", "d1")], num_retries=0) + results: Final = await core.abatch_completion_one_model_multiple_requests(model="a", messages=[MESSAGES] * 3) + assert len(results) == 3 + assert sum(isinstance(result, litellm.BadRequestError) for result in results) == 1 + assert len(host.calls) == 3 + + +async def test_fastest_response_returns_the_first_success_marked_in_hidden_params( + make_core: MakeCore, host: EndpointHost +): + host.answer("openai/d1", bad_request("d1")) + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0) + response: Final = await core.abatch_completion_fastest_response(model="a, b", messages=MESSAGES) + assert response._hidden_params["model_id"] == "d2" + assert response._hidden_params["fastest_response_batch_completion"] is True + + +async def test_fastest_response_raises_when_every_model_fails(make_core: MakeCore, host: EndpointHost): + host.answer("*", bad_request()) + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0) + with pytest.raises(Exception, match="All tasks failed"): + await core.abatch_completion_fastest_response(model="a,b", messages=MESSAGES) + + +async def test_fastest_response_with_an_unknown_model_counts_it_as_failed(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("a", "d1")], num_retries=0) + response: Final = await core.abatch_completion_fastest_response(model="nope, a", messages=MESSAGES) + assert response._hidden_params["model_id"] == "d1" + + +# --- scheduler / priority ------------------------------------------------------------------- + + +async def test_schedule_acompletion_marks_the_response_as_prioritized_and_does_not_send_priority( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + response: Final = await core.schedule_acompletion(model="g", messages=MESSAGES, priority=0) + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + assert "priority" not in host.calls[0].kwargs + + +async def test_schedule_acompletion_without_a_priority_or_default_raises(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + with pytest.raises( + litellm.BadRequestError, + match=re.escape("priority is required to queue a request, send one or set router_settings.default_priority"), + ): + await core.schedule_acompletion(model="g", messages=MESSAGES) + assert host.calls == [] + + +async def test_schedule_acompletion_without_a_priority_uses_the_default_priority( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5) + response: Final = await core.schedule_acompletion(model="g", messages=MESSAGES) + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize("endpoint", ["schedule_acompletion", "acompletion"]) +async def test_a_non_integer_priority_raises_bad_request(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core(model_list=[deployment("g", "d1")]) + with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'high'"): + await core.call(endpoint, {"model": "g", "messages": MESSAGES, "priority": "high"}) + assert host.calls == [] + + +async def test_a_non_integer_priority_with_drop_params_is_served_unscheduled(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + response: Final = await core.acompletion(model="g", messages=MESSAGES, priority="high", drop_params=True) + assert "x-litellm-request-prioritization-used" not in response._hidden_params["additional_headers"] + assert "priority" not in host.calls[0].kwargs + + +async def test_a_non_integer_priority_with_drop_params_queues_at_the_default_priority( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5) + response: Final = await core.acompletion(model="g", messages=MESSAGES, priority="high", drop_params=True) + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize( + ("endpoint", "request_body"), + [("acompletion", {"messages": MESSAGES}), ("atext_completion", {"prompt": "p"})], +) +async def test_a_request_priority_schedules_the_call( + make_core: MakeCore, host: EndpointHost, endpoint: str, request_body: dict[str, object] +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + response: Final = await core.call(endpoint, {"model": "g", "priority": 3, **request_body}) + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True # pyright: ignore[reportAttributeAccessIssue] # the scripted response + assert "priority" not in host.calls[0].kwargs + + +async def test_default_priority_schedules_every_acompletion(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], default_priority=5) + response: Final = await core.acompletion(model="g", messages=MESSAGES) + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +async def test_a_scheduled_call_error_carries_its_priority(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", bad_request("d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(litellm.BadRequestError) as raised: + await core.acompletion(model="g", messages=MESSAGES, priority=4) + assert raised.value.priority == 4 # pyright: ignore[reportAttributeAccessIssue] # stamped by the scheduler + + +@pytest.mark.parametrize(("header", "expected"), [("7", {"priority": 7}), (None, {}), ("", {})]) +async def test_priority_header_update_parses_the_priority_header( + make_core: MakeCore, host: EndpointHost, header: str | None, expected: dict[str, int] +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + assert dict(core.priority_header_update("g", {}, header)) == expected + + +# --- prompt management models ----------------------------------------------------------------- + + +def _logging_obj() -> LiteLLMLogging: + return LiteLLMLogging( + model="pm", + messages=MESSAGES, + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="c1", + function_id="f1", + ) + + +async def test_prompt_management_group_without_a_prompt_id_raises_value_error(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("pm", "p1", model="langfuse/gpt-4o")]) + with pytest.raises(ValueError, match=re.escape("Prompt ID is not set or not a string. Got=None")): + await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1") + assert host.calls == [] + + +async def test_prompt_management_group_with_non_dict_prompt_variables_raises_value_error( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("pm", "p1", model="langfuse/gpt-4o", prompt_id="p", prompt_variables="x")] + ) + with pytest.raises(ValueError, match=re.escape("Prompt variables is set but not a dictionary. Got=x")): + await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1") + + +async def test_prompt_management_group_without_a_call_id_or_logging_object_raises_key_error( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("pm", "p1", model="langfuse/gpt-4o")]) + with pytest.raises(KeyError, match="litellm_call_id"): + await core.acompletion(model="pm", messages=MESSAGES) + + +async def test_litellm_agent_group_without_a_logging_object_raises_key_error(make_core: MakeCore, host: EndpointHost): + """function_setup consumes original_function, so the direct-call path cannot pop it.""" + core: Final = make_core(model_list=[deployment("pm", "p1", model="litellm_agent/openai/gpt-4o")]) + with pytest.raises(KeyError, match="original_function"): + await core.acompletion(model="pm", messages=MESSAGES, litellm_call_id="c1") + assert host.calls == [] + + +async def test_litellm_agent_group_calls_the_stripped_model_directly_with_the_deployment_params( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("pm", "p1", model="litellm_agent/openai/gpt-4o", timeout=9, temperature=0.2)] + ) + response: Final = await core.acompletion(model="pm", messages=MESSAGES, litellm_logging_obj=_logging_obj()) + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id, call.model_group) == ("acompletion", "openai/gpt-4o", "p1", "pm") + assert (call.kwargs["timeout"], call.kwargs["temperature"], "original_function" in call.kwargs) == (9.0, 0.2, False) + assert response._hidden_params == {} + + +async def test_litellm_agent_group_naming_another_group_routes_into_it_with_the_agent_timeout( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("pm", "p1", model="litellm_agent/g", timeout=9), deployment("g", "d1")] + ) + response: Final = await core.acompletion(model="pm", messages=MESSAGES, litellm_logging_obj=_logging_obj()) + call: Final = host.calls[0] + assert (call.model, call.model_group, call.kwargs["timeout"]) == ("openai/d1", "pm", 9.0) + # The agent deployment's api_key travels as a request credential, so the target gets a request-scoped id. + assert call.deployment_id != "d1" + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g" + + +async def test_a_group_with_two_prompt_management_deployments_routes_normally(make_core: MakeCore, host: ScriptedHost): + core: Final = make_core( + model_list=[deployment("pm", "p1", model="langfuse/gpt-4o"), deployment("pm", "p2", model="langfuse/gpt-4o")] + ) + await core.acompletion(model="pm", messages=MESSAGES) + call: Final = host.calls[0] + assert (call.op, call.model, call.model_group) == ("acompletion", "langfuse/gpt-4o", "pm") diff --git a/tests/unit/test_router/core/test_endpoints_characterization_generic.py b/tests/unit/test_router/core/test_endpoints_characterization_generic.py new file mode 100644 index 00000000000..b1a356cf8eb --- /dev/null +++ b/tests/unit/test_router/core/test_endpoints_characterization_generic.py @@ -0,0 +1,850 @@ +""" +Characterization of the generic endpoints (Responses API and its get/delete/cancel/compact/input +items, Google GenAI, OCR, search, decisions, video, containers, skills, interactions, managed +agents, realtime, fine-tuning, files, image edit, LLM pass-through, vector stores, assistants, +guardrails): which host op each reaches, the kwargs the core builds, and how an id in the request +pins the deployment. + +The Python core binds litellm's own functions for some of these endpoints instead of going through +``host.invoke``; the ``host`` fixture replaces those functions with recording stand-ins (see +``endpoints_helpers.py``), so every call shows up in ``host.calls``. +""" + +import re +import inspect +import uuid +from collections.abc import Callable, Mapping +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.constants import REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_core import RouterCore +from litellm.types.videos.utils import encode_video_id_with_provider +from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] + + +@pytest.fixture +def host(monkeypatch: pytest.MonkeyPatch) -> EndpointHost: + endpoint_host: Final = EndpointHost() + patch_direct_endpoints(monkeypatch, endpoint_host) + return endpoint_host + + +async def _call(core: RouterCore, endpoint: str, request: Mapping[str, object]) -> object: + result: Final = getattr(core, endpoint)(**request) + return await result if inspect.isawaitable(result) else result + + +def _bucket(kwargs: Mapping[str, object], name: str) -> Mapping[str, object]: + bucket: Final = kwargs[name] + assert isinstance(bucket, Mapping) + return bucket + + +# endpoint -> (recorded op, request without model) +ROUTED_ASYNC: Final[dict[str, tuple[str, dict[str, object]]]] = { + "aresponses": ("aresponses", {"input": "hi"}), + "agenerate_content": ("agenerate_content", {"contents": []}), + "agenerate_content_stream": ("agenerate_content_stream", {"contents": []}), + "afile_delete": ("afile_delete", {"file_id": "f1"}), + "afile_content": ("afile_content", {"file_id": "f1"}), + "afile_list": ("afile_list", {}), + "aget_responses": ("aget_responses", {"response_id": "resp_plain"}), + "acancel_responses": ("acancel_responses", {"response_id": "resp_plain"}), + "acompact_responses": ("acompact_responses", {"response_id": "resp_plain"}), + "adelete_responses": ("adelete_responses", {"response_id": "resp_plain"}), + "alist_input_items": ("alist_input_items", {"response_id": "resp_plain"}), + "acreate_realtime_client_secret": ("acreate_realtime_client_secret", {}), + "arealtime_calls": ("arealtime_calls", {}), + "acreate_realtime_transcription_session": ("acreate_realtime_transcription_session", {}), + "acreate_realtime_translation_client_secret": ("acreate_realtime_translation_client_secret", {}), + "arealtime_translation_calls": ("arealtime_translation_calls", {}), + "acreate_fine_tuning_job": ("acreate_fine_tuning_job", {"training_file": "f"}), + "acancel_fine_tuning_job": ("acancel_fine_tuning_job", {"fine_tuning_job_id": "j"}), + "alist_fine_tuning_jobs": ("alist_fine_tuning_jobs", {}), + "aretrieve_fine_tuning_job": ("aretrieve_fine_tuning_job", {"fine_tuning_job_id": "j"}), + "aimage_edit": ("aimage_edit", {"image": b"x", "prompt": "p"}), + "allm_passthrough_route": ("allm_passthrough_route", {"method": "POST", "endpoint": "/v1/x"}), + "aocr": ("aocr", {"document": {}}), + "adecisions": ("adecisions", {}), + "asystemone": ("asystemone", {}), + "aextraction": ("aextraction", {}), + "avideo_generation": ("avideo_generation", {"prompt": "p"}), + "avideo_list": ("avideo_list", {}), + "avideo_status": ("avideo_status", {"video_id": "v"}), + "avideo_content": ("avideo_content", {"video_id": "v"}), + "avideo_remix": ("avideo_remix", {"video_id": "v", "prompt": "p"}), + "avideo_create_character": ("avideo_create_character", {}), + "avideo_get_character": ("avideo_get_character", {}), + "avideo_edit": ("avideo_edit", {"video_id": "v"}), + "avideo_extension": ("avideo_extension", {"video_id": "v"}), + "acreate_container": ("acreate_container", {"name": "c"}), + "alist_containers": ("alist_containers", {}), + "aretrieve_container": ("aretrieve_container", {"container_id": "cntr_native"}), + "adelete_container": ("adelete_container", {"container_id": "cntr_native"}), + "aupload_container_file": ("aupload_container_file", {"container_id": "cntr_native", "file": b"x"}), + "alist_container_files": ("alist_container_files", {"container_id": "cntr_native"}), + "aretrieve_container_file": ("aretrieve_container_file", {"container_id": "cntr_native", "file_id": "f"}), + "adelete_container_file": ("adelete_container_file", {"container_id": "cntr_native", "file_id": "f"}), + "aretrieve_container_file_content": ( + "aretrieve_container_file_content", + {"container_id": "cntr_native", "file_id": "f"}, + ), + "acreate_skill": ("acreate_skill", {}), + "alist_skills": ("alist_skills", {}), + "aget_skill": ("aget_skill", {"skill_id": "s"}), + "adelete_skill": ("adelete_skill", {"skill_id": "s"}), + "acreate_interaction": ("interactions.acreate", {"input": "x"}), + "avector_store_search": ("vs.asearch", {"vector_store_id": "v", "query": "q"}), + "avector_store_retrieve": ("vs.aretrieve", {"vector_store_id": "v"}), + "avector_store_list": ("vs.alist", {}), + "avector_store_update": ("vs.aupdate", {"vector_store_id": "v"}), + "avector_store_delete": ("vs.adelete", {"vector_store_id": "v"}), + "avector_store_file_create": ("vsf.acreate", {"vector_store_id": "v", "file_id": "f"}), + "avector_store_file_list": ("vsf.alist", {"vector_store_id": "v"}), + "avector_store_file_retrieve": ("vsf.aretrieve", {"vector_store_id": "v", "file_id": "f"}), + "avector_store_file_content": ("vsf.aretrieve_content", {"vector_store_id": "v", "file_id": "f"}), + "avector_store_file_update": ("vsf.aupdate", {"vector_store_id": "v", "file_id": "f"}), + "avector_store_file_delete": ("vsf.adelete", {"vector_store_id": "v", "file_id": "f"}), +} + +ROUTED_SYNC: Final[dict[str, tuple[str, dict[str, object]]]] = { + "responses": ("responses", {"input": "hi"}), + "generate_content": ("generate_content", {"contents": []}), + "generate_content_stream": ("generate_content_stream", {"contents": []}), + "ocr": ("ocr", {"document": {}}), + "decisions": ("decisions", {}), + "systemone": ("systemone", {}), + "extraction": ("extraction", {}), + "video_generation": ("video_generation", {"prompt": "p"}), + "video_list": ("video_list", {}), + "video_status": ("video_status", {"video_id": "v"}), + "video_content": ("video_content", {"video_id": "v"}), + "video_remix": ("video_remix", {"video_id": "v", "prompt": "p"}), + "create_container": ("create_container", {"name": "c"}), + "list_containers": ("list_containers", {}), + "retrieve_container": ("retrieve_container", {"container_id": "c"}), + "delete_container": ("delete_container", {"container_id": "c"}), + "vector_store_search": ("vs.search", {"vector_store_id": "v", "query": "q"}), + "vector_store_create": ("vs.create", {"name": "v"}), + "vector_store_retrieve": ("vs.retrieve", {"vector_store_id": "v"}), + "vector_store_list": ("vs.list", {}), + "vector_store_update": ("vs.update", {"vector_store_id": "v"}), + "vector_store_delete": ("vs.delete", {"vector_store_id": "v"}), +} + + +@pytest.mark.parametrize("endpoint", sorted(ROUTED_ASYNC)) +async def test_async_generic_endpoint_routes_to_the_group_deployment_with_litellm_metadata( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + op, request = ROUTED_ASYNC[endpoint] + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + response: Final = await _call(core, endpoint, {"model": "g", **request}) + assert [call.op for call in host.calls] == [op] + call: Final = host.calls[0] + assert (call.model, call.deployment_id, call.kwargs["api_base"], call.kwargs["custom_llm_provider"]) == ( + "openai/d1", + "d1", + "https://d1.example", + "openai", + ) + metadata: Final = _bucket(call.kwargs, "litellm_metadata") + assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1") + assert "metadata" not in call.kwargs + assert response._hidden_params["model_id"] == "d1" # pyright: ignore[reportAttributeAccessIssue] # the scripted response + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g" # pyright: ignore[reportAttributeAccessIssue] # the scripted response + + +@pytest.mark.parametrize("endpoint", sorted(ROUTED_SYNC)) +async def test_sync_generic_endpoint_routes_to_the_group_deployment_and_returns_the_response( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + op, request = ROUTED_SYNC[endpoint] + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + response: Final = getattr(core, endpoint)(model="g", **request) + assert not inspect.isawaitable(response) + assert [call.op for call in host.calls] == [op] + call: Final = host.calls[0] + assert (call.model, call.deployment_id, call.kwargs["api_base"], call.kwargs["custom_llm_provider"]) == ( + "openai/d1", + "d1", + "https://d1.example", + "openai", + ) + assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "g" + assert response is not None + + +@pytest.mark.parametrize("endpoint", ["ocr", "generate_content", "video_generation"]) +async def test_sync_generic_endpoints_set_no_routing_hidden_params( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + _, request = ROUTED_SYNC[endpoint] + core: Final = make_core(model_list=[deployment("g", "d1")]) + response: Final = getattr(core, endpoint)(model="g", **request) + assert response._hidden_params == {} + + +@pytest.mark.parametrize("endpoint", ["ocr", "generate_content"]) +async def test_sync_generic_endpoints_are_not_retried(make_core: MakeCore, host: EndpointHost, endpoint: str): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + _, request = ROUTED_SYNC[endpoint] + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) + with pytest.raises(litellm.InternalServerError): + getattr(core, endpoint)(model="g", **request) + assert len(host.calls) == 1 + + +async def test_async_generic_endpoint_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=2) + with pytest.raises(litellm.InternalServerError): + await core.aocr(model="g", document={}) + assert len(host.calls) == 3 + + +@pytest.mark.parametrize( + "endpoint", + [ + "aadapter_generate_content", + "video_create_character", + "video_get_character", + "video_edit", + "video_extension", + "upload_container_file", + "list_container_files", + "retrieve_container_file", + "delete_container_file", + "retrieve_container_file_content", + "get_interaction", + "delete_interaction", + "cancel_interaction", + ], +) +async def test_endpoints_without_a_dispatch_branch_resolve_to_none_without_calling_anything( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + """These are wrapped by factory_function's async wrapper, which has no branch for them.""" + core: Final = make_core(model_list=[deployment("g", "d1")]) + result: Final = getattr(core, endpoint)(model="g", container_id="c", video_id="v", interaction_id="i") + assert inspect.isawaitable(result) + assert await result is None + assert host.calls == [] + + +async def test_sync_create_interaction_is_awaited_and_fails_with_type_error(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(TypeError, match="can't be used in 'await' expression"): + await core.create_interaction(model="g", input="x") + assert [call.op for call in host.calls] == ["interactions.create"] + + +# --- request params, provider and timeouts ------------------------------------------------------ + + +@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) +async def test_generic_request_params_win_over_deployment_params( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1", temperature=0.1)]) + await _call(core, endpoint, {"model": "g", "document": {}, "temperature": 0.9}) + assert host.calls[0].kwargs["temperature"] == 0.9 + + +async def test_generic_call_always_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.aocr(model="g", document={}) + assert host.calls[0].model == "openai/d1" + + +async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", model="my-model", custom_llm_provider="hosted_vllm")]) + await core.aocr(model="g", document={}) + assert (host.calls[0].model, host.calls[0].kwargs["custom_llm_provider"]) == ("my-model", "hosted_vllm") + + +@pytest.mark.parametrize( + ("endpoint", "expected"), + [("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)], +) +async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync( + make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + request: Final = {**(ROUTED_ASYNC.get(endpoint) or ROUTED_SYNC[endpoint])[1], "model": "g"} + await _call(core, endpoint, request) + assert host.calls[0].kwargs["timeout"] == expected + + +@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) +async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) + await _call(core, endpoint, {"model": "g", "document": {}}) + assert host.calls[0].kwargs["timeout"] == 42 + + +@pytest.mark.parametrize( + ("endpoint", "request_body"), [("aresponses", {"input": "hi"}), ("agenerate_content", {"contents": []})] +) +async def test_streaming_generic_call_sends_the_deployment_stream_timeout_as_timeout( + make_core: MakeCore, host: EndpointHost, endpoint: str, request_body: dict[str, object] +): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)]) + await _call(core, endpoint, {"model": "g", "stream": True, **request_body}) + sent: Final = host.calls[0].kwargs + assert (sent["timeout"], sent["stream_timeout"]) == (3.0, 3.0) + + +async def test_non_streaming_aresponses_sends_the_deployment_timeout(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)]) + await core.aresponses(model="g", input="hi") + assert host.calls[0].kwargs["timeout"] == 11.0 + + +async def test_realtime_client_secret_session_model_follows_the_routed_deployment( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acreate_realtime_client_secret(model="g", session={"model": "g", "type": "realtime"}) + assert host.calls[0].kwargs["session"] == {"model": "openai/d1", "type": "realtime"} + + +@pytest.mark.parametrize( + ("session", "sent"), + [ + ( + {"type": "transcription"}, + {"type": "transcription", "audio": {"input": {"transcription": {"model": "openai/d1"}}}}, + ), + ( + {"type": "transcription", "input_audio_transcription": {"language": "en"}}, + {"type": "transcription", "input_audio_transcription": {"language": "en", "model": "openai/d1"}}, + ), + ({"type": "realtime"}, {"type": "realtime"}), + ], + ids=["transcription-no-model", "transcription-flat-model", "realtime-no-model"], +) +async def test_a_transcription_session_with_no_model_follows_the_routed_deployment( + make_core: MakeCore, host: EndpointHost, session: dict[str, object], sent: dict[str, object] +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acreate_realtime_translation_client_secret(model="g", session=session) + assert host.calls[0].kwargs["session"] == sent + + +@pytest.mark.parametrize("endpoint", ["acreate_realtime_translation_client_secret", "arealtime_translation_calls"]) +async def test_a_realtime_translation_endpoint_keeps_the_callers_client( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + client: Final = object() + await _call(core, endpoint, {"model": "g", "client": client}) + assert host.calls[0].kwargs["client"] is client + + +def _realtime_session_logging(details: Mapping[str, object]) -> LiteLLMLogging: + session_logging: Final = LiteLLMLogging( + model="gpt-realtime-2.1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="realtime-session", + ) + session_logging.model_call_details.update(details) + return session_logging + + +@pytest.mark.parametrize( + ("details", "counted"), + [ + ({REALTIME_SESSION_SUCCESS_LOGGED_KEY: True}, 1), + ({}, None), + ({REALTIME_SESSION_SUCCESS_LOGGED_KEY: True, REALTIME_SESSION_ENDED_ABNORMALLY_KEY: True}, None), + ], + ids=["clean-session", "no-success-logged", "ended-abnormally"], +) +async def test_a_realtime_session_counts_a_deployment_success_only_when_it_ended_cleanly( + make_core: MakeCore, host: EndpointHost, details: dict[str, object], counted: int | None +): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + await core._arealtime(model="g", litellm_logging_obj=_realtime_session_logging(details)) + assert core.health().store.get_cache("d1:successes", local_only=True) == counted + + +async def test_generic_endpoint_with_an_unknown_group_raises_without_calling_the_host( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + with pytest.raises(litellm.BadRequestError, match="model=nope"): + await core.agenerate_content(model="nope", contents=[]) + assert host.calls == [] + + +# --- Responses API ids pin the deployment ----------------------------------------------------- + +TWO_DEPLOYMENTS: Final = [deployment("g", "d1"), deployment("g", "d2")] +ENCODED_RESPONSE_ID: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "d2", "resp_abc") # pyright: ignore[reportPrivateUsage] # litellm's encoder + + +@pytest.mark.parametrize( + "endpoint", ["aget_responses", "acancel_responses", "acompact_responses", "adelete_responses", "alist_input_items"] +) +async def test_a_litellm_response_id_pins_the_deployment_it_encodes( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(5): + await _call(core, endpoint, {"response_id": ENCODED_RESPONSE_ID}) + assert host.deployments_called() == ["d2"] * 5 + + +async def test_a_litellm_response_id_overrides_the_request_model(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(5): + await core.adelete_responses(response_id=ENCODED_RESPONSE_ID, model="g") + assert host.deployments_called() == ["d2"] * 5 + assert _bucket(host.calls[0].kwargs, "litellm_metadata")["model_group"] == "d2" + + +async def test_the_response_id_reaches_the_host_still_encoded(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.aget_responses(response_id=ENCODED_RESPONSE_ID) + assert host.calls[0].kwargs["response_id"] == ENCODED_RESPONSE_ID + + +async def test_a_plain_response_id_without_a_model_fails_with_type_error(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + with pytest.raises(TypeError, match="missing 1 required positional argument: 'model'"): + await core.aget_responses(response_id="resp_plain") + assert host.calls == [] + + +# --- container ids ------------------------------------------------------------------------ + +ENCODED_CONTAINER_ID: Final = ResponsesAPIRequestUtils.build_container_id("azure", "d2", "cntr_upstream") + + +async def test_a_litellm_container_id_pins_its_deployment_and_is_sent_decoded(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(5): + await core.aretrieve_container(container_id=ENCODED_CONTAINER_ID) + assert host.deployments_called() == ["d2"] * 5 + assert {call.kwargs["container_id"] for call in host.calls} == {"cntr_upstream"} + + +async def test_a_routed_container_call_sends_the_deployment_provider_not_the_decoded_one( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.aretrieve_container(container_id=ENCODED_CONTAINER_ID, custom_llm_provider="openai") + assert host.calls[0].kwargs["custom_llm_provider"] == "openai" + + +async def test_a_native_container_id_routes_by_the_forwarded_model_id(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(5): + await core.aretrieve_container(container_id="cntr_native", model_id=" d1 ") + assert host.deployments_called() == ["d1"] * 5 + assert host.calls[0].kwargs["container_id"] == "cntr_native" + + +async def test_a_native_container_id_without_a_model_calls_the_handler_directly( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.aretrieve_container(container_id="cntr_native", custom_llm_provider="azure") + call: Final = host.calls[0] + assert (call.deployment_id, call.kwargs["container_id"], call.kwargs["custom_llm_provider"]) == ( + None, + "cntr_native", + "azure", + ) + assert "litellm_metadata" not in call.kwargs + + +async def test_create_container_with_an_unknown_model_passes_through_with_that_model( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.acreate_container(name="c", model="not-a-group") + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id) == ("acreate_container", "not-a-group", None) + + +async def test_create_container_without_a_model_calls_the_handler_directly(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.acreate_container(name="c") + call: Final = host.calls[0] + assert (call.op, call.deployment_id, "model" in call.kwargs) == ("acreate_container", None, False) + + +# --- video ids ----------------------------------------------------------------------------- + +ENCODED_VIDEO_ID: Final = encode_video_id_with_provider("video_up", "openai", model_id="d2", deployment_id="d2") + + +@pytest.mark.parametrize( + "endpoint", ["avideo_status", "avideo_content", "avideo_remix", "avideo_edit", "avideo_extension"] +) +async def test_a_litellm_video_id_pins_the_deployment_that_created_it( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(6): + await _call(core, endpoint, {"model": "g", "video_id": ENCODED_VIDEO_ID, "prompt": "p"}) + assert host.deployments_called() == ["d2"] * 6 + + +async def test_a_video_id_on_a_video_generation_call_does_not_pin(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(30): + await core.avideo_generation(model="g", prompt="p", video_id=ENCODED_VIDEO_ID) + assert set(host.deployments_called()) == {"d1", "d2"} + + +async def test_sync_video_status_pins_the_deployment_of_the_video_id(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(6): + core.video_status(model="g", video_id=ENCODED_VIDEO_ID) + assert host.deployments_called() == ["d2"] * 6 + + +# --- LLM pass-through ---------------------------------------------------------------------- + + +async def test_llm_passthrough_with_an_unknown_model_calls_the_handler_with_that_model( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.allm_passthrough_route(model="not-a-group", method="POST", endpoint="/model/not-a-group/invoke") + call: Final = host.calls[0] + assert (call.model, call.deployment_id, call.kwargs["endpoint"]) == ( + "not-a-group", + None, + "/model/not-a-group/invoke", + ) + + +async def test_llm_passthrough_rewrites_the_group_name_in_the_endpoint_to_the_provider_model( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="bedrock/us.anthropic.claude")]) + await core.allm_passthrough_route(model="g", method="POST", endpoint="/model/g/invoke") + call: Final = host.calls[0] + assert (call.model, call.kwargs["endpoint"], call.kwargs["custom_llm_provider"]) == ( + "bedrock/us.anthropic.claude", + "/model/us.anthropic.claude/invoke", + "bedrock", + ) + + +async def test_llm_passthrough_forwards_the_request_client(make_core: MakeCore, host: EndpointHost): + client: Final = object() + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.allm_passthrough_route(model="g", method="POST", endpoint="/v1/x", client=client) + assert host.calls[0].kwargs["client"] is client + + +# --- interactions and managed agents ---------------------------------------------------------- + + +async def test_interactions_get_without_a_model_calls_the_handler_with_the_given_provider( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.aget_interaction(interaction_id="i", custom_llm_provider="vertex_ai") + call: Final = host.calls[0] + assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"]) == ("interactions.aget", None, "vertex_ai") + + +@pytest.mark.parametrize("endpoint", ["aget_interaction", "adelete_interaction", "acancel_interaction"]) +async def test_interactions_default_to_the_gemini_provider(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await _call(core, endpoint, {"interaction_id": "i"}) + assert host.calls[0].kwargs["custom_llm_provider"] == "gemini" + + +async def test_interaction_with_an_agent_named_as_the_model_is_not_routed(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("ag", "d1")]) + await core.acreate_interaction(agent="ag", model="ag", input="x") + call: Final = host.calls[0] + assert (call.deployment_id, call.kwargs["model"], call.kwargs["custom_llm_provider"]) == (None, None, "gemini") + + +@pytest.mark.parametrize( + ("endpoint", "op"), + [ + ("acreate_agent", "agents.acreate"), + ("alist_agents", "agents.alist"), + ("aget_agent", "agents.aget"), + ("adelete_agent", "agents.adelete"), + ("alist_agent_versions", "agents.alist_versions"), + ("create_agent", "agents.create"), + ("list_agents", "agents.list"), + ("get_agent", "agents.get"), + ("delete_agent", "agents.delete"), + ("list_agent_versions", "agents.list_versions"), + ], +) +async def test_managed_agent_endpoints_call_the_handler_directly_with_gemini_by_default( + make_core: MakeCore, host: EndpointHost, endpoint: str, op: str +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await _call(core, endpoint, {"name": "a", "model": "g"}) + call: Final = host.calls[0] + assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"], call.kwargs["model"]) == ( + op, + None, + "gemini", + "g", + ) + + +@pytest.mark.parametrize("endpoint", ["acreate_agent", "create_agent"]) +async def test_managed_agent_endpoints_keep_the_given_provider(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await _call(core, endpoint, {"name": "a", "custom_llm_provider": "vertex_ai"}) + assert host.calls[0].kwargs["custom_llm_provider"] == "vertex_ai" + + +# --- assistants ---------------------------------------------------------------------------- + +ASSISTANT_ENDPOINTS: Final = ( + "acreate_assistants", + "adelete_assistant", + "aget_assistants", + "acreate_thread", + "aget_thread", + "a_add_message", + "aget_messages", + "arun_thread", +) + + +@pytest.mark.parametrize("endpoint", ASSISTANT_ENDPOINTS) +async def test_assistants_endpoints_without_a_provider_or_config_raise( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + with pytest.raises(Exception, match=re.escape("'custom_llm_provider' must be set. Either via:")): + await getattr(core, endpoint)() + assert host.calls == [] + + +@pytest.mark.parametrize("endpoint", ASSISTANT_ENDPOINTS) +async def test_assistants_endpoints_use_the_assistants_config(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core( + model_list=[deployment("g", "d1")], + assistants_config={ + "custom_llm_provider": "azure", + "litellm_params": {"api_key": "k", "api_base": "https://az"}, + }, + ) + await getattr(core, endpoint)(thread_id="t") + call: Final = host.calls[0] + assert (call.op, call.kwargs["custom_llm_provider"], call.kwargs["api_key"], call.kwargs["api_base"]) == ( + endpoint, + "azure", + "k", + "https://az", + ) + assert (call.kwargs["client"], call.kwargs["thread_id"]) == (None, "t") + + +async def test_assistants_endpoint_with_an_explicit_provider_ignores_the_config( + make_core: MakeCore, host: EndpointHost +): + client: Final = object() + core: Final = make_core( + model_list=[deployment("g", "d1")], + assistants_config={"custom_llm_provider": "azure", "litellm_params": {"api_key": "k"}}, + ) + await core.arun_thread(custom_llm_provider="openai", client=client, thread_id="t") + call: Final = host.calls[0] + assert (call.kwargs["custom_llm_provider"], call.kwargs["client"], "api_key" in call.kwargs) == ( + "openai", + client, + False, + ) + + +# --- vector stores --------------------------------------------------------------------------- + + +async def test_vector_store_search_without_a_model_gets_the_router_and_overrides_a_client_router( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.avector_store_search(vector_store_id="v", query="q", router="client-value") + call: Final = host.calls[0] + assert (call.op, call.deployment_id, call.kwargs["router"]) == ("vs.asearch", None, core) + assert "_direct_vector_store_embedding_executor" in call.kwargs + + +async def test_sync_vector_store_search_without_a_model_gets_the_router(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + core.vector_store_search(vector_store_id="v", query="q") + assert host.calls[0].kwargs["router"] is core + + +async def test_routed_vector_store_search_gets_the_embedding_executor_but_not_the_router( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.avector_store_search(model="g", vector_store_id="v", query="q") + sent: Final = host.calls[0].kwargs + assert ("router" in sent, "_direct_vector_store_embedding_executor" in sent) == (False, True) + + +async def test_vector_store_retrieve_without_a_model_calls_the_handler_directly( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + await core.avector_store_retrieve(vector_store_id="v", custom_llm_provider="openai") + call: Final = host.calls[0] + assert (call.deployment_id, call.kwargs["custom_llm_provider"], "router" in call.kwargs) == (None, "openai", False) + + +async def test_sync_vector_store_create_without_a_model_passes_the_provider(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + core.vector_store_create(name="v", custom_llm_provider="openai") + call: Final = host.calls[0] + assert (call.op, call.deployment_id, call.kwargs["custom_llm_provider"]) == ("vs.create", None, "openai") + + +@pytest.mark.parametrize( + ("endpoint", "op"), + [ + ("vector_store_file_create", "vsf.create"), + ("vector_store_file_list", "vsf.list"), + ("vector_store_file_retrieve", "vsf.retrieve"), + ("vector_store_file_content", "vsf.retrieve_content"), + ("vector_store_file_update", "vsf.update"), + ("vector_store_file_delete", "vsf.delete"), + ], +) +async def test_sync_vector_store_file_endpoints_never_route_even_with_a_model( + make_core: MakeCore, host: EndpointHost, endpoint: str, op: str +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + getattr(core, endpoint)(model="g", vector_store_id="v", file_id="f") + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"], call.kwargs["client"]) == ( + op, + "g", + None, + None, + None, + ) + + +async def test_avector_store_create_with_a_model_routes_with_metadata_and_no_model_group( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + await core.avector_store_create(model="g", name="n") + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id, call.kwargs["custom_llm_provider"]) == ( + "vs.acreate", + "openai/d1", + "d1", + "openai", + ) + metadata: Final = _bucket(call.kwargs, "metadata") + assert (metadata["deployment"], "model_group" in metadata) == ("openai/d1", False) + + +async def test_avector_store_create_without_a_model_calls_the_handler_directly(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.avector_store_create(model=None, name="n") + call: Final = host.calls[0] + assert (call.op, call.deployment_id) == ("vs.acreate", None) + + +async def test_avector_store_create_is_not_retried(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) + with pytest.raises(litellm.InternalServerError): + await core.avector_store_create(model="g", name="n") + assert len(host.calls) == 1 + + +# --- search tools --------------------------------------------------------------------------- + +SEARCH_TOOLS: Final = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily", "api_key": "tk"}}] + + +async def test_asearch_calls_the_named_search_tool_with_its_litellm_params(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], search_tools=SEARCH_TOOLS) + response: Final = await core.asearch(model="s", query="q") + call: Final = host.calls[0] + assert (call.op, call.kwargs["search_provider"], call.kwargs["api_key"], call.kwargs["query"]) == ( + "asearch", + "tavily", + "tk", + "q", + ) + assert _bucket(call.kwargs, "litellm_metadata")["model_group"] == "s" + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "s" # pyright: ignore[reportAttributeAccessIssue] # the scripted response + + +async def test_asearch_with_an_unknown_tool_raises_value_error(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], search_tools=SEARCH_TOOLS, num_retries=0) + with pytest.raises(ValueError, match=re.escape("Search tool 'g' not found in router.search_tools")): + await core.asearch(model="g", query="q") + assert host.calls == [] + + +# --- guardrails --------------------------------------------------------------------------- + +GUARDRAILS: Final = [{"guardrail_name": "gr", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}] + + +async def test_aguardrail_runs_the_function_with_the_selected_guardrail(make_core: MakeCore, host: ScriptedHost): + seen: Final[list[Mapping[str, object]]] = [] + + async def guard(**kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + seen.append(kwargs) + return "verdict" + + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS) + assert await core.aguardrail(guardrail_name="gr", original_function=guard, data={"x": 1}) == "verdict" + assert sorted(seen[0]) == ["data", "litellm_metadata", "litellm_trace_id", "selected_guardrail"] + assert seen[0]["selected_guardrail"] == GUARDRAILS[0] + assert _bucket(seen[0], "litellm_metadata")["model_group"] == "gr" + assert host.calls == [] + + +async def test_aguardrail_with_an_unknown_name_raises_value_error(make_core: MakeCore, host: ScriptedHost): + async def guard(**_kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + return "verdict" + + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS, num_retries=0) + with pytest.raises(ValueError, match="No guardrail found with name: nope"): + await core.aguardrail(guardrail_name="nope", original_function=guard) + + +async def test_aguardrail_does_not_retry_a_failing_guardrail_function(make_core: MakeCore, host: ScriptedHost): + """A guardrail name is no model group (model_group_size 0), so the router num_retries do not apply.""" + attempts: Final[list[int]] = [] + + async def guard(**_kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + attempts.append(1) + raise litellm.InternalServerError(message="boom", llm_provider="", model="gr") + + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS, num_retries=2) + with pytest.raises(litellm.InternalServerError, match="boom"): + await core.aguardrail(guardrail_name="gr", original_function=guard) + assert len(attempts) == 1 diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py new file mode 100644 index 00000000000..8db0d6bea12 --- /dev/null +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -0,0 +1,648 @@ +""" +Characterization of the inference endpoints (completion, text completion, embedding, image, +speech, transcription, rerank, adapter completion, moderation): which host op each invokes and +which keyword arguments the core builds for it. + +Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core. +""" + +import asyncio +import io +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, timeout + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] + + +@pytest.fixture +def host() -> EndpointHost: + return EndpointHost() + + +def _sent(host: ScriptedHost, index: int = -1) -> Mapping[str, object]: + return host.calls[index].kwargs + + +def _bucket(kwargs: Mapping[str, object], name: str = "metadata") -> Mapping[str, object]: + bucket: Final = kwargs[name] + assert isinstance(bucket, Mapping) + return bucket + + +# --- host op per endpoint ------------------------------------------------------------------- + + +async def test_aadapter_completion_invokes_the_adapter_host_op_with_the_adapter_id( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.aadapter_completion(adapter_id="my-adapter", model="g", messages=MESSAGES) + call: Final = host.calls[0] + assert (call.op, call.model, call.kwargs["adapter_id"], call.deployment_id, call.model_group) == ( + "aadapter_completion", + "openai/d1", + "my-adapter", + "d1", + "g", + ) + + +async def test_amoderation_routes_a_known_group_through_its_deployment(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + await core.amoderation(model="g", input="x") + call: Final = host.calls[0] + assert (call.op, call.model, call.kwargs["api_base"], call.deployment_id, call.model_group) == ( + "amoderation", + "openai/d1", + "https://d1.example", + "d1", + "g", + ) + assert "litellm_metadata" not in call.kwargs + + +async def test_amoderation_calls_the_host_directly_when_the_model_is_not_a_group( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.amoderation(model="omni-moderation-latest", input="x") + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id) == ("amoderation", "omni-moderation-latest", None) + assert _bucket(call.kwargs)["model_group"] == "omni-moderation-latest" + + +async def test_amoderation_deployment_params_overwrite_the_request_params(make_core: MakeCore, host: EndpointHost): + """The deployment's litellm_params are applied after the request's (kwargs.update(data)).""" + core: Final = make_core(model_list=[deployment("g", "d1", temperature=0.5)]) + await core.amoderation(model="g", input="x", temperature=1.0) + assert _sent(host)["temperature"] == 0.5 + + +async def test_amoderation_with_a_request_api_key_keeps_the_deployment_key_under_a_request_scoped_id( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.amoderation(model="g", input="x", api_key="sk-request") + sent: Final = _sent(host) + model_info: Final = _bucket(sent, "model_info") + assert sent["api_key"] == "sk-test" + assert (model_info["original_model_id"], model_info["id"] != "d1") == ("d1", True) + + +async def test_amoderation_is_not_retried(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) + with pytest.raises(litellm.InternalServerError): + await core.amoderation(model="g", input="x") + assert len(host.calls) == 1 + assert host.sleeps == [] + + +async def test_arerank_sends_the_builtin_input_function_as_input(make_core: MakeCore, host: EndpointHost): + """arerank sets kwargs["input"] = input, the Python builtin, and it reaches the host.""" + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.arerank(model="g", query="q", documents=["a"]) + sent: Final = _sent(host) + assert (sent["input"], sent["query"], sent["documents"]) == (input, "q", ["a"]) + + +@pytest.mark.parametrize(("request_voice", "expected"), [(None, "deployment-voice"), ("alloy", "alloy")]) +async def test_aspeech_voice_defaults_to_the_deployment_voice( + make_core: MakeCore, host: EndpointHost, request_voice: str | None, expected: str +): + core: Final = make_core(model_list=[deployment("g", "d1", voice="deployment-voice")]) + await core.aspeech(model="g", input="hello", voice=request_voice) + assert (_sent(host)["voice"], _sent(host)["input"]) == (expected, "hello") + + +async def test_aspeech_sends_no_caching_flag(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], cache_responses=True) + await core.aspeech(model="g", input="hello", voice="alloy") + assert "caching" not in _sent(host) + + +async def test_atranscription_forwards_the_same_file_object(make_core: MakeCore, host: EndpointHost): + audio: Final = io.BytesIO(b"audio") + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.atranscription(model="g", file=audio) + assert _sent(host)["file"] is audio + + +@pytest.mark.parametrize( + "endpoint", + ["acompletion", "atext_completion", "aembedding", "aimage_generation", "atranscription", "arerank", "completion"], +) +async def test_cache_responses_reaches_the_host_as_caching(make_core: MakeCore, host: EndpointHost, endpoint: str): + core: Final = make_core(model_list=[deployment("g", "d1")], cache_responses=True) + requests: Final[dict[str, dict[str, object]]] = { + "acompletion": {"messages": MESSAGES}, + "completion": {"messages": MESSAGES}, + "atext_completion": {"prompt": "hi"}, + "aembedding": {"input": ["a"]}, + "aimage_generation": {"prompt": "a cat"}, + "atranscription": {"file": io.BytesIO(b"x")}, + "arerank": {"query": "q", "documents": ["a"]}, + } + await core.call(endpoint, {"model": "g", **requests[endpoint]}) + assert _sent(host)["caching"] is True + + +@pytest.mark.parametrize("endpoint", ["acompletion", "completion"]) +async def test_cache_responses_turned_on_after_construction_reaches_the_host( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + """The proxy turns ``cache_responses`` on through ``apply`` once its cache exists.""" + from litellm.router_api import ChangeSet + + core: Final = make_core(model_list=[deployment("g", "d1")]) + _ = core.apply(ChangeSet(assign={"cache_responses": True})) + await core.call(endpoint, {"model": "g", "messages": MESSAGES}) + assert core.settings().cache_responses is True + assert _sent(host)["caching"] is True + + +# --- sync text completion and image generation ---------------------------------------------- + + +async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4) + core.text_completion(model="g", prompt="p") + sent: Final = _sent(host) + assert sent["num_retries"] == 4 + assert "model_info" not in sent + assert "litellm_trace_id" not in sent + assert sent.get("timeout") is None + metadata: Final = _bucket(sent) + assert metadata["model_group"] == "g" + assert "deployment" not in metadata + + +async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4) + core.text_completion(model="g", prompt="p", num_retries=1) + assert _sent(host)["num_retries"] == 1 + + +async def test_sync_text_completion_merges_default_metadata_into_request_metadata( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"x": 1}, "user": "u"} + ) + core.text_completion(model="g", prompt="p", metadata={"y": 2}) + sent: Final = _sent(host) + metadata: Final = _bucket(sent) + assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u") + + +async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model( + make_core: MakeCore, host: EndpointHost +): + """The request's model="g" is spread after the deployment params, so it replaces "openai/d1".""" + core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) + core.text_completion(model="g", prompt="p") + call: Final = host.calls[0] + assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example") + + +async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost): + host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) + with pytest.raises(litellm.InternalServerError): + core.text_completion(model="g", prompt="p") + assert len(host.calls) == 1 + assert host.sleeps == [] + + +async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + core.image_generation(model="g", prompt="p") + sent: Final = _sent(host) + assert "litellm_trace_id" not in sent + assert (_bucket(sent)["model_group"], _bucket(sent)["deployment"], sent["prompt"]) == ("g", "openai/d1", "p") + + +@pytest.mark.parametrize("endpoint", ["acompletion", "atext_completion", "aembedding", "aresponses"]) +async def test_async_endpoints_generate_a_trace_id_unless_the_request_sends_one( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + request: Final[dict[str, object]] = ( + {"messages": MESSAGES} + if endpoint == "acompletion" + else {"prompt": "p"} + if endpoint == "atext_completion" + else {"input": ["a"]} + if endpoint == "aembedding" + else {"input": "hi"} + ) + await core.call(endpoint, {"model": "g", **request}) + await core.call(endpoint, {"model": "g", "litellm_trace_id": "trace-1", **request}) + generated: Final = _sent(host, 0)["litellm_trace_id"] + assert isinstance(generated, str) and len(generated) == 36 + assert _sent(host, 1)["litellm_trace_id"] == "trace-1" + + +# --- routing metadata --------------------------------------------------------------------- + + +async def test_a_model_group_alias_is_recorded_as_the_model_group_and_its_alias( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], model_group_alias={"al": "g"}) + await core.acompletion(model="al", messages=MESSAGES) + metadata: Final = _bucket(_sent(host)) + assert (metadata["model_group"], metadata["model_group_alias"], metadata["deployment_model_name"]) == ( + "al", + "al", + "g", + ) + + +async def test_a_request_for_a_deployment_id_routes_to_that_deployment(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("g", "d2")]) + for _ in range(4): + await core.acompletion(model="d2", messages=MESSAGES) + assert host.deployments_called() == ["d2"] * 4 + assert _bucket(_sent(host))["model_group"] == "d2" + + +async def test_deployment_tags_and_credential_name_join_the_request_tags(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", tags=["t1"], litellm_credential_name="cred")]) + await core.acompletion(model="g", messages=MESSAGES, metadata={"tags": ["r1"]}) + sent: Final = _sent(host) + assert _bucket(sent)["tags"] == ["r1", "t1", "Credential: cred"] + assert sent["litellm_credential_name"] == "cred" + + +async def test_daily_reset_time_is_not_sent_to_the_host(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acompletion(model="g", messages=MESSAGES, daily_reset_time="00:00") + assert "daily_reset_time" not in _sent(host) + + +async def test_deployment_model_id_param_is_sent_as_model_id(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", model_id="arn:pinned")]) + await core.acompletion(model="g", messages=MESSAGES) + core.text_completion(model="g", prompt="p") + assert [call.kwargs["model_id"] for call in host.calls] == ["arn:pinned", "arn:pinned"] + + +async def test_deployment_model_id_reaches_the_host_even_when_clientside_auth_allows_model_id( + make_core: MakeCore, host: EndpointHost +): + """The pin is skipped, but the deployment's litellm_params (with model_id) are still spread into the call.""" + core: Final = make_core( + model_list=[deployment("g", "d1", model_id="arn:pinned", configurable_clientside_auth_params=["model_id"])] + ) + await core.acompletion(model="g", messages=MESSAGES) + assert _sent(host)["model_id"] == "arn:pinned" + + +async def test_request_model_id_wins_over_a_clientside_configurable_deployment_model_id( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("g", "d1", model_id="arn:pinned", configurable_clientside_auth_params=["model_id"])] + ) + await core.acompletion(model="g", messages=MESSAGES, model_id="arn:request") + assert _sent(host)["model_id"] == "arn:request" + + +# --- default litellm params ------------------------------------------------------------------- + + +async def test_default_litellm_params_metadata_merges_into_request_metadata(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}}) + await core.acompletion(model="g", messages=MESSAGES, metadata={"trace": "x"}) + metadata: Final = _bucket(_sent(host)) + assert (metadata["team"], metadata["trace"], metadata["model_group"]) == ("t", "x", "g") + + +async def test_default_litellm_params_metadata_overwrites_the_same_request_metadata_key( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}}) + await core.acompletion(model="g", messages=MESSAGES, metadata={"team": "request"}) + assert _bucket(_sent(host))["team"] == "t" + + +async def test_default_litellm_params_metadata_goes_to_litellm_metadata_for_generic_endpoints( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}}) + await core.aresponses(model="g", input="hi") + sent: Final = _sent(host) + assert _bucket(sent, "litellm_metadata")["team"] == "t" + assert "metadata" not in sent + + +async def test_default_litellm_params_with_none_values_are_not_sent(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"user": None}) + await core.acompletion(model="g", messages=MESSAGES) + assert "user" not in _sent(host) + + +# --- tools ---------------------------------------------------------------------------------- + +DEPLOYMENT_TOOL: Final = {"type": "function", "function": {"name": "dep", "parameters": {}}} +REQUEST_TOOL: Final = {"type": "function", "function": {"name": "req", "parameters": {}}} + + +async def test_deployment_tools_come_before_request_tools(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])]) + await core.acompletion(model="g", messages=MESSAGES, tools=[REQUEST_TOOL]) + assert _sent(host)["tools"] == [DEPLOYMENT_TOOL, REQUEST_TOOL] + + +async def test_deployment_tools_are_sent_when_the_request_has_none(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])]) + await core.acompletion(model="g", messages=MESSAGES) + assert _sent(host)["tools"] == [DEPLOYMENT_TOOL] + + +@pytest.mark.parametrize(("request_choice", "expected"), [(None, "required"), ("none", "none")]) +async def test_tool_choice_comes_from_the_request_else_the_deployment( + make_core: MakeCore, host: EndpointHost, request_choice: str | None, expected: str +): + core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL], tool_choice="required")]) + extra: Final = {} if request_choice is None else {"tool_choice": request_choice} + await core.acompletion(model="g", messages=MESSAGES, **extra) + assert _sent(host)["tool_choice"] == expected + + +async def test_merging_tools_does_not_change_the_deployment_tools(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", tools=[DEPLOYMENT_TOOL])]) + await core.acompletion(model="g", messages=MESSAGES, tools=[REQUEST_TOOL]) + await core.acompletion(model="g", messages=MESSAGES) + assert _sent(host)["tools"] == [DEPLOYMENT_TOOL] + + +# --- reasoning effort ----------------------------------------------------------------------- + + +async def test_request_reasoning_effort_drops_the_deployment_thinking_param(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", thinking={"type": "enabled", "budget_tokens": 100})]) + await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + sent: Final = _sent(host) + assert (sent["reasoning_effort"], "thinking" in sent) == ("low", False) + + +async def test_deployment_thinking_is_kept_without_a_request_reasoning_effort(make_core: MakeCore, host: EndpointHost): + thinking: Final = {"type": "enabled", "budget_tokens": 100} + core: Final = make_core(model_list=[deployment("g", "d1", thinking=thinking)]) + await core.acompletion(model="g", messages=MESSAGES) + assert _sent(host)["thinking"] == thinking + + +async def test_request_reasoning_effort_strips_effort_from_deployment_extra_body( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[ + deployment( + "g", + "d1", + extra_body={"reasoning_effort": "high", "thinking": {"type": "enabled"}, "other": 1}, + reasoning={"effort": "high", "summary": "auto"}, + ) + ] + ) + await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + sent: Final = _sent(host) + assert sent["extra_body"] == {"other": 1} + assert sent["reasoning"] == {"summary": "auto"} + + +async def test_request_reasoning_effort_drops_an_extra_body_left_empty(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", extra_body={"reasoning_effort": "high"})]) + await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + assert "extra_body" not in _sent(host) + + +async def test_reasoning_override_does_not_change_the_deployment_config(make_core: MakeCore, host: EndpointHost): + thinking: Final = {"type": "enabled", "budget_tokens": 100} + core: Final = make_core(model_list=[deployment("g", "d1", thinking=thinking)]) + await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + await core.acompletion(model="g", messages=MESSAGES) + assert _sent(host)["thinking"] == thinking + + +CLASSIFIER_METADATA: Final = {"internal_call_origin": "autorouter_classifier"} + + +async def test_classifier_call_drops_reasoning_effort_the_deployment_does_not_support( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="openai/gpt-3.5-turbo")]) + body: Final[dict[str, object]] = {"reasoning_effort": "low", "keep": 1} + await core.acompletion( + model="g", + messages=MESSAGES, + reasoning_effort="low", + metadata=dict(CLASSIFIER_METADATA), + proxy_server_request={"body": body}, + ) + sent: Final = _sent(host) + assert "reasoning_effort" not in sent + assert sent["proxy_server_request"] == {"body": {"keep": 1}} + + +async def test_non_classifier_call_keeps_reasoning_effort_on_a_deployment_without_support( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="openai/gpt-3.5-turbo")]) + await core.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + assert _sent(host)["reasoning_effort"] == "low" + + +async def test_classifier_call_keeps_reasoning_effort_on_a_deployment_with_support( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="openai/o3-mini")]) + await core.acompletion( + model="g", messages=MESSAGES, reasoning_effort="low", litellm_metadata=dict(CLASSIFIER_METADATA) + ) + assert _sent(host)["reasoning_effort"] == "low" + + +# --- silent experiments --------------------------------------------------------------------- + + +async def _settle() -> None: + for _ in range(20): + await asyncio.sleep(0) + + +async def test_silent_model_mirrors_acompletion_to_the_silent_group(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + await core.acompletion(model="g", messages=MESSAGES, metadata={"trace": "t"}) + await _settle() + assert sorted(host.deployments_called("acompletion")) == ["d1", "s1"] + primary: Final = next(call for call in host.calls if call.deployment_id == "d1") + shadow: Final = next(call for call in host.calls if call.deployment_id == "s1") + assert "silent_model" not in primary.kwargs + assert "is_silent_experiment" not in _bucket(primary.kwargs) + shadow_metadata: Final = _bucket(shadow.kwargs) + assert (shadow_metadata["is_silent_experiment"], shadow_metadata["model_group"], shadow_metadata["trace"]) == ( + True, + "shadow", + "t", + ) + + +async def test_silent_model_list_mirrors_to_every_listed_group(make_core: MakeCore, host: EndpointHost): + core: Final = make_core( + model_list=[ + deployment("g", "d1", silent_model=["shadow-a", "shadow-b"]), + deployment("shadow-a", "sa"), + deployment("shadow-b", "sb"), + ] + ) + await core.acompletion(model="g", messages=MESSAGES) + await _settle() + assert sorted(host.deployments_called("acompletion")) == ["d1", "sa", "sb"] + + +async def test_a_failing_silent_experiment_does_not_fail_the_primary_call(make_core: MakeCore, host: EndpointHost): + host.answer("openai/s1", litellm.BadRequestError(message="shadow broke", llm_provider="openai", model="s1")) + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + response: Final = await core.acompletion(model="g", messages=MESSAGES) + await _settle() + assert response is not None + assert "s1" in host.deployments_called("acompletion") + + +async def test_a_malformed_silent_model_is_ignored(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", silent_model=5)]) + await core.acompletion(model="g", messages=MESSAGES) + await _settle() + assert host.deployments_called() == ["d1"] + assert "silent_model" not in _sent(host) + + +# --- timeouts ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("stream", "expected"), [(False, (99.0, None)), (True, (99.0, 33.0))]) +async def test_service_tier_timeouts_come_from_the_deployment_tier_maps( + make_core: MakeCore, host: EndpointHost, stream: bool, expected: tuple[float, float | None] +): + core: Final = make_core( + model_list=[ + deployment( + "g", "d1", timeout=11, timeout_by_service_tier={"flex": 99}, stream_timeout_by_service_tier={"flex": 33} + ) + ] + ) + await core.acompletion(model="g", messages=MESSAGES, service_tier="flex", stream=stream) + sent: Final = _sent(host) + assert (sent["timeout"], sent.get("stream_timeout")) == expected + + +async def test_router_stream_timeout_applies_to_streaming_acompletion(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], stream_timeout=5) + await core.acompletion(model="g", messages=MESSAGES, stream=True) + sent: Final = _sent(host) + assert (sent["timeout"], sent["stream_timeout"]) == (11.0, 5.0) + + +async def test_non_streaming_acompletion_sends_no_stream_timeout(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", stream_timeout=3)]) + await core.acompletion(model="g", messages=MESSAGES) + assert "stream_timeout" not in _sent(host) or _sent(host)["stream_timeout"] == 3 + + +async def test_aembedding_uses_the_request_timeout_over_the_deployment_timeout(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)]) + await core.aembedding(model="g", input=["a"], request_timeout=4) + assert _sent(host)["timeout"] == 4 + + +async def test_timeout_message_carries_deployment_timeouts_when_router_debug_is_exposed( + make_core: MakeCore, host: EndpointHost +): + host.set_dynamic(expose_router_debug_in_errors=True) + host.answer("openai/d1", timeout("d1")) + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11, stream_timeout=3)], num_retries=0) + with pytest.raises(litellm.Timeout) as raised: + await core.acompletion(model="g", messages=MESSAGES) + assert "Deployment Info: request_timeout: None\ntimeout: 11.0\nstream_timeout: 3.0" in str(raised.value) + + +async def test_timeout_message_has_no_deployment_info_by_default(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", timeout("d1")) + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], num_retries=0) + with pytest.raises(litellm.Timeout) as raised: + await core.acompletion(model="g", messages=MESSAGES) + assert "Deployment Info" not in str(raised.value) + + +# --- clientside credentials ------------------------------------------------------------------- + + +@pytest.mark.parametrize("credential", [{"api_key": "sk-client"}, {"api_base": "https://client.example"}]) +async def test_clientside_credentials_get_a_request_scoped_deployment_id( + make_core: MakeCore, host: EndpointHost, credential: dict[str, str] +): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acompletion(model="g", messages=MESSAGES, **credential) + sent: Final = _sent(host) + model_info: Final = _bucket(sent, "model_info") + assert model_info["original_model_id"] == "d1" + assert isinstance(model_info["id"], str) and len(model_info["id"]) == 64 + assert _bucket(_bucket(sent), "model_info")["id"] == model_info["id"] + + +async def test_clientside_credential_id_is_stable_for_the_same_credentials(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acompletion(model="g", messages=MESSAGES, api_key="sk-a") + await core.acompletion(model="g", messages=MESSAGES, api_key="sk-a") + await core.acompletion(model="g", messages=MESSAGES, api_key="sk-b") + ids: Final = [call.deployment_id for call in host.calls] + assert ids[0] == ids[1] != ids[2] + + +async def test_clientside_credentials_do_not_register_a_deployment(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acompletion(model="g", messages=MESSAGES, api_key="sk-client") + await core.acompletion(model="g", messages=MESSAGES) + assert host.deployments_called()[-1] == "d1" + assert [d["model_info"]["id"] for d in core.registry().deployments("g")] == ["d1"] # pyright: ignore[reportIndexIssue, reportUnknownMemberType] # registry rows are deployment dicts + + +# --- embedding batch size -------------------------------------------------------------------- + + +@pytest.mark.parametrize("endpoint", ["aembedding", "embedding"]) +async def test_embedding_over_the_deployment_batch_size_raises_without_calling_the_host( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core( + model_list=[deployment("g", "d1", model_info={"max_embedding_batch_size": 2})], num_retries=0 + ) + with pytest.raises( + litellm.BadRequestError, + match="Embedding request has 3 inputs, which exceeds max_embedding_batch_size=2 for model=g", + ): + await core.call(endpoint, {"model": "g", "input": ["a", "b", "c"]}) + assert host.calls == [] + + +@pytest.mark.parametrize("endpoint", ["aembedding", "embedding"]) +async def test_embedding_within_the_deployment_batch_size_is_sent( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1", model_info={"max_embedding_batch_size": 2})]) + await core.call(endpoint, {"model": "g", "input": ["a", "b"]}) + assert host.calls[0].kwargs["input"] == ["a", "b"] diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..ef51634b920 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -1,6 +1,7 @@ """Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised.""" from collections.abc import Callable, Mapping +from typing import Final import pytest @@ -8,8 +9,11 @@ import litellm from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.router_api import ChangeSet from litellm.router_core import RouterCore +from litellm.types.utils import Usage +from tests.unit.test_router.core.construction_precall_helpers import windowed from tests.unit.test_router.core.scripted_host import ( ScriptedHost, + agentic_followup_failure, bad_request, content_policy, context_window, @@ -18,6 +22,7 @@ from tests.unit.test_router.core.scripted_host import ( server_error, timeout, ) +from tests.unit.test_router.core.streaming_helpers import request_logging MakeCore = Callable[..., RouterCore] @@ -355,21 +360,13 @@ async def test_an_error_specific_chain_wins_over_the_general_chain( @pytest.mark.asyncio -@pytest.mark.parametrize( - ("error", "field", "raised"), - [ - (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError), - (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError), - ], - ids=["context-window", "content-policy"], -) -async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain( - make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception] +async def test_an_unmatched_content_policy_chain_does_not_try_the_general_chain( + make_core: MakeCore, host: ScriptedHost ) -> None: - core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]}) - host.answer("openai/a", error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", content_policy()) - with pytest.raises(raised): + with pytest.raises(litellm.ContentPolicyViolationError): await complete(core) assert host.deployments_called("acompletion") == ["a"] @@ -483,6 +480,179 @@ async def test_a_context_window_error_skips_the_order_tiers_for_its_own_chain( assert host.deployments_called("acompletion") == ["a1", "b"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("window_key", "tiered", "client_side", "expected"), + [ + (None, True, False, "a2"), + ("other", True, False, "a2"), + ("other", False, False, "b"), + ("other", False, True, "b"), + ("a", True, False, "c"), + ("*", True, False, "c"), + ("a", True, True, "c"), + ("a", False, True, "c"), + ], + ids=[ + "no-list-takes-the-order-tier", + "unmatched-list-takes-the-order-tier", + "unmatched-list-falls-through-to-the-general-chain", + "unmatched-list-leaves-the-client-side-list", + "matched-list-beats-the-order-tier", + "matched-star-beats-the-order-tier", + "matched-list-beats-the-order-tier-and-the-client-side-list", + "matched-list-beats-the-client-side-list", + ], +) +async def test_a_context_window_error_takes_its_own_chain_only_when_the_group_matches( + make_core: MakeCore, + host: ScriptedHost, + window_key: str | None, + tiered: bool, + client_side: bool, + expected: str, +) -> None: + core: Final = build( + make_core, + model_list=[ + deployment("a", "a1", order=1), + deployment("a" if tiered else "unused", "a2", order=2), + deployment("b", "b"), + deployment("c", "c"), + ], + fallbacks=[{"a": ["b"]}], + context_window_fallbacks=None if window_key is None else [{window_key: ["c"]}], + ) + host.answer("openai/a1", context_window()) + + await complete(core, **({"fallbacks": ["b"]} if client_side else {})) + + assert host.deployments_called("acompletion") == ["a1", expected] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first", "raised"), + [(context_window, litellm.ContextWindowExceededError), (server_error, litellm.Timeout)], + ids=["context-window-keeps-the-original-error", "any-other-error-raises-the-last-hop"], +) +async def test_a_failed_order_chain_raises_by_the_error_that_started_it( + make_core: MakeCore, host: ScriptedHost, first: Callable[[], Exception], raised: type[Exception] +) -> None: + core: Final = build( + make_core, model_list=TIERED_A, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}] + ) + host.answer("openai/a1", first()) + host.answer("openai/a2", rate_limit()) + host.answer("openai/b", timeout()) + + with pytest.raises(raised): + await complete(core) + + assert host.deployments_called("acompletion") == ["a1", "a2", "b"] + + +@pytest.mark.asyncio +async def test_a_failed_order_chain_appends_no_outcome_message(make_core: MakeCore, host: ScriptedHost) -> None: + show_router_debug(host) + core: Final = build(make_core, model_list=TIERED_A) + host.answer("openai/a1", context_window()) + host.answer("openai/a2", rate_limit()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert "LiteLLM: model group" not in str(raised.value) + + +@pytest.mark.asyncio +async def test_an_unmatched_context_window_list_reports_the_general_chain_it_fell_through_to( + make_core: MakeCore, host: ScriptedHost +) -> None: + show_router_debug(host) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", context_window()) + host.answer("openai/b", rate_limit()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert "No fallback was attempted" not in str(raised.value) + assert "context_window_fallbacks=None" not in str(raised.value) + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_pre_call_context_window_error_is_kept_when_its_order_chain_fails( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build( + make_core, + model_list=[windowed("a", "a1", 1, order=1), windowed("a", "a2", 1, order=2), deployment("b", "b")], + enable_pre_call_checks=True, + fallbacks=[{"a": ["b"]}], + context_window_fallbacks=[{"other": ["c"]}], + ) + host.answer("openai/b", rate_limit()) + + with pytest.raises(litellm.ContextWindowExceededError): + await complete(core) + + assert host.deployments_called("acompletion") == ["b"] + + +@pytest.mark.asyncio +async def test_weighted_failover_skips_a_context_window_error_with_an_unmatched_list( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build( + make_core, + model_list=[deployment("a", "a1"), deployment("a", "a2"), deployment("b", "b")], + enable_weighted_failover=True, + fallbacks=[{"a": ["b"]}], + context_window_fallbacks=[{"other": ["c"]}], + ) + host.answer("openai/a1", context_window()) + host.answer("openai/a2", context_window()) + + await complete(core) + + called: Final = host.deployments_called("acompletion") + assert len(called) == 2 + assert called[-1] == "b" + + +# --- a failed agentic follow-up ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_failed_agentic_followup_still_falls_back(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=3) + host.answer("openai/a", lambda **_: agentic_followup_failure(server_error())) + + response: Final = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert headers(response)["x-litellm-model-group"] == "b" + + +@pytest.mark.asyncio +async def test_a_fallback_hop_carries_the_superseded_attempt_usage(make_core: MakeCore, host: ScriptedHost) -> None: + logging_obj: Final = request_logging() + usage: Final = Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2) + + def failed_after_usage(**_: object) -> Exception: + logging_obj.model_call_details["combined_usage_object"] = usage + return server_error() + + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", failed_after_usage) + + await complete(core, litellm_logging_obj=logging_obj) + + assert logging_obj.model_call_details["failed_attempts_usage"] == ((usage, 0.0),) + + # --- when no fallback runs ----------------------------------------------------------------------- diff --git a/tests/unit/test_router/core/test_fallbacks_characterization.py b/tests/unit/test_router/core/test_fallbacks_characterization.py new file mode 100644 index 00000000000..41a751358a9 --- /dev/null +++ b/tests/unit/test_router/core/test_fallbacks_characterization.py @@ -0,0 +1,584 @@ +""" +Characterization of a router core's fallbacks (non-streaming): chain sources and formats, +client-side fallback lists, per-fallback params, validation, mock-testing flags, depth bounds, +default fallbacks and wildcard lookup. Complements ``test_fallbacks.py``. +""" + +import pytest + +import litellm +from tests.unit.test_router.core.fallbacks_helpers import ( + TIERED_A, + MakeCore, + build, + complete, + complete_sync, + headers, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + content_policy, + context_window, + deployment, + rate_limit, +) + +# --- validation of the configured chains --------------------------------------------------------- + + +@pytest.mark.parametrize( + ("field", "value", "error", "message"), + [ + ("fallbacks", ["b"], ValueError, "Item 'b' is not a dictionary."), + ( + "fallbacks", + [{"a": ["b"], "c": ["b"]}], + ValueError, + "Dictionary '{'a': ['b'], 'c': ['b']}' must have exactly one key, but has 2 keys.", + ), + ( + "fallbacks", + [{"a": "b"}], + TypeError, + "Fallbacks for 'a' must be a list of model names, got 'b'. Use {'a': ['b']}", + ), + ( + "content_policy_fallbacks", + [{"a": "b"}], + TypeError, + "Fallbacks for 'a' must be a list of model names, got 'b'. Use {'a': ['b']}", + ), + ], + ids=["plain-string-item", "two-keys", "string-target", "content-policy-string-target"], +) +def test_a_malformed_fallbacks_configuration_is_refused_at_construction( + make_core: MakeCore, field: str, value: object, error: type[Exception], message: str +) -> None: + with pytest.raises(error) as raised: + build(make_core, **{field: value}) + + assert str(raised.value) == message + + +@pytest.mark.asyncio +async def test_context_window_fallbacks_are_not_validated_at_construction( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, context_window_fallbacks=[{"a": "b"}]) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_constructor_chain_of_parameter_dicts_passes_validation_and_applies_to_any_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"model": "b"}]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_an_empty_constructor_chain_defers_to_the_process_setting( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(fallbacks=[{"a": ["c"]}]) + core = build(make_core, fallbacks=[]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +# --- client-side fallback lists ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_plain_request_list_is_walked_in_order(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + response = await complete(core, fallbacks=["b", "c"]) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + + +@pytest.mark.asyncio +async def test_the_context_window_chain_beats_a_plain_request_list(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, context_window_fallbacks=[{"a": ["c"]}]) + host.answer("openai/a", context_window()) + + await complete(core, fallbacks=["b"]) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +@pytest.mark.asyncio +async def test_a_dict_target_with_params_overrides_the_request_for_that_attempt( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + host.answer("openai/a", rate_limit()) + override = [{"role": "user", "content": "shorter"}] + + await complete(core, fallbacks=[{"model": "b", "temperature": 0.3, "messages": override}]) + + primary, fallback = host.calls_to("acompletion") + assert (primary.deployment_id, primary.kwargs.get("temperature")) == ("a", None) + assert primary.kwargs["messages"] == [{"role": "user", "content": "hi"}] + assert (fallback.deployment_id, fallback.kwargs["temperature"], fallback.kwargs["messages"]) == ("b", 0.3, override) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "arguments", + [ + {"request": [{"model": "b", "temperature": 0.3}, {"model": "c"}]}, + {"constructor": [{"a": [{"model": "b", "temperature": 0.3}, "c"]}]}, + ], + ids=["request-list", "constructor-chain"], +) +async def test_a_dict_targets_params_carry_over_to_the_next_target( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, list[object]] +) -> None: + core = build(make_core, **({"fallbacks": arguments["constructor"]} if "constructor" in arguments else {})) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + await complete(core, **({"fallbacks": arguments["request"]} if "request" in arguments else {})) + + assert [(c.deployment_id, c.kwargs.get("temperature")) for c in host.calls_to("acompletion")] == [ + ("a", None), + ("b", 0.3), + ("c", 0.3), + ] + + +@pytest.mark.asyncio +async def test_a_dict_target_naming_the_failed_group_with_params_retries_that_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + host.answer("openai/a", rate_limit(), None) + + response = await complete(core, fallbacks=[{"model": "a", "temperature": 0}]) + + assert [(c.deployment_id, c.kwargs.get("temperature")) for c in host.calls_to("acompletion")] == [ + ("a", None), + ("a", 0), + ] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +async def test_a_dict_target_with_its_own_api_key_runs_on_a_client_credential_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + host.answer("openai/a", rate_limit()) + + response = await complete(core, fallbacks=[{"model": "b", "api_key": "sk-other"}]) + + primary, fallback = host.calls_to("acompletion") + assert (fallback.model, fallback.kwargs["api_key"]) == ("openai/b", "sk-other") + assert fallback.deployment_id not in {None, "b"} + assert headers(response)["x-litellm-model-group"] == "b" + assert primary.deployment_id == "a" + + +@pytest.mark.asyncio +async def test_a_dict_target_carrying_an_oidc_api_key_is_refused_and_the_primary_error_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core, fallbacks=[{"model": "b", "api_key": "oidc/google/x"}]) + + assert host.deployments_called("acompletion") == ["a"] + assert str(raised.value) == ( + "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to {'model': 'b', 'api_key': 'oidc*********'} also failed: Rejected Request: api_key holds an " + "oidc/ secret reference, which only a deployment or credential a proxy admin configures can carry." + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("fallbacks", "expected"), + [(["c", {"a": ["b"]}], ["a", "b", "c"]), (["b", {"x": ["c"]}], ["a", "b"])], + ids=["keyed-entry-wins-then-string-catches-the-next-hop", "string-entry-is-a-chain-for-every-group"], +) +async def test_a_mixed_list_of_names_and_keyed_entries_is_resolved_as_a_chain_map( + make_core: MakeCore, host: ScriptedHost, fallbacks: list[object], expected: list[str] +) -> None: + core = build(make_core) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, fallbacks=fallbacks) + + assert host.deployments_called("acompletion") == expected + + +@pytest.mark.asyncio +async def test_the_request_fallbacks_kwarg_is_not_forwarded_to_the_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + host.answer("openai/a", rate_limit()) + + await complete(core, fallbacks=["b"]) + + assert [("fallbacks" in c.kwargs) for c in host.calls_to("acompletion")] == [False, False] + + +# --- disable_fallbacks ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_disable_fallbacks_given_as_a_string_does_not_disable_them( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + await complete(core, disable_fallbacks="true") + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_disable_fallbacks_also_stops_the_order_tier_walk(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=TIERED_A) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, disable_fallbacks=True) + + assert host.deployments_called("acompletion") == ["a1"] + + +@pytest.mark.asyncio +async def test_disable_fallbacks_is_not_forwarded_to_the_provider(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core) + + await complete(core, disable_fallbacks=True) + + assert "disable_fallbacks" not in host.calls_to("acompletion")[0].kwargs + + +# --- depth bounds ----------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_request_max_fallbacks_overrides_the_routers(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}]) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, max_fallbacks=1) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_max_fallbacks_zero_also_stops_the_order_tier_walk(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=TIERED_A, max_fallbacks=0) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a1"] + + +@pytest.mark.asyncio +async def test_max_fallbacks_one_still_tries_every_target_of_the_first_level( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b", "c"]}, {"b": ["c"]}], max_fallbacks=1) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_a_fallback_group_spends_its_own_retries(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=1) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "a", "b", "b"] + assert str(raised.value) == "litellm.RateLimitError: slow down LiteLLM Retried: 1 times, LiteLLM Max Retries: 1" + + +@pytest.mark.asyncio +async def test_a_fallback_hop_resumes_the_original_groups_chain(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b", "c"]}], max_fallbacks=5) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + response = await complete(core, include_fallback_errors=True) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + # c answered inside b's hop (b resumed a's chain), so b's error is listed before a's. + assert headers(response)["x-litellm-fallback-errors"] == ( + '[{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}, ' + '{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}]' + ) + + +# --- order tiers with other chains --------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "arguments", + [{"request": {"fallbacks": ["c"]}}, {"constructor": {"default_fallbacks": ["c"]}}], + ids=["plain-request-list", "default-fallbacks"], +) +async def test_order_tiers_run_before_a_plain_list_or_the_default_chain( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, dict[str, object]] +) -> None: + core = build(make_core, model_list=TIERED_A, **arguments.get("constructor", {})) + host.answer("*", rate_limit()) + host.answer("openai/c", None) + + response = await complete(core, **arguments.get("request", {})) + + assert host.deployments_called("acompletion") == ["a1", "a2", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + + +# --- default fallbacks --------------------------------------------------------------------------- + + +def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_in_place( + make_core: MakeCore, host: ScriptedHost +) -> None: + process_fallbacks: list[object] = [{"a": ["b"]}] + host.set_dynamic(fallbacks=process_fallbacks) + + core = build(make_core, default_fallbacks=["c"]) + + assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}] + assert process_fallbacks == [{"a": ["b"]}, {"*": ["c"]}] + + +def test_default_fallbacks_leave_the_constructor_fallbacks_list_unchanged(make_core: MakeCore) -> None: + fallbacks: list[object] = [{"a": ["b"]}] + + core = build(make_core, fallbacks=fallbacks, default_fallbacks=["c"]) + + assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}] + assert fallbacks == [{"a": ["b"]}] + + +@pytest.mark.asyncio +async def test_an_unknown_group_is_served_by_the_first_default_fallback_without_a_fallback_hop( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, default_fallbacks=["b", "c"]) + + response = await complete(core, model="unknown") + + assert host.deployments_called("acompletion") == ["b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 0 + assert headers(response)["x-litellm-model-group"] == "unknown" + assert host.calls_to("acompletion")[0].model_group == "unknown" + + +@pytest.mark.asyncio +async def test_an_unknown_group_with_its_own_chain_falls_back_through_it( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"unknown": ["b"]}]) + + response = await complete(core, model="unknown") + + assert host.deployments_called("acompletion") == ["b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + assert headers(response)["x-litellm-model-group"] == "b" + + +@pytest.mark.asyncio +async def test_default_fallbacks_catch_a_content_policy_error_without_a_content_policy_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, default_fallbacks=["c"]) + host.answer("openai/a", content_policy()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +@pytest.mark.asyncio +async def test_a_request_context_window_fallbacks_of_none_falls_through_to_the_general_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, context_window_fallbacks=[{"a": ["c"]}], fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", context_window()) + + await complete(core, context_window_fallbacks=None) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +# --- wildcard groups ----------------------------------------------------------------------------- + +WILDCARD = [deployment("openai/*", "w", model="openai/*"), deployment("b", "b")] + + +@pytest.mark.asyncio +async def test_a_wildcard_routed_request_uses_the_chain_keyed_by_the_bare_model_name( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=WILDCARD, fallbacks=[{"gpt-x": ["b"]}]) + host.answer("openai/gpt-x", rate_limit()) + + response = await complete(core, model="openai/gpt-x") + + assert host.deployments_called("acompletion") == ["w", "b"] + assert [c.model for c in host.calls_to("acompletion")] == ["openai/gpt-x", "openai/b"] + assert headers(response)["x-litellm-model-group"] == "b" + + +@pytest.mark.asyncio +async def test_a_chain_keyed_by_the_wildcard_pattern_does_not_match_a_routed_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=WILDCARD, fallbacks=[{"openai/*": ["b"]}]) + host.answer("openai/gpt-x", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, model="openai/gpt-x") + + assert host.deployments_called("acompletion") == ["w"] + + +# --- mock testing flags ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("flag", "field", "expected"), + [ + ("mock_testing_fallbacks", "fallbacks", "b"), + ("mock_testing_context_fallbacks", "context_window_fallbacks", "c"), + ("mock_testing_content_policy_fallbacks", "content_policy_fallbacks", "c"), + ], + ids=["general", "context-window", "content-policy"], +) +async def test_a_mock_testing_flag_skips_the_primary_and_walks_the_matching_chain( + make_core: MakeCore, host: ScriptedHost, flag: str, field: str, expected: str +) -> None: + core = build(make_core, num_retries=2, **{"fallbacks": [{"a": ["b"]}], field: [{"a": [expected]}]}) + + response = await complete(core, **{flag: True}) + + assert host.deployments_called("acompletion") == [expected] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + assert headers(response)["x-litellm-attempted-retries"] == 0 + assert host.sleeps == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("flag", "error", "message"), + [ + ( + "mock_testing_fallbacks", + litellm.InternalServerError, + "litellm.InternalServerError: This is a mock exception for model=a, to trigger a fallback. Fallbacks=None", + ), + ( + "mock_testing_context_fallbacks", + litellm.ContextWindowExceededError, + "litellm.ContextWindowExceededError: litellm.BadRequestError: This is a mock exception for model=a, " + "to trigger a fallback. Context_Window_Fallbacks=None", + ), + ( + "mock_testing_content_policy_fallbacks", + litellm.ContentPolicyViolationError, + "litellm.BadRequestError: litellm.ContentPolicyViolationError: This is a mock exception for model=a, " + "to trigger a fallback. Context_Policy_Fallbacks=None", + ), + ], + ids=["general", "context-window", "content-policy"], +) +async def test_a_mock_testing_flag_without_a_chain_raises_the_mock_error_without_calling_the_provider( + make_core: MakeCore, host: ScriptedHost, flag: str, error: type[Exception], message: str +) -> None: + core = build(make_core) + + with pytest.raises(error) as raised: + await complete(core, **{flag: True}) + + assert str(raised.value) == message + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_the_mock_error_names_the_configured_chain_and_the_fallback_outcome( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.InternalServerError) as raised: + await complete(core, mock_testing_fallbacks=True) + + assert str(raised.value) == ( + "litellm.InternalServerError: This is a mock exception for model=a, to trigger a fallback. " + "Fallbacks=[{'a': ['b']}]\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to b also failed: litellm.RateLimitError: slow down" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("value", "expected"), [("true", ["b"]), ("false", ["a"]), (False, ["a"])]) +async def test_a_mock_testing_flag_given_as_a_string_is_parsed_as_a_bool( + make_core: MakeCore, host: ScriptedHost, value: object, expected: list[str] +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + + await complete(core, mock_testing_fallbacks=value) + + assert host.deployments_called("acompletion") == expected + + +@pytest.mark.asyncio +async def test_the_mock_testing_flag_is_not_forwarded_to_the_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + + await complete(core, mock_testing_fallbacks=True) + + assert "mock_testing_fallbacks" not in host.calls_to("acompletion")[0].kwargs + + +def test_sync_completion_honours_the_mock_testing_flag(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + + response = complete_sync(core, mock_testing_fallbacks=True) + + assert host.deployments_called("completion") == ["b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 diff --git a/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py b/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py new file mode 100644 index 00000000000..b5589e24cf7 --- /dev/null +++ b/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py @@ -0,0 +1,690 @@ +""" +Characterization of what a router core's fallbacks leave behind (non-streaming): response headers +and hidden params, the metadata each attempt carries, the error text when fallbacks give up, +fallback access and budget checks, fallback events, content-filter responses and weighted failover. +""" + +from collections.abc import Mapping + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from tests.unit.test_router.core.fallbacks_helpers import ( + GROUP_A_OF_THREE, + MakeCore, + build, + complete, + complete_sync, + fails_once_then_answers, + headers, + hidden, + metadata_of, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + completion_response, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) + +ERROR_429 = '{"message": "litellm.RateLimitError: slow down", "type": "throttling_error", "param": null, "code": "429"}' +ERROR_500 = ( + '{"message": "litellm.InternalServerError: boom", "type": "internal_server_error", "param": null, "code": "500"}' +) + +# --- headers and hidden params ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_two_hop_chain_reports_one_attempted_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}]) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + assert headers(response)["x-litellm-model-group"] == "c" + + +@pytest.mark.asyncio +async def test_the_answering_fallback_deployment_is_the_responses_model_id( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + response = await complete(core) + + assert hidden(response)["model_id"] == "b" + + +@pytest.mark.asyncio +async def test_success_headers_after_a_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + response = await complete(core) + + assert headers(response) == { + "x-litellm-model-group": "b", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 1, + } + + +@pytest.mark.asyncio +async def test_include_fallback_errors_lists_the_primary_error_in_a_header( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + response = await complete(core, include_fallback_errors=True) + + assert headers(response)["x-litellm-fallback-errors"] == f"[{ERROR_429}]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks", + [[{"a": ["b", "c"]}], [{"a": ["b"]}, {"b": ["c"]}]], + ids=["one-list", "chained"], +) +async def test_include_fallback_errors_lists_the_inner_hops_errors_first( + make_core: MakeCore, host: ScriptedHost, fallbacks: list[object] +) -> None: + core = build(make_core, fallbacks=fallbacks) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", server_error()) + + response = await complete(core, include_fallback_errors=True) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + assert headers(response)["x-litellm-fallback-errors"] == f"[{ERROR_500}, {ERROR_429}]" + + +@pytest.mark.asyncio +async def test_without_include_fallback_errors_no_errors_header_is_set(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + response = await complete(core) + + assert "x-litellm-fallback-errors" not in headers(response) + + +# --- what each attempt carries to the host ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_each_attempt_carries_the_fallback_count_and_original_group_in_its_metadata( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}]) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + await complete(core) + + assert [ + (meta["model_group"], meta["original_model_group"], meta["attempted_fallbacks"]) + for meta in (metadata_of(call.kwargs) for call in host.calls_to("acompletion")) + ] == [("a", "a", 0), ("b", "a", 1), ("c", "a", 2)] + + +@pytest.mark.asyncio +async def test_a_fallback_attempt_also_receives_the_routers_walk_state_as_kwargs( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + primary, fallback = host.calls_to("acompletion") + assert {"fallback_depth", "max_fallbacks", "attempted_targets"} & set(primary.kwargs) == set() + assert (fallback.kwargs["fallback_depth"], fallback.kwargs["max_fallbacks"]) == (1, 5) + assert "attempted_targets" in fallback.kwargs + + +@pytest.mark.asyncio +async def test_the_fallback_attempt_sees_the_failed_attempt_in_previous_models( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + previous = metadata_of(host.calls_to("acompletion")[1].kwargs)["previous_models"] + assert [ + (entry["model_group"], entry["deployment_id"], entry["exception_type"]) + for entry in previous # pyright: ignore[reportGeneralTypeIssues] # a tuple of dicts + ] == [("a", "a", "RateLimitError")] + + +# --- the error when fallbacks give up ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_router_debug_on_a_context_window_error_without_its_chain_names_the_missing_setting( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + show_router_debug(host) + host.answer("openai/a", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert str(raised.value) == ( + "litellm.ContextWindowExceededError: litellm.BadRequestError: too long\n" + "model=a. context_window_fallbacks=None. fallbacks=None.\n\n" + "Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks\n\n" + "LiteLLM: model group 'a' failed with the error above. No fallback was attempted." + ) + + +@pytest.mark.asyncio +async def test_router_debug_on_a_context_window_error_that_also_fails_its_fallback_nests_the_hint( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("*", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + hint = "context_window_fallbacks=None. fallbacks=[{'a': ['b']}].\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" + assert str(raised.value) == ( + f"litellm.ContextWindowExceededError: litellm.BadRequestError: too long\nmodel=a. {hint}\n\n" + "LiteLLM: model group 'a' failed with the error above. Fallback to b also failed: " + f"litellm.ContextWindowExceededError: litellm.BadRequestError: too long\nmodel=b. {hint}" + ) + + +@pytest.mark.asyncio +async def test_router_debug_names_a_plain_request_list_that_failed(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core) + show_router_debug(host) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core, fallbacks=["b", "c"]) + + assert str(raised.value) == ( + "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to b, c also failed: litellm.RateLimitError: slow down" + ) + + +@pytest.mark.asyncio +async def test_router_debug_with_max_fallbacks_one_names_only_the_first_level( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}], max_fallbacks=1) + show_router_debug(host) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert str(raised.value).endswith("Fallback to b also failed: litellm.RateLimitError: slow down") + + +@pytest.mark.asyncio +async def test_router_debug_reports_the_first_fallback_errors_message_not_the_last( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", server_error()) + host.answer("openai/c", timeout()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert str(raised.value) == ( + "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to b also failed: litellm.InternalServerError: boom" + ) + + +@pytest.mark.asyncio +async def test_router_debug_truncates_a_long_fallback_error(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit(message="x" * 3000)) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + detail = "litellm.RateLimitError: " + "x" * 3000 + assert str(raised.value) == ( + "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. " + f"Fallback to b also failed: {detail[:2000]}... [truncated {len(detail) - 2000} characters]" + ) + + +@pytest.mark.asyncio +async def test_router_debug_on_an_unmatched_content_policy_chain_says_no_fallback_was_attempted( + make_core: MakeCore, host: ScriptedHost +) -> None: + from tests.unit.test_router.core.scripted_host import content_policy + + core = build(make_core, content_policy_fallbacks=[{"other": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", content_policy()) + + with pytest.raises(litellm.ContentPolicyViolationError) as raised: + await complete(core) + + assert str(raised.value) == ( + "litellm.BadRequestError: litellm.ContentPolicyViolationError: blocked\n\n" + "LiteLLM: model group 'a' failed with the error above. No fallback was attempted." + ) + + +@pytest.mark.asyncio +async def test_router_debug_no_chain_message_without_any_configured_keys( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core, fallbacks=[{"x": ["b"]}, {"y": ["c"]}]) + + assert str(raised.value).endswith( + "LiteLLM: model group 'a' failed with the error above and no fallback model group was found for it, " + "so the request was not retried on another model. Fallbacks are configured for: x, y. " + "Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks) " + "to retry on another model." + ) + + +# --- fallback access and budget checks ------------------------------------------------------------- + + +def denying(denied: str, seen: list[str]): # noqa: ANN201 # a FallbackAccessCheck / FallbackBudgetCheck + async def check(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + seen.append(model) + return model != denied + + return check + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["fallback_access_check", "fallback_budget_check"]) +async def test_a_fallback_target_the_check_rejects_is_skipped( + make_core: MakeCore, host: ScriptedHost, field: str +) -> None: + seen: list[str] = [] + core = build(make_core, fallbacks=[{"a": ["b", "c"]}], **{field: denying("b", seen)}) + host.answer("openai/a", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + assert seen == ["b", "c"] + + +def test_sync_completion_runs_the_fallback_access_check(make_core: MakeCore, host: ScriptedHost) -> None: + seen: list[str] = [] + core = build(make_core, fallbacks=[{"a": ["b", "c"]}], fallback_access_check=denying("b", seen)) + host.answer("openai/a", rate_limit()) + + complete_sync(core) + + assert host.deployments_called("completion") == ["a", "c"] + assert seen == ["b", "c"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "sentence"), + [ + ("fallback_access_check", "Fallback to b was skipped because the caller is not authorized to call it."), + ("fallback_budget_check", "Fallback to b was skipped because the caller is over budget for it."), + ], +) +async def test_router_debug_explains_a_skipped_fallback_target( + make_core: MakeCore, host: ScriptedHost, field: str, sentence: str +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: denying("b", [])}) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + assert str(raised.value) == ( + f"litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. {sentence}" + ) + + +@pytest.mark.asyncio +async def test_router_debug_names_both_the_failed_and_the_skipped_targets( + make_core: MakeCore, host: ScriptedHost +) -> None: + seen: list[str] = [] + core = build( + make_core, + fallbacks=[{"a": ["b", "c"]}], + fallback_access_check=denying("b", seen), + fallback_budget_check=denying("b", seen), + ) + show_router_debug(host) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + assert str(raised.value) == ( + "litellm.RateLimitError: slow down\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to c also failed: litellm.RateLimitError: slow down " + "Fallback to b was skipped because the caller is not authorized to call it." + ) + # The access check runs before the budget check; a skipped target is asked again in c's hop. + assert seen == ["b", "c", "c", "b"] + + +# --- fallback events --------------------------------------------------------------------------------- + + +class FallbackEvents(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: list[tuple[str, str, object, str]] = [] + + async def log_success_fallback_event( + self, original_model_group: str, kwargs: dict, original_exception: Exception + ) -> None: + self.events.append(("success", original_model_group, kwargs.get("model"), type(original_exception).__name__)) + + async def log_failure_fallback_event( + self, original_model_group: str, kwargs: dict, original_exception: Exception + ) -> None: + self.events.append(("failure", original_model_group, kwargs.get("model"), type(original_exception).__name__)) + + +@pytest.mark.asyncio +async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_first( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + events = FallbackEvents() + monkeypatch.setattr(litellm, "callbacks", [events]) + core = build(make_core, fallbacks=[{"a": ["b", "c"]}]) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", server_error()) + + await complete(core) + + assert events.events == [ + ("success", "b", "c", "InternalServerError"), + ("success", "a", "b", "RateLimitError"), + ] + + +@pytest.mark.asyncio +async def test_a_fallback_that_fails_everywhere_logs_failure_events( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + events = FallbackEvents() + monkeypatch.setattr(litellm, "callbacks", [events]) + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert events.events == [("failure", "a", "b", "RateLimitError")] + + +# --- content-filter responses ------------------------------------------------------------------------ + + +def content_filtered() -> object: + response = completion_response() + response.choices[0].finish_reason = "content_filter" # pyright: ignore[reportAttributeAccessIssue] # Choices + return response + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + ({"content_policy_fallbacks": [{"a": ["b"]}]}, ["a", "b"]), + ({"default_fallbacks": ["c"]}, ["a", "c"]), + ({"fallbacks": [{"a": ["b"]}]}, ["a"]), + ({"content_policy_fallbacks": [{"other": ["b"]}], "default_fallbacks": ["c"]}, ["a"]), + ], + ids=["content-policy-chain", "default-fallbacks", "general-chain-only", "unmatched-content-policy-chain"], +) +async def test_a_content_filtered_response_falls_back_only_when_a_content_policy_fallback_resolves( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], expected: list[str] +) -> None: + core = build(make_core, **arguments) + host.answer("openai/a", content_filtered()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == expected + finish_reason = response.choices[0].finish_reason # pyright: ignore[reportAttributeAccessIssue] # ModelResponse + assert finish_reason == ("content_filter" if expected == ["a"] else "stop") + + +def test_sync_completion_falls_back_on_a_content_filtered_response(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", content_filtered()) + + complete_sync(core) + + assert host.deployments_called("completion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_content_filtered_response_with_fallbacks_disabled_raises_content_policy_violation( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", content_filtered()) + + with pytest.raises(litellm.ContentPolicyViolationError) as raised: + await complete(core, disable_fallbacks=True) + + assert ( + str(raised.value) + == "litellm.BadRequestError: litellm.ContentPolicyViolationError: Response output was blocked." + ) + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_the_last_content_filtered_response_is_returned_when_every_fallback_is_filtered( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}]) + host.answer("*", content_filtered()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert response.choices[0].finish_reason == "content_filter" # pyright: ignore[reportAttributeAccessIssue] # ModelResponse + assert headers(response)["x-litellm-model-group"] == "b" + + +# --- weighted failover --------------------------------------------------------------------------- + + +def a_deployments_fail(host: ScriptedHost, error: object) -> None: + for deployment_id in ("a1", "a2", "a3"): + host.answer(f"openai/{deployment_id}", error) + + +@pytest.mark.asyncio +async def test_weighted_failover_tries_every_deployment_of_the_group_before_the_external_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}]) + a_deployments_fail(host, rate_limit()) + + response = await complete(core) + + called = host.deployments_called("acompletion") + assert sorted(called[:3]) == ["a1", "a2", "a3"] + assert called[3:] == ["b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +async def test_without_weighted_failover_one_deployment_fails_over_to_the_external_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, fallbacks=[{"a": ["b"]}]) + a_deployments_fail(host, rate_limit()) + + await complete(core) + + called = host.deployments_called("acompletion") + assert len(called) == 2 + assert called[0] in {"a1", "a2", "a3"} + assert called[1] == "b" + + +@pytest.mark.asyncio +async def test_weighted_failover_moves_an_embedding_to_another_deployment_of_the_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True) + host.answer("*", fails_once_then_answers(rate_limit())) + + await core.aembedding(model="a", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + calls = host.calls_to("aembedding") + assert len(calls) == 2 + assert calls[0].deployment_id != calls[1].deployment_id + assert [metadata_of(call.kwargs)["attempted_fallbacks"] for call in calls] == [0, 1] + + +@pytest.mark.asyncio +async def test_a_weighted_failover_that_answers_counts_as_a_fallback_inside_the_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True) + host.answer("*", fails_once_then_answers(rate_limit())) + + response = await complete(core) + + calls = host.calls_to("acompletion") + assert len(calls) == 2 + assert calls[0].deployment_id != calls[1].deployment_id + assert [call.model_group for call in calls] == ["a", "a"] + assert [metadata_of(call.kwargs)["attempted_fallbacks"] for call in calls] == [0, 1] + assert headers(response)["x-litellm-model-group"] == "a" + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("max_fallbacks", "group_attempts"), [(0, 1), (1, 2), (2, 3)]) +async def test_weighted_failover_is_bounded_by_max_fallbacks( + make_core: MakeCore, host: ScriptedHost, max_fallbacks: int, group_attempts: int +) -> None: + core = build( + make_core, + model_list=GROUP_A_OF_THREE, + enable_weighted_failover=True, + fallbacks=[{"a": ["b"]}], + max_fallbacks=max_fallbacks, + ) + a_deployments_fail(host, rate_limit()) + + if max_fallbacks == 0: + with pytest.raises(litellm.RateLimitError): + await complete(core) + else: + await complete(core) + + called = host.deployments_called("acompletion") + assert len(set(called[:group_attempts])) == group_attempts + assert called[group_attempts:] == ([] if max_fallbacks == 0 else ["b"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "error"), + [ + ({"routing_strategy": "latency-based-routing"}, rate_limit), + ({}, context_window), + ], + ids=["not-simple-shuffle", "context-window-error"], +) +async def test_weighted_failover_does_not_apply( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], error: object +) -> None: + core = build( + make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}], **arguments + ) + a_deployments_fail(host, error()) # pyright: ignore[reportCallIssue] # an error factory + + await complete(core) + + called = host.deployments_called("acompletion") + assert len(called) == 2 + assert called[1] == "b" + + +@pytest.mark.asyncio +async def test_weighted_failover_with_fallbacks_disabled_tries_one_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}]) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, disable_fallbacks=True) + + assert len(host.deployments_called("acompletion")) == 1 + + +def test_sync_completion_also_runs_weighted_failover(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=GROUP_A_OF_THREE, enable_weighted_failover=True, fallbacks=[{"a": ["b"]}]) + a_deployments_fail(host, rate_limit()) + + complete_sync(core) + + called = host.deployments_called("completion") + assert sorted(called[:3]) == ["a1", "a2", "a3"] + assert called[3:] == ["b"] + + +@pytest.mark.asyncio +async def test_weighted_failover_on_a_single_deployment_group_goes_straight_to_the_external_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build( + make_core, + model_list=[deployment("a", "a1"), deployment("b", "b")], + enable_weighted_failover=True, + fallbacks=[{"a": ["b"]}], + ) + host.answer("openai/a1", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a1", "b"] diff --git a/tests/unit/test_router/core/test_fallbacks_characterization_refusal.py b/tests/unit/test_router/core/test_fallbacks_characterization_refusal.py new file mode 100644 index 00000000000..d7c893a36f1 --- /dev/null +++ b/tests/unit/test_router/core/test_fallbacks_characterization_refusal.py @@ -0,0 +1,101 @@ +""" +Characterization of a router core's Anthropic safeguard-refusal fallbacks on /v1/messages +(non-streaming): a ``stop_reason: "refusal"`` response carrying ``stop_details`` re-enters the +fallback chain only when a fallback would resolve for it. + +The core recognises a /v1/messages call by its endpoint's name, so this module runs on +``NamedEndpointHost`` (a ``ScriptedHost`` whose ``anthropic_messages`` endpoint is named so). +""" + +import pytest + +from litellm.router_core import RouterCore +from tests.unit.test_router.core.fallbacks_helpers import ( + MESSAGES, + MakeCore, + NamedEndpointHost, + anthropic_message, + anthropic_refusal, + build, +) + + +@pytest.fixture +def host() -> NamedEndpointHost: + return NamedEndpointHost() + + +async def messages(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.aanthropic_messages(model="a", messages=MESSAGES, max_tokens=10, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + ({"content_policy_fallbacks": [{"a": ["b"]}]}, ["a", "b"]), + ({"fallbacks": [{"a": ["b"]}]}, ["a", "b"]), + ({"default_fallbacks": ["c"]}, ["a", "c"]), + ], + ids=["content-policy-chain", "general-chain", "default-fallbacks"], +) +async def test_a_safeguard_refusal_falls_back_when_a_chain_resolves( + make_core: MakeCore, host: NamedEndpointHost, arguments: dict[str, object], expected: list[str] +) -> None: + core = build(make_core, **arguments) + host.answer("openai/a", anthropic_refusal()) + host.answer(f"openai/{expected[-1]}", anthropic_message()) + + response = await messages(core) + + assert host.deployments_called("anthropic_messages") == expected + assert isinstance(response, dict) + assert response["stop_reason"] == "end_turn" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "request_kwargs", "refusal"), + [ + ({}, {}, anthropic_refusal()), + ({"content_policy_fallbacks": [{"other": ["b"]}], "fallbacks": [{"a": ["b"]}]}, {}, anthropic_refusal()), + ({"fallbacks": [{"a": ["b"]}]}, {"disable_fallbacks": True}, anthropic_refusal()), + ({"fallbacks": [{"a": ["b"]}]}, {}, anthropic_refusal(with_stop_details=False)), + ], + ids=[ + "no-chain", + "unmatched-content-policy-chain-is-authoritative", + "fallbacks-disabled", + "plain-refusal-without-stop-details", + ], +) +async def test_a_refusal_is_returned_to_the_caller_when_no_fallback_would_resolve( + make_core: MakeCore, + host: NamedEndpointHost, + arguments: dict[str, object], + request_kwargs: dict[str, object], + refusal: dict[str, object], +) -> None: + core = build(make_core, **arguments) + host.answer("openai/a", refusal) + + response = await messages(core, **request_kwargs) + + assert host.deployments_called("anthropic_messages") == ["a"] + assert isinstance(response, dict) + assert response["stop_reason"] == "refusal" + assert response["_hidden_params"]["additional_headers"]["x-litellm-attempted-fallbacks"] == 0 + + +@pytest.mark.asyncio +async def test_the_last_refusal_is_returned_when_every_fallback_refuses( + make_core: MakeCore, host: NamedEndpointHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("*", anthropic_refusal()) + + response = await messages(core) + + assert host.deployments_called("anthropic_messages") == ["a", "b"] + assert isinstance(response, dict) + assert response["stop_reason"] == "refusal" diff --git a/tests/unit/test_router/core/test_fallbacks_walk_parity.py b/tests/unit/test_router/core/test_fallbacks_walk_parity.py new file mode 100644 index 00000000000..47c035d096f --- /dev/null +++ b/tests/unit/test_router/core/test_fallbacks_walk_parity.py @@ -0,0 +1,353 @@ +""" +The fallback walk over what the Python router walks and a smaller walk would miss: order levels +behind any name ``_core_get_model_list`` lists, errors raised while picking, fallback checks that +raise, dict targets naming no group, weighted failover past an affinity pin, and the encrypted +reasoning a fallback hop strips. +""" + +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck +from litellm.types.router import RoutingContext as PluginContext +from tests.unit.test_router.core.fallbacks_helpers import ( + MakeCore, + build, + complete, + headers, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + completion_response, + deployment, + rate_limit, + server_error, +) + +TEAM_METADATA: Final = {"user_api_key_team_id": "T1"} + +# --- order levels behind a name ------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_order_levels_walk_through_a_wildcard(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build( + make_core, + model_list=[ + deployment("openai/*", "w1", model="openai/*", order=1), + deployment("openai/*", "w2", model="openai/*", order=2), + ], + ) + host.answer("openai/gpt-4o", rate_limit(), completion_response()) + + await complete(core, "openai/gpt-4o") + + assert host.deployments_called("acompletion") == ["w1", "w2"] + + +@pytest.mark.asyncio +async def test_order_levels_walk_through_an_alias(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build( + make_core, + model_list=[deployment("gg", "d1", order=1), deployment("gg", "d2", order=2)], + model_group_alias={"al": "gg"}, + ) + host.answer("openai/d1", rate_limit()) + + await complete(core, "al") + + assert host.deployments_called("acompletion") == ["d1", "d2"] + + +@pytest.mark.asyncio +async def test_order_levels_walk_through_a_team_public_name(make_core: MakeCore, host: ScriptedHost) -> None: + public: Final = {"team_id": "T1", "team_public_model_name": "tp"} + core: Final = build( + make_core, + model_list=[ + deployment("t1a", "ta1", order=1, model_info=public), + deployment("t1b", "ta2", order=2, model_info=public), + ], + ) + host.answer("openai/ta1", rate_limit()) + + await complete(core, "tp", metadata=dict(TEAM_METADATA)) + + assert host.deployments_called("acompletion") == ["ta1", "ta2"] + + +# --- a team wildcard ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_team_wildcard_public_name_routes_with_its_wildcard_filled_in( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build( + make_core, + model_list=[ + deployment( + "t1-w", + "tw1", + model="openai/*", + model_info={"team_id": "T1", "team_public_model_name": "tw-*"}, + ), + deployment("g", "gp"), + ], + ) + + await complete(core, "tw-gpt-4o", metadata=dict(TEAM_METADATA)) + + assert host.deployments_called("acompletion") == ["tw1"] + assert host.calls[0].kwargs["model"] == "openai/gpt-4o" + + +# --- errors raised while picking ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_usage_refusal_takes_the_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build( + make_core, + model_list=[deployment("one", "o", rpm=1), deployment("safe", "s")], + routing_strategy="usage-based-routing-v2", + fallbacks=[{"one": ["safe"]}], + ) + await complete(core, "one") + + response: Final = await complete(core, "one") + + assert host.deployments_called("acompletion") == ["o", "s"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +async def test_a_usage_refusal_carries_the_routing_note(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", rpm=1)], routing_strategy="usage-based-routing-v2", num_retries=0 + ) + show_router_debug(host) + await complete(core, "g") + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core, "g") + + assert "No fallback" in str(raised.value) + + +class _RaisingPlugin: + """Refuses every pick of ``refused`` in the pre-routing hook.""" + + def __init__(self, refused: str) -> None: + self.refused: Final = refused + self.runs: int = 0 + + async def run(self, context: PluginContext) -> PluginContext: + if self.refused in context.candidate_models: + self.runs += 1 + raise rate_limit() + return context + + +@pytest.mark.asyncio +async def test_a_pre_routing_error_runs_the_pipeline_once_per_retry(make_core: MakeCore, host: ScriptedHost) -> None: + plugin: Final = _RaisingPlugin("openai/a") + core: Final = build(make_core, num_retries=1, retry_after=0, plugins=[plugin]) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert plugin.runs == 2 + assert host.calls == [] + assert getattr(raised.value, "max_retries", None) == 1 + + +@pytest.mark.asyncio +async def test_a_pre_routing_error_takes_the_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build(make_core, plugins=[_RaisingPlugin("openai/a")], fallbacks=[{"a": ["b"]}]) + + await complete(core) + + assert host.deployments_called("acompletion") == ["b"] + + +# --- fallback checks that raise -------------------------------------------------------------------- + + +def _sync_check(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + return True + + +@pytest.mark.asyncio +async def test_a_sync_access_check_ends_the_chain_with_the_levels_own_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], fallback_access_check=_sync_check) + show_router_debug(host) + host.answer("openai/a", server_error()) + + with pytest.raises(litellm.InternalServerError) as raised: + await complete(core) + + assert "also failed" in str(raised.value) + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_sync_access_check_on_the_order_path_raises_its_own_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build( + make_core, + model_list=[deployment("a", "a1", order=1), deployment("a", "a2", order=2), deployment("b", "b")], + fallbacks=[{"a": ["b"]}], + fallback_access_check=_sync_check, + ) + host.answer("openai/a1", server_error()) + host.answer("openai/a2", server_error()) + + with pytest.raises(TypeError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a1", "a2"] + + +# --- dict targets naming no group ------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [5, None], ids=["int", "none"]) +async def test_a_dict_target_with_a_non_string_model_is_a_hop_that_calls_nothing( + make_core: MakeCore, host: ScriptedHost, model: object +) -> None: + core: Final = build(make_core, fallbacks=[{"a": [{"model": model}, "b"]}]) + host.answer("openai/a", server_error()) + + response: Final = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 2 + + +@pytest.mark.asyncio +async def test_a_chain_ending_on_a_non_string_model_notes_the_hops_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build(make_core, fallbacks=[{"a": [{"model": 5}]}]) + show_router_debug(host) + host.answer("openai/a", server_error()) + + with pytest.raises(litellm.InternalServerError) as raised: + await complete(core) + + assert "argument of type 'int'" in str(raised.value) + assert host.deployments_called("acompletion") == ["a"] + + +# --- weighted failover --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("repin", [True, False]) +async def test_weighted_failover_skips_an_affinity_pin_on_the_failed_deployment( + make_core: MakeCore, host: ScriptedHost, repin: bool +) -> None: + core: Final = build( + make_core, + model_list=[deployment("g", "a"), deployment("g", "b")], + enable_weighted_failover=True, + optional_pre_call_checks=["session_affinity"], + session_affinity_repin_on_failover=repin, + ) + check: Final = next( + callback + for callback in core.optional_callbacks or () # pyright: ignore[reportAttributeAccessIssue, reportUnknownVariableType, reportUnknownMemberType] + if isinstance(callback, DeploymentAffinityCheck) + ) + key: Final = DeploymentAffinityCheck.get_session_affinity_cache_key("g", "s", "c") + await check.cache.async_set_cache(key, {"model_id": "a"}) # pyright: ignore[reportUnknownMemberType] + host.answer("openai/a", server_error()) + + response: Final = await complete(core, "g", metadata={"session_id": "s", "user_api_key_hash": "c"}) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +def _fails_then_raises(raised: BaseException): # noqa: ANN202 # a scripted outcome + """One outcome both deployments of a group share: a provider failure, then ``raised``.""" + calls: Final = [0] + + def outcome(**_request: object) -> object: # kwargs-ok: the scripted request + calls[0] += 1 + return server_error() if calls[0] == 1 else raised + + return outcome + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("hop_error", "error"), + [(TypeError("a bug"), TypeError), (rate_limit(), litellm.InternalServerError)], + ids=["a-bug-propagates", "a-provider-error-defers"], +) +async def test_weighted_failover_swallows_only_api_and_rate_limit_errors( + make_core: MakeCore, host: ScriptedHost, hop_error: BaseException, error: type[Exception] +) -> None: + core: Final = build( + make_core, + model_list=[deployment("a", "a1", model="openai/same"), deployment("a", "a2", model="openai/same")], + enable_weighted_failover=True, + ) + host.answer("openai/same", _fails_then_raises(hop_error)) + + with pytest.raises(error): + await complete(core) + + assert len(host.deployments_called("acompletion")) == 2 + + +# --- refusals the walk falls back from --------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_paused_deployment_named_by_id_falls_back(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build( + make_core, + model_list=[deployment("a", "a1", model_info={"blocked": True}), deployment("b", "b")], + fallbacks=[{"a1": ["b"]}], + ) + + await complete(core, "a1") + + assert host.deployments_called("acompletion") == ["b"] + + +# --- encrypted reasoning on a fallback hop -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_fallback_hop_strips_the_reasoning_the_target_cannot_decrypt( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build( + make_core, + model_list=[deployment("a", "a", api_key="k-a"), deployment("b", "b", api_key="k-b")], + fallbacks=[{"a": ["b"]}], + ) + host.answer("openai/a", server_error()) + reasoning: Final = { + "type": "reasoning", + "encrypted_content": "opaque", + "summary": [{"type": "summary_text", "text": "thought"}], + } + + await core.aresponses(model="a", input=[reasoning, {"role": "user", "content": "hi"}]) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + sent: Final = host.calls_to("aresponses")[-1].kwargs["input"] + assert isinstance(sent, list) + assert not any(isinstance(item, dict) and item.get("encrypted_content") for item in sent) diff --git a/tests/unit/test_router/core/test_gaps_adapter_timeout.py b/tests/unit/test_router/core/test_gaps_adapter_timeout.py new file mode 100644 index 00000000000..4b895673481 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_adapter_timeout.py @@ -0,0 +1,34 @@ +""" +Characterization gap: a streaming adapter completion splits its timeouts like a streaming chat +completion, sending the request timeout as ``timeout`` and the stream timeout beside it, so a +fake-streaming provider stays bounded by the request timeout rather than the stream timeout. + +Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core. +""" + +from collections.abc import Callable +from typing import Final + +import pytest + +from litellm.router_core import RouterCore +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.scripted_host import deployment + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] + + +@pytest.fixture +def host() -> EndpointHost: + return EndpointHost() + + +@pytest.mark.parametrize(("stream", "expected"), [(False, (11.0, None)), (True, (11.0, 5.0))]) +async def test_adapter_completion_sends_the_request_timeout_and_the_stream_timeout_beside_it( + make_core: MakeCore, host: EndpointHost, stream: bool, expected: tuple[float, float | None] +): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], stream_timeout=5) + await core.aadapter_completion(adapter_id="my-adapter", model="g", messages=MESSAGES, stream=stream) + sent: Final = host.calls[-1].kwargs + assert (sent["timeout"], sent.get("stream_timeout")) == expected diff --git a/tests/unit/test_router/core/test_gaps_fuzz_access.py b/tests/unit/test_router/core/test_gaps_fuzz_access.py new file mode 100644 index 00000000000..3adec609033 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_fuzz_access.py @@ -0,0 +1,267 @@ +""" +Access and policy gaps differential fuzzing found between the cores: dict fallback targets that +may not be set from any list, which metadata bucket names the caller's team, the routing plugins on +the async pass-through pick, and a name several teams share falling back. +""" + +from typing import Final + +import pytest + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.router import CustomRoutingStrategyBase, RouterRateLimitError +from tests.unit.test_router.core.fallbacks_helpers import ( + MakeCore, + build, + complete, + complete_sync, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, content_policy, deployment, rate_limit +from tests.unit.test_router.core.test_selection_characterization_prerouting import Narrowing + +OIDC_TARGET: Final = {"model": "b", "api_key": "oidc/google/x"} + +# --- dict fallback targets from a router's lists --------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target", + [OIDC_TARGET, {"model": "b", "anthropic_issuer_url": "https://issuer.example"}], + ids=["oidc-api-key", "federation-field"], +) +async def test_a_router_level_dict_target_refuses_the_whole_chain( + make_core: MakeCore, host: ScriptedHost, target: dict[str, str] +) -> None: + core: Final = build(make_core, fallbacks=[{"a": [target, "c"]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_refused_router_level_chain_explains_itself_under_router_debug( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build(make_core, fallbacks=[{"a": [OIDC_TARGET]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert "also failed: Rejected Request: api_key holds an oidc/ secret reference" in str(raised.value) + + +@pytest.mark.asyncio +async def test_a_content_policy_list_with_a_refused_target_tries_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = build(make_core, content_policy_fallbacks=[{"a": [OIDC_TARGET]}]) + host.answer("openai/a", content_policy()) + + with pytest.raises(litellm.ContentPolicyViolationError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_refused_chain_on_the_order_path_raises_the_refusal_itself( + make_core: MakeCore, host: ScriptedHost +) -> None: + # The order path's `run_async_fallback` sits outside the `try` that re-raises the original error. + core: Final = make_core( + model_list=[deployment("a", "a1", order=1), deployment("a", "a2", order=2), deployment("b", "b")], + num_retries=0, + fallbacks=[{"a": [OIDC_TARGET]}], + ) + host.answer("openai/a1", rate_limit()) + + with pytest.raises(ValueError, match="oidc/ secret reference"): + await complete(core) + + assert host.deployments_called("acompletion") == ["a1"] + + +@pytest.mark.asyncio +async def test_a_refused_chain_fails_only_its_own_level(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b", "c"]}, {"b": [{"model": "c", "api_key": "oidc/google/x"}]}]) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + + +def test_a_refused_router_level_chain_is_refused_on_the_sync_path(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": [OIDC_TARGET]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + complete_sync(core) + + assert host.deployments_called("completion") == ["a"] + + +# --- which bucket names the team ------------------------------------------------------------------- + +_TEAM: Final = {"user_api_key_team_id": "t1"} +_OWN_OR_PUBLIC: Final = [ + deployment("g", "own", weight=1, model_info={"team_id": "t1"}), + deployment("g", "public", weight=0), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("buckets", "expected"), + [ + ({"litellm_metadata": _TEAM}, "own"), + ({"metadata": {"trace": "x"}, "litellm_metadata": _TEAM}, "own"), + ({"metadata": {"user_api_key_team_id": ""}, "litellm_metadata": _TEAM}, "own"), + ({"metadata": {"user_api_key_team_id": "t2"}, "litellm_metadata": _TEAM}, "public"), + ], + ids=["only-litellm-metadata", "metadata-without-a-team", "metadata-with-an-empty-team", "metadata-wins"], +) +async def test_the_first_bucket_holding_a_team_routes_the_call( + make_core: MakeCore, host: ScriptedHost, buckets: dict[str, dict[str, str]], expected: str +) -> None: + core: Final = make_core(model_list=list(_OWN_OR_PUBLIC), num_retries=0) + + await complete(core, "g", **{name: dict(bucket) for name, bucket in buckets.items()}) + + assert host.deployments_called("acompletion") == [expected] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("bucket", "raised"), + [("metadata", litellm.RateLimitError), ("litellm_metadata", RouterRateLimitError)], + ids=["one-level-listed-so-the-level-raises-its-own-error", "no-team-listed-so-the-second-level-raises"], +) +async def test_the_order_levels_of_a_failed_group_read_only_the_metadata_bucket( + make_core: MakeCore, host: ScriptedHost, bucket: str, raised: type[Exception] +) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "own", order=1, model_info={"team_id": "t1"}), + deployment("g", "theirs", order=2, model_info={"team_id": "t2"}), + ], + num_retries=0, + ) + host.answer("openai/own", rate_limit()) + + with pytest.raises(raised): + await complete(core, "g", **{bucket: dict(_TEAM)}) + + assert host.deployments_called("acompletion") == ["own"] + + +# --- the routing plugins on the async pass-through pick -------------------------------------------- + +_PASS_THROUGH_PAIR: Final = [ + deployment("g", "a", use_in_pass_through=True, weight=1), + deployment("g", "b", use_in_pass_through=True, weight=0), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy", ["simple-shuffle", "usage-based-routing"]) +async def test_the_async_pass_through_pick_runs_the_routing_plugins(make_core: MakeCore, strategy: str) -> None: + plugin: Final = Narrowing(frozenset({"openai/b"})) + core: Final = make_core(model_list=list(_PASS_THROUGH_PAIR), plugins=[plugin], routing_strategy=strategy) + + chosen: Final = await core.async_get_available_deployment_for_pass_through("g", {}) # pyright: ignore[reportAttributeAccessIssue] + + assert chosen["model_info"]["id"] == "b" + assert plugin.seen == [["openai/a", "openai/b"]] + + +@pytest.mark.asyncio +async def test_a_custom_strategy_does_not_answer_the_async_pass_through_pick(make_core: MakeCore) -> None: + class Refuses(CustomRoutingStrategyBase): + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: never called + raise AssertionError("a pass-through pick never reaches the custom strategy") + + core: Final = make_core(model_list=[deployment("g", "a", use_in_pass_through=True)]) + core.register_plugin(Refuses()) + + chosen: Final = await core.async_get_available_deployment_for_pass_through("g", {}) # pyright: ignore[reportAttributeAccessIssue] + + assert chosen["model_info"]["id"] == "a" + + +# --- a name several teams share -------------------------------------------------------------------- + +_SHARED_NAME: Final = [ + deployment("g", "gp"), + deployment("t1-pub", "p1", model_info={"team_id": "T1", "team_public_model_name": "pub"}), + deployment("t2-pub", "p2", model_info={"team_id": "T2", "team_public_model_name": "pub"}), +] +_REFUSAL: Final = ( + "litellm.BadRequestError: Model name 'pub' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." +) + + +def _admin() -> dict[str, object]: # mutable-ok: a request's metadata + return {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")} # pyright: ignore[reportCallIssue] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "requested"), + [({"fallbacks": [{"pub": ["g"]}]}, {}), ({}, {"fallbacks": ["g"]})], + ids=["router-list", "request-list"], +) +async def test_a_name_several_teams_share_falls_back( + make_core: MakeCore, host: ScriptedHost, configured: dict[str, object], requested: dict[str, object] +) -> None: + core: Final = make_core(model_list=list(_SHARED_NAME), num_retries=0, **configured) + + await complete(core, "pub", metadata=_admin(), **requested) + + assert host.deployments_called("acompletion") == ["gp"] + + +def test_a_name_several_teams_share_falls_back_on_the_sync_path(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=list(_SHARED_NAME), num_retries=0) + + complete_sync(core, "pub", metadata=_admin(), fallbacks=["g"]) + + assert host.deployments_called("completion") == ["gp"] + + +@pytest.mark.asyncio +async def test_a_shared_name_raises_its_refusal_when_the_fallback_fails( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_SHARED_NAME), num_retries=0) + host.answer("openai/gp", rate_limit()) + + with pytest.raises(litellm.BadRequestError) as raised: + await complete(core, "pub", metadata=_admin(), fallbacks=["g"]) + + assert raised.value.status_code == 400 + assert str(raised.value) == _REFUSAL + + +@pytest.mark.asyncio +async def test_a_shared_name_explains_the_failed_fallback_under_router_debug( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_SHARED_NAME), num_retries=0) + show_router_debug(host) + host.answer("openai/gp", rate_limit()) + + with pytest.raises(litellm.BadRequestError) as raised: + await complete(core, "pub", metadata=_admin(), fallbacks=["g"]) + + assert str(raised.value).startswith(f"{_REFUSAL}\n\nLiteLLM: model group 'pub' failed with the error above.") diff --git a/tests/unit/test_router/core/test_gaps_l0001.py b/tests/unit/test_router/core/test_gaps_l0001.py new file mode 100644 index 00000000000..bb6a4fe6c44 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l0001.py @@ -0,0 +1,445 @@ +""" +Gap tests for the first part of ``litellm/router_python_core.py`` (module helpers, the entry points, +``__init__``, callback registration, strategy setup, routing groups and the strategy dispatch): +behaviors no other characterization test pinned down. + +Each test asserts what the Python core does today; oddities are named in the test's docstring. +""" + +import datetime +import re +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router_api import ChangeSet +from litellm.types.router import Deployment, RouterRateLimitError +from litellm.utils import reapply_runtime_model_cost_registrations +from tests.unit.test_router.core.construction_precall_helpers import collect_other_routers, host_callback_names +from tests.unit.test_router.core.registry_helpers import unique_id +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, picked_ids + +NO_DEPLOYMENTS_MESSAGE: Final = ( + "No deployments available for selected model, Try again in 5 seconds. Passed model={model}. " + "pre-call-checks=False, cooldown_list=[]" +) + + +def _group( + name: str, models: list[str], strategy: str, **extra: object +) -> dict[str, object]: # mutable-ok: a routing_groups entry # kwargs-ok: extra RoutingGroup fields + return {"group_name": name, "models": models, "routing_strategy": strategy, **extra} + + +def _logging_obj() -> LiteLLMLogging: + return LiteLLMLogging( + model="g", + messages=MESSAGES, + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="c1", + function_id="f1", + ) + + +def _dynamic_callback_names(logging_obj: LiteLLMLogging) -> list[str]: # mutable-ok: a fresh list + return [type(callback).__name__ for callback in logging_obj.dynamic_async_success_callbacks or ()] + + +# --- construction: routing_strategy=None and legacy attributes ------------------------------------- + + +@pytest.mark.asyncio +async def test_routing_strategy_none_is_accepted_but_leaves_every_pick_without_a_deployment( + make_core: MakeCore, +) -> None: + """Oddity: ``routing_strategy=None`` passes validation and is stored as None (not the + ``simple-shuffle`` default), so no strategy arm matches and every pick rate-limits, while the + healthy listing still lists the deployment.""" + core: Final = make_core(model_list=[deployment("g", "a")], routing_strategy=None) + assert core.settings().routing_strategy is None + assert core.settings().as_dict()["routing_strategy"] is None + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g") + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g") + healthy: Final = await core.select("g", kind="healthy") + assert isinstance(healthy, list) + assert [entry["model_info"]["id"] for entry in healthy] == ["a"] + + +LEGACY_ATTRIBUTE_DEFAULTS: Final = { + "total_calls": {}, + "fail_calls": {}, + "success_calls": {}, + "deployment_stats": {}, + "default_cache_time_seconds": 3600, + "tenacity": None, + "lowesttpm_logger_v2": None, + "lowestlatency_logger": None, + "lowestcost_logger": None, +} + + +@pytest.mark.parametrize(("name", "expected"), sorted(LEGACY_ATTRIBUTE_DEFAULTS.items())) +def test_legacy_attribute_default(make_core: MakeCore, name: str, expected: object) -> None: + assert getattr(make_core(model_list=[deployment("g", "a")]), name) == expected + + +def test_healthy_deployments_is_the_live_deployment_list(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.healthy_deployments is core.registry().all() # pyright: ignore[reportAttributeAccessIssue] + + +def test_chat_completions_create_calls_the_router_with_the_default_litellm_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], default_litellm_params={"temperature": 0.3}) + core.chat.completions.create(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert [(call.op, call.deployment_id, call.kwargs.get("temperature")) for call in host.calls] == [ + ("completion", "a", 0.3) + ] + + +@pytest.mark.asyncio +async def test_a_provider_budget_config_adds_the_budget_limiter_after_the_listed_checks(make_core: MakeCore) -> None: + """A provider budget appends ``router_budget_limiting`` to the optional pre-call checks given.""" + core: Final = make_core( + model_list=[deployment("g", "a")], + provider_budget_config={"openai": {"budget_limit": 1, "time_period": "1d"}}, + optional_pre_call_checks=["prompt_caching"], + ) + assert [type(callback).__name__ for callback in core.optional_callbacks] == [ # pyright: ignore[reportAttributeAccessIssue] + "PromptCachingDeploymentCheck", + "RouterBudgetLimiting", + ] + core.retire() + + +# --- price data reloads --------------------------------------------------------------------------- + + +def test_a_price_data_reload_restores_a_live_routers_deployment_pricing_until_it_retires( + make_core: MakeCore, +) -> None: + model_id: Final = unique_id("l0001") + core: Final = make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3)]) + try: + litellm.model_cost.pop(model_id) + reapply_runtime_model_cost_registrations() + assert litellm.model_cost[model_id]["input_cost_per_token"] == 1e-3 + core.retire() + litellm.model_cost.pop(model_id) + reapply_runtime_model_cost_registrations() + assert model_id not in litellm.model_cost + finally: + litellm.model_cost.pop(model_id, None) + + +# --- runtime routing_strategy_args ---------------------------------------------------------------- + + +def test_new_routing_strategy_args_rebuild_the_selector_and_replace_it_on_the_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 5} + ) + (before,) = host.lists["callbacks"] + core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 9}})) + (after,) = host.lists["callbacks"] + assert after is not before + assert after is core.lowestlatency_logger # pyright: ignore[reportAttributeAccessIssue] + assert after.routing_args.ttl == 9 # pyright: ignore[reportAttributeAccessIssue] + assert core.settings().as_dict()["routing_strategy_args"]["ttl"] == 9 + + +def test_invalid_routing_strategy_args_at_runtime_keep_the_previous_selector( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the invalid args are still stored as ``routing_strategy_args``, while the selector (and + so ``as_dict``, which reads the selector) keeps the previous ones.""" + core: Final = make_core( + model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 5} + ) + (before,) = host.lists["callbacks"] + core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": "abc"}})) + assert host.lists["callbacks"] == [before] + assert core.lowestlatency_logger is before # pyright: ignore[reportAttributeAccessIssue] + assert core.settings().as_dict()["routing_strategy_args"]["ttl"] == 5 + assert core.routing_strategy_args == {"ttl": "abc"} # pyright: ignore[reportAttributeAccessIssue] + + +def test_new_least_busy_args_replace_the_selector_in_callbacks_and_input_callbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], routing_strategy="least-busy") + before: Final = core.leastbusy_logger # pyright: ignore[reportAttributeAccessIssue] + core.apply(ChangeSet(settings={"routing_strategy_args": {"x": 1}})) + after: Final = core.leastbusy_logger # pyright: ignore[reportAttributeAccessIssue] + assert after is not before + assert (host.lists["callbacks"], host.lists["input_callback"]) == ([after], [after]) + + +def test_new_routing_strategy_args_under_simple_shuffle_register_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 9}})) + assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 9} + assert (host.lists["callbacks"], host.lists["input_callback"]) == ([], []) + + +# --- routing groups: selectors on the host, shadowing --------------------------------------------- + + +def test_routing_group_selectors_are_registered_on_the_host_and_replaced_when_groups_change( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m2", "b")], + routing_strategy="least-busy", + routing_groups=[_group("g1", ["m1"], "latency-based-routing", routing_strategy_args={"ttl": 7})], + ) + assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler", "LowestLatencyLoggingHandler"] + (latency,) = ( + callback for callback in host.lists["callbacks"] if type(callback).__name__ == "LowestLatencyLoggingHandler" + ) + assert latency.routing_args.ttl == 7 # pyright: ignore[reportAttributeAccessIssue] + core.apply(ChangeSet(settings={"routing_groups": [_group("g2", ["m2"], "usage-based-routing-v2")]})) + assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler", "LowestTPMLoggingHandler_v2"] + core.apply(ChangeSet(settings={"routing_groups": []})) + assert host_callback_names(host)["callbacks"] == ["LeastBusyLoggingHandler"] + assert host_callback_names(host)["input_callback"] == ["LeastBusyLoggingHandler"] + + +@pytest.mark.asyncio +async def test_a_group_shadowed_by_a_model_name_still_routes_its_members_with_its_strategy( + make_core: MakeCore, +) -> None: + """The shadowed group's name is not callable (calling it reaches the model), but its least-busy + strategy still serves its member: with no traffic least-busy always picks the first deployment.""" + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m2", "b"), deployment("m2", "c")], + routing_groups=[_group("m1", ["m2"], "least-busy")], + ) + assert await picked_ids(core, "m2", times=30, messages=MESSAGES) == {"b"} + assert core.registry().routing_group("m1") is None + + +@pytest.mark.asyncio +async def test_a_group_named_like_an_alias_is_shadowed_by_the_alias(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m2", "b")], + model_group_alias={"al": "m1"}, + routing_groups=[_group("al", ["m2"], "simple-shuffle")], + ) + assert await picked_ids(core, "al") == {"a"} + assert core.registry().routing_group("al") is None + + +def test_a_priority_group_named_like_an_alias_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match=re.escape("Priority routing group names must not shadow a model or alias")): + make_core( + model_list=[deployment("m1", "a")], + model_group_alias={"al": "m1"}, + routing_groups=[_group("al", ["m1"], "priority", model_priorities={"m1": 1})], + ) + + +@pytest.mark.asyncio +async def test_a_deployment_added_under_a_group_name_shadows_the_group(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m2", "b")], + routing_groups=[_group("grp", ["m1", "m2"], "simple-shuffle")], + ) + core.apply(ChangeSet(upsert=[Deployment(**deployment("grp", "z"))])) # pyright: ignore[reportArgumentType] + assert await picked_ids(core, "grp") == {"z"} + assert core.registry().routing_group("grp") is None + + +# --- registry().resolve kinds backed by this range ------------------------------------------------ + + +def test_routing_group_alternatives_count_the_deployments_of_every_member(make_core: MakeCore) -> None: + """Only a callable routing group (or an alias to one) can have alternatives; a plain model group + with two deployments has none by this rule.""" + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m1", "a2"), deployment("m2", "b"), deployment("m3", "c")], + model_group_alias={"al": "grp"}, + routing_groups=[ + _group("grp", ["m2", "m3"], "simple-shuffle"), + _group("one", ["m2"], "priority", model_priorities={"m2": 1}), + ], + ) + registry: Final = core.registry() + answers: Final = { + name: registry.resolve(name, kind="routing_group_alternatives") # pyright: ignore[reportCallIssue, reportArgumentType] + for name in (None, "grp", "one", "al", "m1", "zzz") + } + assert answers == {None: False, "grp": True, "one": False, "al": True, "m1": False, "zzz": False} + + +TEAM_MODELS: Final = [ + deployment("pub", "t1", model_info={"team_id": "T", "team_public_model_name": "pubm"}), + deployment("pub2", "t2", model_info={"team_id": "T", "team_public_model_name": "pubm"}), + deployment("solo", "t3", model_info={"team_id": "T", "team_public_model_name": "solo"}), + deployment("plain", "p1"), + deployment("half", "h1", model_info={"team_id": "T"}), +] + + +@pytest.mark.parametrize( + ("deployment_id", "expected"), + [("t1", True), ("t3", False), ("p1", False), ("h1", False), ("zzz", False)], + ids=["two-siblings", "only-one", "no-team", "no-public-name", "unknown-id"], +) +def test_team_model_alternatives_need_another_routable_deployment_under_the_same_team_name( + make_core: MakeCore, deployment_id: str, expected: bool +) -> None: + core: Final = make_core(model_list=list(TEAM_MODELS)) + assert core.registry().resolve(deployment_id, kind="team_model_alternatives") is expected # pyright: ignore[reportCallIssue, reportArgumentType] + + +def test_a_blocked_team_sibling_is_no_alternative(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("pub", "t1", model_info={"team_id": "T", "team_public_model_name": "pubm"}), + deployment("pub2", "t2", model_info={"team_id": "T", "team_public_model_name": "pubm", "blocked": True}), + ] + ) + assert core.registry().resolve("t1", kind="team_model_alternatives") is False # pyright: ignore[reportCallIssue, reportArgumentType] + + +def test_search_tool_support_comes_from_model_info_else_an_anthropic_request_shape(make_core: MakeCore) -> None: + """Per routable deployment: a configured ``supports_search_tool`` wins, a deployment whose chat + config builds an Anthropic request answers True, anything else None; an unknown model has none.""" + core: Final = make_core( + model_list=[ + deployment("g", "o1"), + deployment("g", "an1", model="anthropic/claude-3-5-sonnet-20240620"), + deployment( + "g", "cfg", model="anthropic/claude-3-5-sonnet-20240620", model_info={"supports_search_tool": False} + ), + ] + ) + registry: Final = core.registry() + assert registry.resolve("g", kind="search_tool_support") == (None, True, False) # pyright: ignore[reportCallIssue, reportArgumentType] + assert registry.resolve("zzz", kind="search_tool_support") == () # pyright: ignore[reportCallIssue, reportArgumentType] + + +# --- per-request strategy overrides --------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_override_selector_is_bound_to_the_request_logging_object_and_reused(make_core: MakeCore) -> None: + """An override strategy the router does not register globally is bound to the request's logging + object; one selector per strategy serves every request. An override matching the router's own + strategy reuses the global selector and binds nothing.""" + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="least-busy") + own: Final = _logging_obj() + await core.select("g", {"routing_strategy": "least-busy", "litellm_logging_obj": own}) + first: Final = _logging_obj() + await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": first}) + second: Final = _logging_obj() + await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": second}) + assert _dynamic_callback_names(own) == [] + assert _dynamic_callback_names(first) == ["LowestTPMLoggingHandler_v2"] + assert (first.dynamic_async_success_callbacks or [None])[0] is (second.dynamic_async_success_callbacks or [])[0] + + +@pytest.mark.asyncio +async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the_request(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("m", "b")], + routing_groups=[_group("gg", ["m"], "usage-based-routing-v2")], + ) + logging_obj: Final = _logging_obj() + await core.select("g", {"routing_strategy": "usage-based-routing-v2", "litellm_logging_obj": logging_obj}) + assert _dynamic_callback_names(logging_obj) == [] + + +def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None: + """The override selector is on no callback list, so the sync pick runs its pre-call check itself: + the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync + path raises a bare ``ValueError`` where the async one raises ``RateLimitError``.""" + core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0) + core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] + with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")): + core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a"] + + +@pytest.mark.asyncio +async def test_an_async_usage_based_v2_override_counts_rpm_before_the_call( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0) + await core.acompletion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] + with pytest.raises(litellm.RateLimitError, match=re.escape("No deployments available for selected model.")): + await core.acompletion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a"] + + +@pytest.mark.asyncio +async def test_request_scoped_routing_still_runs_the_strategy_rpm_check( + make_core: MakeCore, host: ScriptedHost +) -> None: + """After ``observe(request_scoped=True)`` the selector is off the host's lists, but the router's + own pre-call checks still reach it, so ``rpm=1`` still holds.""" + collect_other_routers() + core: Final = make_core( + model_list=[deployment("g", "a", rpm=1)], routing_strategy="usage-based-routing-v2", num_retries=0 + ) + core.observe(request_scoped=True) + assert host.lists["callbacks"] == [] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a"] + + +# --- strategy dispatch arms ----------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_async_pick_under_usage_based_routing_v1_picks_the_first_configured(make_core: MakeCore) -> None: + """The async pick calls v1's sync selector inline; with no traffic it picks the first deployment.""" + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="usage-based-routing" + ) + assert await picked_ids(core, "g", times=10, messages=MESSAGES) == {"a"} + + +def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="latency-based-routing" + ) + for _ in range(10): + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called()) <= {"a", "b"} + assert len(host.calls) == 10 + + +def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + """Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing + rate-limits without calling a provider.""" + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0 + ) + with pytest.raises(RouterRateLimitError) as raised: + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g") + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing" + ) + with pytest.raises(RouterRateLimitError): + await core.select("g", kind="pass_through") diff --git a/tests/unit/test_router/core/test_gaps_l11100_registry.py b/tests/unit/test_router/core/test_gaps_l11100_registry.py new file mode 100644 index 00000000000..808c12a693c --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l11100_registry.py @@ -0,0 +1,732 @@ +""" +Gap tests for the deployment registry of ``router_python_core.py`` (about lines 11550-14060): +provider validation in ``_add_deployment``, the os.environ references of Azure ``dataSources``, +vertex pass-through registration, config-sourced rows, index repair after a delete, cost-map +registration (``responses/`` keys, strategy-marker pricing, replay), deployment budgets kept in +step with upserts and deletes, credentials with provider, router / deployment model info, and how +model group info merges its deployments. +""" + +import os +import uuid +from collections.abc import Generator, Iterator +from typing import Final + +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.types.router import Deployment +from tests.unit.test_router.core.construction_precall_helpers import restoring_model_cost +from tests.unit.test_router.core.gaps_l11100_helpers import MakeCore, ModelsEndpoint, complexity_marker, model_ids +from tests.unit.test_router.core.registry_helpers import team_deployment +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment + + +def _unique(prefix: str = "l11100") -> str: + return f"{prefix}-{uuid.uuid4().hex}" + + +@pytest.fixture +def data_source_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("L11100_ENDPOINT", "https://search.example") + monkeypatch.delenv("L11100_MISSING", raising=False) + yield + + +# --- provider validation ------------------------------------------------------------------------------ + + +def test_a_provider_missing_from_the_hosts_provider_list_is_unsupported( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(provider_list=["anthropic"]) + + with pytest.raises(Exception, match=r"^Unsupported provider - openai$"): + make_core(model_list=[deployment("gpt", "a")]) + + +def test_ignore_invalid_deployments_drops_a_deployment_of_an_unsupported_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(provider_list=["anthropic"]) + + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("claude", "c", model="anthropic/claude-opus-4-6")], + ignore_invalid_deployments=True, + ) + + assert model_ids(core) == ["c"] + + +def test_a_prompt_management_model_skips_provider_validation(make_core: MakeCore, host: ScriptedHost) -> None: + """A ``/...`` model (here langfuse) is resolved from the prompt at call time, so it is + accepted even when no provider is supported at all.""" + host.set_dynamic(provider_list=[]) + + core: Final = make_core(model_list=[deployment("prompted", "p", model="langfuse/my-prompt")]) + + assert model_ids(core) == ["p"] + assert core.registry().names(kind="deployment_names") == ["langfuse/my-prompt"] + + +# --- Azure dataSources and vertex pass-through --------------------------------------------------------- + + +@pytest.mark.usefixtures("data_source_env") +def test_data_source_endpoint_and_key_env_references_are_resolved(make_core: MakeCore) -> None: + """Only ``endpoint`` and ``key`` are read from the environment; a missing variable becomes ``""``.""" + core: Final = make_core( + model_list=[ + deployment( + "gpt", + "a", + dataSources=[ + { + "type": "AzureCognitiveSearch", + "parameters": { + "endpoint": "os.environ/L11100_ENDPOINT", + "key": "os.environ/L11100_MISSING", + "indexName": "os.environ/L11100_ENDPOINT", + }, + } + ], + ) + ] + ) + + stored: Final = core.registry().all()[0]["litellm_params"]["dataSources"] + assert stored == [ + { + "type": "AzureCognitiveSearch", + "parameters": { + "endpoint": "https://search.example", + "key": "", + "indexName": "os.environ/L11100_ENDPOINT", + }, + } + ] + + +def test_a_vertex_pass_through_deployment_without_project_and_location_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="vertex_project") as raised: + make_core(model_list=[deployment("gem", "g", model="vertex_ai/gemini-pro", use_in_pass_through=True)]) + assert str(raised.value) == ( + "vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints." + ) + + +def test_pass_through_needs_no_registration_for_other_providers(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a", use_in_pass_through=True)]) + + assert model_ids(core) == ["a"] + + +# --- config rows and index repair ----------------------------------------------------------------------- + + +def test_config_deployments_leave_out_rows_that_came_from_the_db(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("gpt", "a"), + deployment("gpt", "b", model_info={"db_model": True}), + deployment("other", "c", model_info={"db_model": False}), + ] + ) + + assert [entry["model_info"]["id"] for entry in core.registry().deployments(source="config")] == ["a", "c"] + + +def test_deleting_a_later_deployment_keeps_earlier_team_indices_and_shifts_later_ones(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + team_deployment("internal-1", "t1", "TA", "pub"), + deployment("gpt", "b"), + team_deployment("internal-2", "t2", "TA", "other"), + ] + ) + + core.apply(ChangeSet(delete=["b"])) + + assert dict(core.registry().index("team_model")) == {("TA", "pub"): [0], ("TA", "other"): [1]} + + +def test_deleting_a_teams_only_wildcard_deployment_drops_its_team_pattern_router(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[team_deployment("internal", "tw", "TA", "openai/*"), deployment("gpt", "b")], + ) + assert list(core.registry().patterns(team=True)) == ["TA"] + + core.apply(ChangeSet(delete=["tw"])) + + assert dict(core.registry().patterns(team=True)) == {} + + +# --- cost-map registration --------------------------------------------------------------------------------- + + +@pytest.fixture +def catalog_backend_keys() -> Generator[None, None, None]: + yield from restoring_model_cost("gpt-4o-mini", "openai/gpt-4o-mini") + + +def _limits(key: str) -> tuple[object, object]: + entry: Final = litellm.model_cost.get(key) or {} + return entry.get("max_input_tokens"), entry.get("max_output_tokens") + + +@pytest.mark.usefixtures("catalog_backend_keys") +def test_a_deployments_token_limits_stay_off_a_shared_key_the_catalog_sets(make_core: MakeCore) -> None: + shared_keys: Final = ("gpt-4o-mini", "openai/gpt-4o-mini") + catalog_limits: Final = {key: _limits(key) for key in shared_keys} + override_id: Final = _unique("l11100-limits") + + make_core( + model_list=[ + deployment( + "g", + override_id, + model="openai/gpt-4o-mini", + model_info={"max_input_tokens": 4321, "max_output_tokens": 987, "max_tokens": 987}, + ) + ] + ) + + assert {key: _limits(key) for key in shared_keys} == catalog_limits + assert _limits(override_id) == (4321, 987) + + +def test_a_deployments_token_limits_fill_a_shared_key_the_catalog_does_not_list(make_core: MakeCore) -> None: + backend: Final = f"hosted_vllm/{_unique()}" + + make_core( + model_list=[ + deployment( + "g", + _unique("l11100-limits"), + model=backend, + model_info={"max_input_tokens": 4321, "max_output_tokens": 987}, + ) + ] + ) + + assert _limits(backend) == (4321, 987) + + +def test_a_responses_model_registers_its_backend_key_with_and_without_the_responses_segment( + make_core: MakeCore, +) -> None: + """The shared backend keys carry no custom pricing: that stays under the deployment id.""" + name: Final = _unique() + deployment_id: Final = _unique("l11100-id") + make_core(model_list=[deployment("gpt", deployment_id, model=f"openai/responses/{name}", input_cost_per_token=0.1)]) + + assert litellm.model_cost[f"openai/responses/{name}"].get("input_cost_per_token") is None + assert litellm.model_cost[f"openai/{name}"].get("input_cost_per_token") is None + assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.1 + + +def test_custom_pricing_on_a_strategy_marker_is_not_registered_as_a_price(make_core: MakeCore) -> None: + """A marker is never the deployment that is billed, so even an explicit zero price stays out of its + cost-map entry.""" + marker_id: Final = _unique() + make_core( + model_list=[ + deployment("small", "s"), + deployment("big", "b"), + complexity_marker("smart", marker_id, input_cost_per_token=0.0, output_cost_per_token=0.0), + ] + ) + + entry: Final = litellm.model_cost[marker_id] + assert "input_cost_per_token" not in entry + assert "output_cost_per_token" not in entry + + +def test_replaying_cost_registrations_restores_the_entries_of_deployments_still_served(make_core: MakeCore) -> None: + kept: Final = _unique() + deleted: Final = _unique() + core: Final = make_core( + model_list=[ + deployment("gpt", kept, input_cost_per_token=0.25), + deployment("gpt", deleted, input_cost_per_token=0.5), + ] + ) + core.apply(ChangeSet(delete=[deleted])) + litellm.model_cost.pop(kept, None) + litellm.model_cost.pop(deleted, None) + + core.replay_model_cost_registrations() # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert litellm.model_cost[kept]["input_cost_per_token"] == 0.25 + assert deleted not in litellm.model_cost + + +# --- deployment budgets -------------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_deployment_budgets_from_the_model_list_are_registered_at_construction(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("gpt", "a", max_budget=1.0, budget_duration="1d"), + deployment("gpt", "b", max_budget=2.0, budget_duration="1d"), + deployment("gpt", "c"), + ] + ) + + limiter: Final = core.registry().plugins("budget_limiter") + assert limiter is not None + assert limiter.deployment_budget_config is not None + assert {key: config.max_budget for key, config in limiter.deployment_budget_config.items()} == {"a": 1.0, "b": 2.0} + + +@pytest.mark.asyncio +async def test_deleting_and_upserting_deployments_keep_the_budget_limiter_in_step(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("gpt", "a", max_budget=1.0, budget_duration="1d"), + deployment("gpt", "b", max_budget=2.0, budget_duration="1d"), + ] + ) + limiter: Final = core.registry().plugins("budget_limiter") + assert limiter is not None + + core.apply(ChangeSet(delete=["a"])) + assert list(limiter.deployment_budget_config or {}) == ["b"] + + core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "b"))])) # pyright: ignore[reportArgumentType] + assert limiter.deployment_budget_config is None + + core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "b", max_budget=3.0, budget_duration="1d"))])) # pyright: ignore[reportArgumentType] + assert {key: config.max_budget for key, config in (limiter.deployment_budget_config or {}).items()} == {"b": 3.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("params", [{"max_budget": 1.0}, {"budget_duration": "1d"}], ids=["no-duration", "no-budget"]) +async def test_adding_a_deployment_with_half_a_budget_installs_no_budget_limiter( + make_core: MakeCore, params: dict[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a")]) + + core.add_deployment(Deployment(**deployment("gpt", "half", **params))) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + assert core.registry().plugins("budget_limiter") is None + + +# --- credentials with provider ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("entry", "expected_provider"), + [ + (deployment("gpt", "a", model="gpt-4o"), "openai"), + (deployment("gpt", "a", model="gpt-4o", custom_llm_provider="azure", api_base="https://x"), "azure"), + (deployment("gpt", "a", model="anthropic/claude-opus-4-6"), "anthropic"), + ], + ids=["bare-model-defaults-to-openai", "explicit-provider-wins", "provider-prefix"], +) +def test_credentials_with_provider_name_the_provider( + make_core: MakeCore, entry: dict[str, object], expected_provider: str +) -> None: + found: Final = make_core(model_list=[entry]).registry().credentials("a", form="with_provider") + + assert found is not None + assert found["custom_llm_provider"] == expected_provider + + +def test_credentials_with_provider_drop_an_unknown_credential_name(make_core: MakeCore) -> None: + found: Final = ( + make_core(model_list=[deployment("gpt", "a", litellm_credential_name="l11100-no-such-credential")]) + .registry() + .credentials("a", form="with_provider") + ) + + assert found == {"api_key": "sk-test", "model": "openai/a", "custom_llm_provider": "openai"} + + +# --- router and deployment model info ------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("model", "model_info", "expected_key"), + [ + ("azure/gpt-4o", {}, "azure/gpt-4o"), + ("azure/my-deployment", {"base_model": "azure/gpt-4o"}, "azure/gpt-4o"), + ], + ids=["mapped-deployment-name", "base-model"], +) +def test_router_model_info_of_an_azure_deployment( + make_core: MakeCore, model: str, model_info: dict[str, object], expected_key: str +) -> None: + core: Final = make_core( + model_list=[ + deployment("az", "az", model=model, api_base="https://x", api_version="2024", model_info=model_info) + ] + ) + + info: Final = core.registry().router_model_info(None, "az", id="az") + builtin: Final = litellm.get_model_info("azure/gpt-4o") + assert info["key"] == expected_key + assert info["max_input_tokens"] == builtin["max_input_tokens"] + assert info["input_cost_per_token"] == builtin["input_cost_per_token"] + + +def test_router_model_info_of_an_unmapped_azure_deployment_is_its_zeroed_stub(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("az", "az", model="azure/l11100-unmapped", api_base="https://x", api_version="2024")] + ) + + info: Final = core.registry().router_model_info(None, "az", id="az") + assert info["key"] == "azure/l11100-unmapped" + assert info["max_input_tokens"] is None + assert info["input_cost_per_token"] == 0 + + +def test_deployment_model_info_takes_defaults_from_a_custom_base_model(make_core: MakeCore) -> None: + """The deployment's own pricing wins; the rest comes from its ``base_model``'s built-in entry.""" + name: Final = _unique() + core: Final = make_core( + model_list=[ + deployment( + "gpt", name, model=f"openai/{name}", input_cost_per_token=0.5, model_info={"base_model": "gpt-4o"} + ) + ] + ) + + info: Final = core.registry().deployment_model_info(name, f"openai/{name}") + builtin: Final = litellm.get_model_info("gpt-4o") + assert info is not None + assert info["input_cost_per_token"] == 0.5 + assert info["output_cost_per_token"] == builtin["output_cost_per_token"] + assert info["max_input_tokens"] == builtin["max_input_tokens"] + + +def test_deployment_model_info_of_an_unpriced_unmapped_model_reports_no_prices(make_core: MakeCore) -> None: + """``get_model_info`` reads a missing price as 0; the deployment reports ``None`` instead.""" + name: Final = _unique() + core: Final = make_core(model_list=[deployment("gpt", name, model=f"openai/{name}")]) + + info: Final = core.registry().deployment_model_info(name, f"openai/{name}") + assert info is not None + assert (info["input_cost_per_token"], info["output_cost_per_token"]) == (None, None) + + +# --- model group info ------------------------------------------------------------------------------------------ + + +def test_group_info_takes_the_widest_limits_highest_prices_and_any_capability(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", model="openai/gpt-3.5-turbo"), + deployment("g", "b", model="openai/gpt-4o"), + deployment("g", "c", model="openai/o3-mini"), + deployment("g", "d", model="anthropic/claude-opus-4-6"), + ] + ) + + info: Final = core.registry().group("g") + opus: Final = litellm.get_model_info("anthropic/claude-opus-4-6") + assert info is not None + assert info.providers == ["openai", "anthropic"] + assert (info.max_input_tokens, info.max_output_tokens) == (opus["max_input_tokens"], opus["max_output_tokens"]) + assert (info.input_cost_per_token, info.output_cost_per_token) == ( + opus["input_cost_per_token"], + opus["output_cost_per_token"], + ) + assert info.supports_vision is True + assert info.supports_function_calling is True + assert info.supports_parallel_function_calling is True + assert info.supports_web_search is True + assert info.supports_reasoning is True + assert info.supports_url_context is False + assert info.supports_fast_mode is False + + +def test_group_info_supports_url_context_when_any_deployment_does(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", model="openai/gpt-4o"), deployment("g", "b", model="gemini/gemini-2.5-flash")] + ) + + info: Final = core.registry().group("g") + assert info is not None + assert info.supports_url_context is True + + +def test_group_info_supports_continuation_when_any_deployment_does(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", model="openai/gpt-4o"), + deployment("g", "b", model="openai/gpt-4o-mini", model_info={"supports_continuation": True}), + deployment("plain", "c", model="openai/gpt-4o"), + ] + ) + + grouped: Final = core.registry().group("g") + plain: Final = core.registry().group("plain") + assert grouped is not None and grouped.supports_continuation is True + assert plain is not None and plain.supports_continuation is False + + +@pytest.mark.parametrize("value", ["maybe", "", ["x"], "true", 1]) +def test_group_info_counts_only_a_boolean_true_supports_continuation(make_core: MakeCore, value: object) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", model="openai/gpt-4o", model_info={"supports_continuation": value})] + ) + + info: Final = core.registry().group("g") + assert info is not None + assert info.supports_continuation is False + + +@pytest.mark.parametrize( + ("models", "fast"), + [ + (["anthropic/claude-opus-4-8"], True), + (["anthropic/claude-opus-4-8", "anthropic/claude-opus-4-6"], False), + ], + ids=["every-deployment-fast", "one-not-fast"], +) +def test_group_info_supports_fast_mode_only_when_every_deployment_does( + make_core: MakeCore, models: list[str], fast: bool +) -> None: + core: Final = make_core(model_list=[deployment("g", f"d{i}", model=m) for i, m in enumerate(models)]) + + info: Final = core.registry().group("g") + assert info is not None + assert info.supports_fast_mode is fast + + +@pytest.mark.parametrize( + ("models", "efforts"), + [ + (["anthropic/claude-opus-4-8"], ("none", "minimal", "low", "medium", "high", "xhigh", "max")), + ( + ["anthropic/claude-opus-4-8", "anthropic/claude-opus-4-6"], + ("none", "minimal", "low", "medium", "high", "max"), + ), + (["anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-opus-4-6"], None), + (["anthropic/claude-opus-4-6", "anthropic/claude-sonnet-4-5-20250929"], None), + (["openai/gpt-4o", "openai/gpt-4o"], ()), + ], + ids=["one", "intersection", "unknown-first-stays-unknown", "unknown-later-clears", "none-supported"], +) +def test_group_info_reasoning_efforts_are_the_intersection_or_unknown( + make_core: MakeCore, models: list[str], efforts: tuple[str, ...] | None +) -> None: + core: Final = make_core(model_list=[deployment("g", f"d{i}", model=m) for i, m in enumerate(models)]) + + info: Final = core.registry().group("g") + assert info is not None + assert info.supported_reasoning_efforts == efforts + + +def test_group_info_supported_endpoints_are_the_union_in_deployment_order(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", model="openai/gpt-5"), + deployment("g", "b", model="openai/gpt-4o", model_info={"supported_endpoints": ["/v1/embeddings"]}), + ] + ) + + info: Final = core.registry().group("g") + assert info is not None + assert info.supported_endpoints == ("/v1/chat/completions", "/v1/batch", "/v1/responses", "/v1/embeddings") + + +def test_group_info_of_a_prompt_management_model_names_its_logger_as_provider(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("prompted", "p", model="langfuse/my-prompt")]) + + info: Final = core.registry().group("prompted") + assert info is not None + assert info.providers == ["langfuse"] + + +def test_group_info_of_an_alias_to_neither_a_name_nor_a_dict_is_none(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a")]) + core.apply(ChangeSet(assign={"model_group_alias": {"odd": 5}})) + + assert core.registry().group("odd") is None + + +def test_group_info_of_an_unpriced_unmapped_model_has_no_prices_or_limits(make_core: MakeCore) -> None: + name: Final = _unique() + core: Final = make_core(model_list=[deployment("g", name, model=f"openai/{name}")]) + + info: Final = core.registry().group("g") + assert info is not None + assert (info.input_cost_per_token, info.output_cost_per_token, info.max_input_tokens) == (None, None, None) + + +# --- lookups --------------------------------------------------------------------------------------------------- + + +def test_from_id_of_a_deployment_id_is_its_group_name(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a")]) + + assert core.registry().resolve("a", kind="from_id") == "gpt" + + +def test_from_id_on_an_empty_core_or_of_an_empty_id_is_none(make_core: MakeCore) -> None: + assert make_core(model_list=[]).registry().resolve("x", kind="from_id") is None + assert make_core(model_list=[deployment("gpt", "a")]).registry().resolve("", kind="from_id") is None + + +def test_route_candidates_of_a_routing_group_are_its_members_deployments(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("h", "b"), deployment("other", "c")], + routing_groups=[{"group_name": "rg", "models": ["g", "h"], "routing_strategy": "simple-shuffle"}], + ) + + assert core.registry().names("rg", kind="route_candidates") == frozenset({"a", "b"}) + + +def test_search_tool_support_reads_model_info_then_the_anthropic_route(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", model="anthropic/claude-opus-4-6"), + deployment("g", "b", model="openai/gpt-4o", model_info={"supports_search_tool": False}), + deployment("g", "c", model="openai/gpt-4o"), + ] + ) + + assert core.registry().resolve("g", kind="search_tool_support") == (True, False, None) + + +# --- discovered model info (aapply refresh_model_info) ---------------------------------------------------- + +VLLM_CARDS: Final = {"data": [{"id": "my-model", "max_model_len": 8192}, {"id": "other", "max_model_len": 4096}]} + + +def _vllm( + deployment_id: str = "v", **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry # kwargs-ok: litellm_params + return deployment( + "vllm", deployment_id, model="hosted_vllm/my-model", api_base="http://vllm.local", api_key="sk-vllm", **params + ) + + +@pytest.mark.asyncio +async def test_a_refresh_reads_the_upstream_models_endpoint_with_the_deployments_key(make_core: MakeCore) -> None: + endpoint: Final = ModelsEndpoint(VLLM_CARDS) + core: Final = make_core(model_list=[_vllm(extra_headers={"X-Team": "a"})]) + + await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint)) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + assert endpoint.requests == [("http://vllm.local/v1/models", {"authorization": "Bearer sk-vllm", "x-team": "a"})] + assert dict(core.registry().discovered_model_info("v")) == { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + } + + +@pytest.mark.asyncio +async def test_discovered_limits_never_override_configured_ones_and_feed_the_group_limits(make_core: MakeCore) -> None: + endpoint: Final = ModelsEndpoint(VLLM_CARDS) + core: Final = make_core(model_list=[_vllm(model_info={"max_output_tokens": 1000})]) + + await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint)) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + registry: Final = core.registry() + assert dict(registry.discovered_model_info("v")) == {"max_tokens": 8192, "max_input_tokens": 8192} + assert registry.group("vllm", field="token_limits") == (8192, 1000) + listing: Final = registry.group("vllm", field="listing_info") + assert listing is not None + assert (listing.max_input_tokens, listing.max_output_tokens) == (8192, 1000) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "entry", + [ + deployment("plain", "p", model="anthropic/claude-opus-4-6", api_base="http://vllm.local"), + deployment("wild", "w", model="hosted_vllm/*", api_base="http://vllm.local"), + deployment("nobase", "n", model="openai/gpt-4o"), + ], + ids=["provider-without-discovery", "wildcard-model", "no-api-base"], +) +async def test_a_refresh_skips_deployments_it_cannot_discover(make_core: MakeCore, entry: dict[str, object]) -> None: + endpoint: Final = ModelsEndpoint(VLLM_CARDS) + core: Final = make_core(model_list=[entry]) + + await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint)) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + assert endpoint.requests == [] + + +@pytest.mark.asyncio +async def test_an_unreachable_models_endpoint_discovers_nothing(make_core: MakeCore) -> None: + endpoint: Final = ModelsEndpoint(error=ConnectionError("down")) + core: Final = make_core(model_list=[_vllm()]) + + await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint)) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + assert len(endpoint.requests) == 1 + assert dict(core.registry().discovered_model_info("v")) == {} + + +@pytest.mark.asyncio +async def test_discovered_limits_are_forgotten_when_the_deployment_is_replaced(make_core: MakeCore) -> None: + endpoint: Final = ModelsEndpoint(VLLM_CARDS) + core: Final = make_core(model_list=[_vllm()]) + await core.aapply(ChangeSet(refresh_model_info=True, refresh_client=endpoint)) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + + core.apply(ChangeSet(upsert=[Deployment(**_vllm(rpm=5))])) # pyright: ignore[reportArgumentType] + + assert dict(core.registry().discovered_model_info("v")) == {} + + +# --- failed upserts and replays under ignore_invalid_deployments --------------------------------------------- + + +def test_a_failing_upsert_of_a_new_deployment_under_ignore_invalid_deployments_adds_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a")], ignore_invalid_deployments=True) + host.set_dynamic(provider_list=["anthropic"]) + + result: Final = core.apply(ChangeSet(upsert=[Deployment(**deployment("gpt", "new"))])) # pyright: ignore[reportArgumentType] + + assert list(result.upserted) == [None] + assert model_ids(core) == ["a"] + + +def test_a_replay_retries_deployments_whose_provider_did_not_resolve_and_keeps_them_out_while_it_still_fails( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("odd", "u", model="l11100-no-provider/x")], + ignore_invalid_deployments=True, + ) + assert model_ids(core) == ["a"] + + core.replay_model_cost_registrations() # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert model_ids(core) == ["a"] + + +# --- group info without any model info ------------------------------------------------------------------------ + + +def test_group_info_without_a_cost_map_entry_falls_back_to_the_configured_mode_and_costs(make_core: MakeCore) -> None: + """With the entries construction registered withdrawn from the cost map (as a price data reload + does before the replay) and a model litellm does not map, the group is described from the + deployment's own model_info: its mode, its costs (a string cost read as a float, a missing one as + None) and the provider's supported params, with no token limits.""" + name: Final = _unique() + core: Final = make_core( + model_list=[ + deployment( + "g", name, model=f"openai/{name}", model_info={"mode": "embedding", "input_cost_per_token": "0.5"} + ) + ] + ) + litellm.model_cost.pop(name, None) + litellm.model_cost.pop(f"openai/{name}", None) + + info: Final = core.registry().group("g") + assert info is not None + assert (info.mode, info.input_cost_per_token, info.output_cost_per_token) == ("embedding", 0.5, None) + assert (info.max_input_tokens, info.max_output_tokens) == (None, None) + assert info.supported_openai_params diff --git a/tests/unit/test_router/core/test_gaps_l11100_strategy.py b/tests/unit/test_router/core/test_gaps_l11100_strategy.py new file mode 100644 index 00000000000..2459c577b9c --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l11100_strategy.py @@ -0,0 +1,438 @@ +""" +Gap tests for the strategy-router (pre-routing) deployments of ``router_python_core.py`` (about lines +11114-11550, plus the strategy hooks of set_model_list / upsert / delete): construction-time +validation of auto, adaptive and quality router markers, the (model_name, tags) slot each marker +holds and when it is released, the licensed capability ceiling, and the adaptive router's derived +inputs and post-call hooks. +""" + +from typing import Final + +import pytest + +from litellm.router_api import ChangeSet +from litellm.types.router import Deployment +from tests.unit.test_router.core.gaps_l11100_helpers import ( + TIERS, + MakeCore, + RemovingHost, + adaptive_marker, + complexity_marker, + marker, + model_ids, + registered_tags, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment + + +@pytest.fixture +def host() -> ScriptedHost: + """The adaptive router's hook sync removes callbacks from every list, so the host must support it.""" + return RemovingHost() + + +PLAIN: Final = [deployment("small", "s"), deployment("big", "b")] + +HEURISTIC_V2: Final = {"classifier_type": "heuristic_v2", "tiers": dict(TIERS)} + +HEURISTIC_V2_LIMIT_MESSAGE: Final = ( + "At most 1 auto-router(s) using the heuristic_v2 classifier can be registered but this would make 2. " + "Choose the heuristic v1 classifier for this router or remove an existing heuristic_v2 router." +) + + +# --- marker validation at construction ------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("params", "message"), + [ + ( + {}, + "auto_router_config_path or auto_router_config is required for auto-router deployments. " + "Please set it in the litellm_params", + ), + ( + {"auto_router_config": "{}"}, + "auto_router_default_model is required for auto-router deployments. Please set it in the litellm_params", + ), + ( + {"auto_router_config": "{}", "auto_router_default_model": "big"}, + "auto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params", + ), + ], + ids=["no-config", "no-default-model", "no-embedding-model"], +) +def test_an_auto_router_marker_missing_a_required_param_is_rejected( + make_core: MakeCore, params: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match="is required") as raised: + make_core(model_list=[*PLAIN, marker("auto", "m", "auto_router/semantic", **params)]) + assert str(raised.value) == message + + +def test_an_adaptive_router_marker_without_its_config_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="is required") as raised: + make_core(model_list=[*PLAIN, marker("ad", "m", "auto_router/adaptive_router")]) + assert str(raised.value) == "adaptive_router_config is required for adaptive-router deployments." + + +def test_a_quality_router_marker_without_a_default_model_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="is required") as raised: + make_core(model_list=[*PLAIN, marker("q", "m", "auto_router/quality_router")]) + assert str(raised.value) == ( + "quality_router_default_model is required for quality-router deployments, or set default_model in " + "quality_router_config. Please configure it in the litellm_params" + ) + + +def test_a_quality_router_takes_its_default_model_from_its_config(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + *PLAIN, + marker("q", "m", "auto_router/quality_router", quality_router_config={"default_model": "big"}), + ] + ) + registry: Final = core.registry().plugins("quality") + + assert registered_tags(core, "quality") == {"q": [()]} + assert registry["q"][0].strategy.config.default_model == "big" + + +def test_quality_router_default_model_param_beats_the_one_in_its_config(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + *PLAIN, + marker( + "q", + "m", + "auto_router/quality_router", + quality_router_default_model="small", + quality_router_config={"default_model": "big"}, + ), + ] + ) + + assert core.registry().plugins("quality")["q"][0].strategy.config.default_model == "small" + + +# --- the (model_name, tags) slot -------------------------------------------------------------------- + + +@pytest.mark.parametrize(("tags", "shown"), [(None, "[]"), (["x"], "['x']")], ids=["untagged", "tagged"]) +def test_a_second_marker_with_the_same_name_and_tags_is_rejected( + make_core: MakeCore, tags: list[str] | None, shown: str +) -> None: + extra: Final = {} if tags is None else {"tags": tags} + with pytest.raises(ValueError, match="already exists") as raised: + make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", **extra), complexity_marker("smart", "m2", **extra)] + ) + assert str(raised.value) == ( + f"Complexity-router deployment smart with tags {shown} already exists. " + "Please use a different model name or set different tags." + ) + + +def test_ignore_invalid_deployments_drops_the_repeated_marker_and_keeps_the_first(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1"), complexity_marker("smart", "m2")], + ignore_invalid_deployments=True, + ) + + assert model_ids(core) == ["s", "b", "m1"] + assert registered_tags(core, "complexity") == {"smart": [()]} + + +def test_markers_of_one_name_with_different_tags_each_register(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", tags=["a"]), complexity_marker("smart", "m2", tags=["b"])] + ) + + assert registered_tags(core, "complexity") == {"smart": [("a",), ("b",)]} + + +def test_adding_a_marker_whose_slot_is_taken_raises_and_adds_nothing(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")]) + + with pytest.raises(ValueError, match="already exists"): + core.add_deployment(Deployment(**complexity_marker("smart", "m2"))) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + assert model_ids(core) == ["s", "b", "m1"] + assert registered_tags(core, "complexity") == {"smart": [()]} + + +def test_deleting_a_marker_frees_its_slot_for_a_new_marker(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")]) + + core.apply(ChangeSet(delete=["m1"])) + assert registered_tags(core, "complexity") == {} + result: Final = core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m2"))])) # pyright: ignore[reportArgumentType] + + assert result.upserted[0] is not None + assert registered_tags(core, "complexity") == {"smart": [()]} + assert model_ids(core) == ["s", "b", "m2"] + + +def test_deleting_one_tagged_marker_keeps_the_other_tags_of_its_name(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", tags=["a"]), complexity_marker("smart", "m2", tags=["b"])] + ) + + core.apply(ChangeSet(delete=["m1"])) + + assert registered_tags(core, "complexity") == {"smart": [("b",)]} + + +def test_deleting_a_plain_deployment_sharing_a_markers_name_keeps_the_marker(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1"), deployment("smart", "p")]) + + core.apply(ChangeSet(delete=["p"])) + + assert registered_tags(core, "complexity") == {"smart": [()]} + + +def test_an_upsert_that_retags_a_marker_moves_it_to_the_new_slot(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*PLAIN, complexity_marker("smart", "m1")]) + + core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1", tags=["t"]))])) # pyright: ignore[reportArgumentType] + + assert registered_tags(core, "complexity") == {"smart": [("t",)]} + assert model_ids(core) == ["s", "b", "m1"] + + +def test_set_model_list_drops_the_strategy_routers_of_the_old_list(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("smart", "m1"), + marker("q", "m2", "auto_router/quality_router", quality_router_default_model="big"), + ] + ) + + core.set_model_list(list(PLAIN)) # pyright: ignore[reportAttributeAccessIssue] + + assert registered_tags(core, "complexity") == {} + assert registered_tags(core, "quality") == {} + + +# --- the licensed capability ceiling ----------------------------------------------------------------- + + +def test_a_router_beyond_the_capability_limit_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="At most 1") as raised: + make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2", config=HEURISTIC_V2), + ], + auto_router_capability_limit=lambda: 1, + ) + assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE + + +def test_routers_within_the_capability_limit_or_without_the_capability_register(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2"), + complexity_marker("three", "m3"), + ], + auto_router_capability_limit=lambda: 1, + ) + + assert registered_tags(core, "complexity") == {"one": [()], "two": [()], "three": [()]} + + +def test_without_a_capability_limit_any_number_of_gated_routers_register(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2", config=HEURISTIC_V2), + ] + ) + + assert sorted(registered_tags(core, "complexity")) == ["one", "two"] + + +def test_the_capability_limit_is_read_again_on_every_registration(make_core: MakeCore) -> None: + limit: Final = {"value": 2} + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2", config=HEURISTIC_V2), + ], + auto_router_capability_limit=lambda: limit["value"], + ) + limit["value"] = 1 + + with pytest.raises(ValueError, match="At most 1") as raised: + core.add_deployment(Deployment(**complexity_marker("three", "m3", config=HEURISTIC_V2))) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] + assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE.replace("would make 2", "would make 3") + + +def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_limit(make_core: MakeCore) -> None: + """The edited router is popped first, so the edit itself counts 1 other + itself = 2 > 1 and fails; + the rollback re-adds the previous router with the ceiling switched off.""" + limit: Final = {"value": 2} + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2", config=HEURISTIC_V2), + ], + auto_router_capability_limit=lambda: limit["value"], + ignore_invalid_deployments=True, + ) + limit["value"] = 1 + edited: Final = complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]) + + result: Final = core.apply(ChangeSet(upsert=[Deployment(**edited)])) # pyright: ignore[reportArgumentType] + + assert list(result.upserted) == [None] + assert model_ids(core) == ["s", "b", "m1", "m2"] + assert registered_tags(core, "complexity") == {"one": [()], "two": [()]} + restored: Final = core.registry().deployment("m2") + assert restored is not None + assert restored.litellm_params.tags is None + + +def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed( + make_core: MakeCore, +) -> None: + limit: Final = {"value": 2} + core: Final = make_core( + model_list=[ + *PLAIN, + complexity_marker("one", "m1", config=HEURISTIC_V2), + complexity_marker("two", "m2", config=HEURISTIC_V2), + ], + auto_router_capability_limit=lambda: limit["value"], + ) + limit["value"] = 1 + + with pytest.raises(ValueError, match="At most 1") as raised: + core.apply( + ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))]) # pyright: ignore[reportArgumentType] + ) + assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE + assert model_ids(core) == ["s", "b", "m1"] + assert registered_tags(core, "complexity") == {"one": [()]} + + +# --- the adaptive router ------------------------------------------------------------------------------- + + +def _adaptive_list() -> list[dict[str, object]]: # mutable-ok: a model_list + return [ + adaptive_marker("ar", "ar1", ["x", "y", "z"]), + deployment( + "x", + "x", + input_cost_per_token=0.5, + model_info={"input_cost_per_token": 0.9, "adaptive_router_preferences": {"quality_tier": 2}}, + ), + deployment("y", "y", model_info={"input_cost_per_token": 0.25}), + ] + + +def _hook_counts(host: ScriptedHost) -> tuple[int, int]: + from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, + ) + + callbacks: Final = host.lists["callbacks"] + return ( + sum(isinstance(callback, AdaptiveRouterPostCallHook) for callback in callbacks), + sum(isinstance(callback, AdaptiveRouterResponseHeaderHook) for callback in callbacks), + ) + + +def test_an_adaptive_router_is_built_after_the_whole_list_from_the_models_it_names(make_core: MakeCore) -> None: + """The marker is listed before x and y, yet sees them: litellm_params pricing wins over model_info + pricing, preferences come from model_info, and z (not configured) gets neither.""" + core: Final = make_core(model_list=_adaptive_list()) + registry: Final = core.registry().plugins("adaptive") + + assert registered_tags(core, "adaptive") == {"ar": [()]} + strategy: Final = registry["ar"][0].strategy + assert strategy.model_to_cost == {"x": 0.5, "y": 0.25} + assert {name: prefs.quality_tier for name, prefs in strategy.model_to_prefs.items()} == {"x": 2} + + +def test_an_adaptive_router_registers_one_post_call_hook_and_a_header_hook( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_adaptive_list()) + if not registered_tags(core, "adaptive"): + pytest.fail("no adaptive router was registered") + + post_call, header = _hook_counts(host) + assert post_call == 1 + assert header >= 1 + + +def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_adaptive_list()) + + core.apply(ChangeSet(delete=["ar1"])) + + assert registered_tags(core, "adaptive") == {} + assert _hook_counts(host)[0] == 0 + + +def test_a_complexity_router_with_adaptive_enabled_also_registers_an_adaptive_router(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})] + ) + + assert registered_tags(core, "complexity") == {"smart": [()]} + assert registered_tags(core, "adaptive") == {"smart": [()]} + + +def test_deleting_an_adaptive_complexity_router_releases_both_registrations(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})] + ) + + core.apply(ChangeSet(delete=["m1"])) + + assert registered_tags(core, "complexity") == {} + assert registered_tags(core, "adaptive") == {} + + +def test_upserting_an_adaptive_complexity_router_registers_its_adaptive_companion(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PLAIN)) + + core.apply( + ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True}))]) # pyright: ignore[reportArgumentType] + ) + + assert registered_tags(core, "adaptive") == {"smart": [()]} + + +def test_an_upsert_that_turns_adaptive_off_releases_the_adaptive_companion(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*PLAIN, complexity_marker("smart", "m1", config={"tiers": TIERS, "adaptive": True})] + ) + + core.apply(ChangeSet(upsert=[Deployment(**complexity_marker("smart", "m1"))])) # pyright: ignore[reportArgumentType] + + assert registered_tags(core, "complexity") == {"smart": [()]} + assert registered_tags(core, "adaptive") == {} + + +def test_editing_an_adaptive_router_rebuilds_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_adaptive_list()) + + core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["y"]))])) # pyright: ignore[reportArgumentType] + + registry: Final = core.registry().plugins("adaptive") + assert registered_tags(core, "adaptive") == {"ar": [()]} + assert registry["ar"][0].strategy.model_to_cost == {"y": 0.25} diff --git a/tests/unit/test_router/core/test_gaps_l14000.py b/tests/unit/test_router/core/test_gaps_l14000.py new file mode 100644 index 00000000000..51c47bbf802 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l14000.py @@ -0,0 +1,299 @@ +""" +Gaps in the characterization of ``router_python_core.py`` lines 14000-17669 that the views answer: +``settings().allowed_fails_for`` under an allowed_fails_policy, ``settings().as_dict()`` for the +latency strategy and after a ``routing_strategy_args`` update, the ``registry()`` deployment +sources ``routable`` / ``alias`` / ``routing_groups`` / ``routed_group`` / ``for_request`` and team +aliases of ``all``, cached clients of a pod-discovery deployment, and PTU-share ceilings. +""" + +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling +from litellm.proxy._types import UserAPIKeyAuth +from litellm.router_api import ChangeSet +from litellm.types.router import AllowedFailsPolicy +from tests.unit.test_router.core.scripted_host import ( + auth_error, + bad_request, + content_policy, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) +from tests.unit.test_router.core.selection_helpers import MakeCore, picked_ids + +# --- settings().allowed_fails_for ---------------------------------------------------------------- + + +def _bad_gateway() -> litellm.BadGatewayError: + return litellm.BadGatewayError(message="bad gateway", llm_provider="openai", model="gpt-4o") + + +def _connection_error() -> litellm.APIConnectionError: + return litellm.APIConnectionError(message="no route", llm_provider="openai", model="gpt-4o") + + +FULL_POLICY: Final = AllowedFailsPolicy( + ContentPolicyViolationErrorAllowedFails=1, + BadRequestErrorAllowedFails=2, + BadGatewayErrorAllowedFails=3, + RateLimitErrorAllowedFails=4, +) + + +@pytest.mark.parametrize( + ("error", "expected"), + [ + (content_policy(), 1), + (bad_request(), 2), + # A context-window error is a BadRequestError, so it takes the bad-request allowance. + (context_window(), 2), + (_bad_gateway(), 3), + (rate_limit(), 4), + # Types the policy names no allowance for answer None, never the router's allowed_fails. + (server_error(), None), + (timeout(), None), + (auth_error(), None), + (_connection_error(), None), + ], + ids=[ + "content-policy", + "bad-request", + "context-window", + "bad-gateway", + "rate-limit", + "server-error", + "timeout", + "auth", + "connection", + ], +) +def test_allowed_fails_for_reads_the_policy_field_of_the_error_type( + make_core: MakeCore, error: Exception, expected: int | None +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails_policy=FULL_POLICY) + assert core.settings().allowed_fails_for(error) == expected + + +def test_allowed_fails_for_a_content_policy_error_falls_back_to_the_bad_request_allowance(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], allowed_fails_policy=AllowedFailsPolicy(BadRequestErrorAllowedFails=2) + ) + assert core.settings().allowed_fails_for(content_policy()) == 2 + + +def test_allowed_fails_for_without_a_policy_is_none(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails=7) + assert core.settings().allowed_fails_for(rate_limit()) is None + + +# --- settings().as_dict() and routing_strategy_args ---------------------------------------------- + +LATENCY_ARGS_DEFAULTS: Final = {"lowest_latency_buffer": 0, "max_latency_list_size": 10, "ttft_percentile": None} + + +def test_latency_routing_reports_the_strategys_full_routing_args(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 30} + ) + assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 30.0, **LATENCY_ARGS_DEFAULTS} + + +def test_updating_routing_strategy_args_reaches_the_latency_strategy(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], routing_strategy="latency-based-routing", routing_strategy_args={"ttl": 30} + ) + core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 99}})) + assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 99.0, **LATENCY_ARGS_DEFAULTS} + + +def test_routing_strategy_args_under_simple_shuffle_are_reported_as_given(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.settings().as_dict()["routing_strategy_args"] == {} + core.apply(ChangeSet(settings={"routing_strategy_args": {"ttl": 99}})) + assert core.settings().as_dict()["routing_strategy_args"] == {"ttl": 99} + + +def test_switching_to_latency_routing_takes_the_args_sent_with_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + core.apply(ChangeSet(settings={"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": 7}})) + settings: Final = core.settings().as_dict() + assert settings["routing_strategy"] == "latency-based-routing" + assert settings["routing_strategy_args"] == {"ttl": 7.0, **LATENCY_ARGS_DEFAULTS} + + +# --- registry() deployment sources --------------------------------------------------------------- + +TEAMS_AND_GROUPS: Final = [ + deployment("internal-x", "tx", model_info={"team_id": "team-x", "team_public_model_name": "pub"}), + deployment("internal-y", "ty", model_info={"team_id": "team-y", "team_public_model_name": "pub"}), + deployment("g", "a"), + deployment("g", "b", model_info={"access_groups": ["ag"]}), +] + +ROUTING_GROUP: Final = {"group_name": "grp", "models": ["g"], "routing_strategy": "simple-shuffle"} + + +def _ids(deployments: object) -> list[str]: # mutable-ok: a fresh list + return [deployment_id for _, deployment_id in _rows(deployments)] + + +def _rows(deployments: object) -> list[tuple[str, str]]: # mutable-ok: a fresh list + assert isinstance(deployments, (list, tuple)) + return [(str(row["model_name"]), str(row["model_info"]["id"])) for row in deployments] + + +def test_all_deployments_of_a_team_public_name_are_renamed_to_the_alias(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS)) + rows: Final = core.registry().deployments("pub", source="all", team_id="team-x", model_alias="nick") + assert _rows(rows) == [("nick", "tx")] + + +def test_an_alias_to_a_routing_group_lists_the_members_under_the_alias_without_access_groups( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=list(TEAMS_AND_GROUPS), routing_groups=[dict(ROUTING_GROUP)], model_group_alias={"al": "grp"} + ) + rows: Final = core.registry().deployments("al") + assert _rows(rows) == [("al", "a"), ("al", "b")] + assert isinstance(rows, list) + assert all("access_groups" not in row["model_info"] for row in rows) + + +def test_the_alias_source_lists_every_alias_row(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=list(TEAMS_AND_GROUPS), + routing_groups=[dict(ROUTING_GROUP)], + model_group_alias={"al": "grp", "al2": "g"}, + ) + rows: Final = core.registry().deployments(source="alias") + assert _rows(rows) == [("al", "a"), ("al", "b"), ("al2", "a"), ("al2", "b")] + # A plain alias row keeps the member's access groups; only routing-group rows drop them. + assert isinstance(rows, list) + assert rows[3]["model_info"].get("access_groups") == ["ag"] + + +def test_the_routing_groups_source_lists_members_under_the_group_without_access_groups(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS), routing_groups=[dict(ROUTING_GROUP)]) + rows: Final = core.registry().deployments(source="routing_groups") + assert _rows(rows) == [("grp", "a"), ("grp", "b")] + assert isinstance(rows, tuple) + assert all("access_groups" not in row["model_info"] for row in rows) + + +def test_the_routed_group_of_a_team_public_name_is_the_teams_own_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS)) + assert _rows(core.registry().deployments("pub", source="routed_group", team_id="team-y")) == [("internal-y", "ty")] + assert core.registry().deployments("pub", source="routed_group") == [] + + +def test_the_routable_source_lists_the_deployments_routing_would_consider(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS)) + registry: Final = core.registry() + assert _ids(registry.deployments("g", source="routable")) == ["a", "b"] + assert registry.deployments("nope", source="routable") == () + assert _ids(registry.deployments("a", source="routable")) == ["a"] + team_request: Final = {"metadata": {"user_api_key_team_id": "team-x"}} + assert _ids(registry.deployments("pub", source="routable", request=team_request)) == ["tx"] + + +def test_the_routable_source_answers_nothing_for_a_name_several_teams_share_for_an_admin(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS)) + admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}} # pyright: ignore[reportCallIssue] + assert core.registry().deployments("pub", source="routable", request=admin) == () + + +def test_the_for_request_source_finds_one_teams_public_name_for_a_proxy_admin(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[TEAMS_AND_GROUPS[0], *TEAMS_AND_GROUPS[2:]]) + admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}} # pyright: ignore[reportCallIssue] + assert _ids(core.registry().deployments("pub", source="for_request", request=admin)) == ["tx"] + assert core.registry().deployments("pub", source="for_request", request={}) == [] + + +def test_the_for_request_source_refuses_a_name_several_teams_share_for_a_proxy_admin(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAMS_AND_GROUPS)) + admin: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role="proxy_admin")}} # pyright: ignore[reportCallIssue] + with pytest.raises(litellm.BadRequestError) as raised: + core.registry().deployments("pub", source="for_request", request=admin) + assert str(raised.value) == ( + "litellm.BadRequestError: Model name 'pub' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ) + + +@pytest.mark.parametrize("client_type", ["async", None]) +def test_a_pod_discovery_deployment_has_no_cached_client(make_core: MakeCore, client_type: str | None) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + pod_discovered: Final = deployment("g", "k", kubernetes_pod_discovery=True) + assert core.registry().client(pod_discovered, client_type=client_type) is None + + +# --- PTU shares ---------------------------------------------------------------------------------- + + +def _ptu_shared(model_name: str, deployment_id: str) -> dict[str, object]: # mutable-ok: a model_list entry + return deployment( + model_name, + deployment_id, + model=f"azure/{deployment_id}", + api_base="https://example.openai.azure.com", + model_info={ + "ptu_shares": {"TA": 50}, + "ptu_count": 50, + "cost_per_ptu_per_hour": 1.0, + "ptu_effective_from": "2026-01-01T00:00:00Z", + "base_model": "gpt-4o", + }, + ) + + +PTU_MODELS: Final = [_ptu_shared("gpt", "shared"), _ptu_shared("solo", "s2"), deployment("solo", "open")] + + +def test_a_share_holding_team_has_a_ptu_ceiling_on_its_group(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PTU_MODELS)) + expected: Final = PTUTeamCeiling( + model_group="gpt", tpm_limit=125000, output_to_input_ratio=4.0, cached_input_ratio=0.0 + ) + assert core.health().team_ptu_ceiling("TA", "gpt") == expected + assert core.health().team_ptu_ceiling("TA", "solo") == PTUTeamCeiling( + model_group="solo", tpm_limit=125000, output_to_input_ratio=4.0, cached_input_ratio=0.0 + ) + assert core.health().team_ptu_ceiling("TB", "gpt") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "group", "expected"), + [ + ({"metadata": {"user_api_key_team_id": "TA"}}, "gpt", "shared"), + ({"metadata": {"user_api_key_team_id": "TA"}}, "solo", "s2"), + ({"metadata": {"user_api_key_team_id": "TB"}}, "solo", "open"), + ({}, "solo", "open"), + ], + ids=["holder-shared", "holder-prefers-its-share", "other-team-open", "no-team-open"], +) +async def test_ptu_shares_route_holders_to_their_share_and_others_to_open_deployments( + make_core: MakeCore, request_kwargs: dict[str, object], group: str, expected: str +) -> None: + core: Final = make_core(model_list=list(PTU_MODELS)) + assert await picked_ids(core, group, request_kwargs=request_kwargs) == {expected} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", [{"metadata": {"user_api_key_team_id": "TB"}}, {}], ids=["other", "none"]) +async def test_a_group_only_share_holders_may_use_refuses_everyone_else( + make_core: MakeCore, request_kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=list(PTU_MODELS)) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("gpt", dict(request_kwargs)) + assert str(raised.value) == ( + "litellm.BadRequestError: Deployment gpt is reserved for the teams holding a PTU share of it" + ) diff --git a/tests/unit/test_router/core/test_gaps_l14000_selection.py b/tests/unit/test_router/core/test_gaps_l14000_selection.py new file mode 100644 index 00000000000..8cff6bcdbe5 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l14000_selection.py @@ -0,0 +1,491 @@ +""" +Gaps in the characterization of ``router_python_core.py`` lines 14000-17669 on the selection path: +health-check routing (the cooldown bypass under an allowed_fails_policy, health-check probes, the +sync filter the pass-through pick uses), pass-through picks (paused deployments by id, priority +groups, cooling, other strategies, the async entry point), the failure dispatch to a request's +logging object, ``model_alias_map``, access groups and litellm model names, tier ``litellm_params`` +(dropped params, effort carriers, restored output ceilings), routing-decision redaction and the +Claude Code session binding's guards. +""" + +import asyncio +import time +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.router import AllowedFailsPolicy, RouterRateLimitError +from tests.unit.test_router.core.scripted_host import ScriptedHost, auth_error, deployment, rate_limit +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, ids_of, picked_ids +from tests.unit.test_router.core.test_selection_characterization_prerouting import ( + HARD_MESSAGES, + SESSION_ID, + TIERS, + _complexity_router, + _session_request, + _tiered, +) + +SPREAD_PICKS: Final = 40 + + +def _unhealthy(*ids: str) -> dict[str, dict[str, object]]: # mutable-ok: health states + return {i: {"is_healthy": False, "timestamp": time.time(), "reason": "probe failed"} for i in ids} + + +def _metadata(request: Mapping[str, object], bucket: str = "metadata") -> Mapping[str, object]: + metadata: Final = request.get(bucket) + assert isinstance(metadata, Mapping) + return metadata + + +# --- health-check routing ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_health_check_routing_with_an_allowed_fails_policy_ignores_cooldowns_when_all_cool( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + enable_health_check_routing=True, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=0), + cooldown_time=60, + ) + assert [core.health().record_failure(i, rate_limit(), 429) for i in ("a", "b")] == [True, True] + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"a", "b"} + + +@pytest.mark.asyncio +async def test_health_check_routing_without_a_policy_keeps_honoring_cooldowns(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True, cooldown_time=60 + ) + for deployment_id in ("a", "b"): + core.health().record_failure(deployment_id, rate_limit(), 429) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g") + assert str(raised.value) == ( + "No deployments available for selected model, Try again in 60 seconds. All deployments for selected " + "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['a', 'b']" + ) + + +@pytest.mark.asyncio +async def test_a_health_check_probe_gets_no_deployments_when_all_are_unhealthy(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True) + core.health().health_checks.set_deployment_health_states(_unhealthy("a", "b")) # pyright: ignore[reportArgumentType] # health states + assert ids_of(await core.select("g", kind="healthy")) == ["a", "b"] + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", kind="healthy", health_check_probe=True) # pyright: ignore[reportCallIssue] + assert str(raised.value) == ( + "No deployments available for selected model, Try again in 5 seconds. Passed model=g. " + "pre-call-checks=False, cooldown_list=[]" + ) + + +@pytest.mark.asyncio +async def test_a_health_check_probe_still_drops_only_the_unhealthy_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], enable_health_check_routing=True) + core.health().health_checks.set_deployment_health_states(_unhealthy("a")) # pyright: ignore[reportArgumentType] # health states + assert ids_of(await core.select("g", kind="healthy", health_check_probe=True)) == ["b"] # pyright: ignore[reportCallIssue] + + +PASS_THROUGH_PAIR: Final = [ + deployment("g", "pt", use_in_pass_through=True), + deployment("g", "pt2", use_in_pass_through=True), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + ({}, {"pt2"}), + ({"allowed_fails_policy": {"RateLimitErrorAllowedFails": 1}}, {"pt", "pt2"}), + ({"background_health_check_model_groups": ["other"]}, {"pt", "pt2"}), + ( + {"background_health_check_model_groups": ["g"], "allowed_fails_policy": {"RateLimitErrorAllowedFails": 1}}, + {"pt2"}, + ), + ], + ids=["filtered", "policy-disables", "group-not-listed", "listed-group-with-policy"], +) +async def test_pass_through_selection_applies_the_health_check_filter( + make_core: MakeCore, arguments: dict[str, object], expected: set[str] +) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), enable_health_check_routing=True, **arguments) + core.health().health_checks.set_deployment_health_states(_unhealthy("pt")) # pyright: ignore[reportArgumentType] # health states + assert await picked_ids(core, "g", times=SPREAD_PICKS, kind="pass_through") == expected + + +@pytest.mark.asyncio +async def test_pass_through_selection_uses_every_deployment_when_all_are_unhealthy(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), enable_health_check_routing=True) + core.health().health_checks.set_deployment_health_states(_unhealthy("pt", "pt2")) # pyright: ignore[reportArgumentType] # health states + assert await picked_ids(core, "g", times=SPREAD_PICKS, kind="pass_through") == {"pt", "pt2"} + + +# --- pass-through selection ---------------------------------------------------------------------- + +PAUSED_PASS_THROUGH: Final = [ + deployment("g", "pt", use_in_pass_through=True, model_info={"blocked": True}), + deployment("g", "pt2", use_in_pass_through=True), +] + +PAUSED_MESSAGE: Final = ( + # Oddity: the message names the deployment's litellm model, not the id that was asked for. + "litellm.ServiceUnavailableError: Model 'openai/pt' is currently paused and cannot accept requests." +) + + +@pytest.mark.asyncio +async def test_pass_through_selection_of_a_paused_deployment_by_id_is_unavailable(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH)) + with pytest.raises(litellm.ServiceUnavailableError) as raised: + await core.select("pt", kind="pass_through") + assert str(raised.value) == PAUSED_MESSAGE + + +@pytest.mark.asyncio +async def test_async_pass_through_selection_of_a_paused_deployment_by_id_is_unavailable(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH)) + with pytest.raises(litellm.ServiceUnavailableError) as raised: + await core.async_get_available_deployment_for_pass_through("pt", {}) # pyright: ignore[reportAttributeAccessIssue] + assert str(raised.value) == PAUSED_MESSAGE + + +@pytest.mark.asyncio +async def test_async_pass_through_selection_by_id_returns_a_pass_through_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PAUSED_PASS_THROUGH)) + chosen: Final = await core.async_get_available_deployment_for_pass_through("pt2", {}) # pyright: ignore[reportAttributeAccessIssue] + assert chosen["model_info"]["id"] == "pt2" + + +@pytest.mark.asyncio +async def test_async_pass_through_selection_by_id_refuses_a_plain_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.async_get_available_deployment_for_pass_through("plain", {}) # pyright: ignore[reportAttributeAccessIssue] + assert str(raised.value) == ( + "litellm.BadRequestError: Deployment plain does not support pass-through endpoint (use_in_pass_through=False)" + ) + + +@pytest.mark.asyncio +async def test_async_pass_through_selection_picks_only_pass_through_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")]) + for _ in range(10): + chosen = await core.async_get_available_deployment_for_pass_through("g", {}) # pyright: ignore[reportAttributeAccessIssue] + assert chosen["model_info"]["id"] == "pt" + + +@pytest.mark.asyncio +async def test_pass_through_selection_with_a_selector_strategy_picks_through_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH_PAIR), routing_strategy="least-busy") + assert await picked_ids(core, "g", kind="pass_through") == {"pt"} + chosen: Final = await core.async_get_available_deployment_for_pass_through("g", {}) # pyright: ignore[reportAttributeAccessIssue] + assert chosen["model_info"]["id"] == "pt" + + +PRIORITY_PASS_THROUGH: Final = [ + deployment("m1", "a", use_in_pass_through=True), + deployment("m2", "b", use_in_pass_through=True), +] +PRIORITY_GROUP: Final = { + "group_name": "pg", + "models": ["m1", "m2"], + "routing_strategy": "priority", + "model_priorities": {"m1": 1, "m2": 2}, +} + + +@pytest.mark.asyncio +async def test_pass_through_selection_of_a_priority_group_keeps_the_lowest_priority(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PRIORITY_PASS_THROUGH), routing_groups=[dict(PRIORITY_GROUP)]) + assert await picked_ids(core, "pg", kind="pass_through") == {"a"} + assert await picked_ids(core, "pg", request_kwargs={"_target_order": 2}, kind="pass_through") == {"b"} + + +@pytest.mark.asyncio +async def test_pass_through_selection_of_a_priority_order_nothing_has_rate_limits(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PRIORITY_PASS_THROUGH), routing_groups=[dict(PRIORITY_GROUP)]) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("pg", {"_target_order": 3}, kind="pass_through") + assert str(raised.value) == ( + "No deployments available for selected model, Try again in 5 seconds. Passed model=pg. " + "pre-call-checks=False, cooldown_list=[]" + ) + + +@pytest.mark.asyncio +async def test_pass_through_selection_with_every_pass_through_deployment_cooling_rate_limits( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")], cooldown_time=60 + ) + assert core.health().record_failure("pt", auth_error(), 401) is True + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", kind="pass_through") + assert str(raised.value) == ( + "No deployments available for selected model, Try again in 60 seconds. All deployments for selected " + "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['pt']" + ) + + +# --- failure dispatch to the request's logging object -------------------------------------------- + + +class _RecordingLoggingObject: + """Stands in for a request's ``litellm_logging_obj``: records the failure dispatches.""" + + def __init__(self) -> None: + self.dispatched: list[dict[str, object]] = [] # mutable-ok: the dispatches, in order + + async def dispatch_failure_handlers(self, **kwargs: object) -> None: # kwargs-ok: the dispatch arguments + self.dispatched.append(kwargs) + + +async def _let_tasks_run() -> None: + for _ in range(3): + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_a_refused_pick_is_dispatched_to_the_requests_logging_object(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + logging_obj: Final = _RecordingLoggingObject() + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("nope", {"litellm_logging_obj": logging_obj}) + await _let_tasks_run() + (dispatched,) = logging_obj.dispatched + assert dispatched["exception"] is raised.value + assert dispatched["prefer_async_handlers"] is True + assert "Traceback" in str(dispatched["traceback_exception"]) + + +@pytest.mark.asyncio +async def test_a_refused_async_pass_through_pick_is_dispatched_to_the_requests_logging_object( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + logging_obj: Final = _RecordingLoggingObject() + with pytest.raises(litellm.BadRequestError) as raised: + await core.async_get_available_deployment_for_pass_through("g", {"litellm_logging_obj": logging_obj}) # pyright: ignore[reportAttributeAccessIssue] + await _let_tasks_run() + assert [entry["exception"] for entry in logging_obj.dispatched] == [raised.value] + + +# --- model_alias_map, access groups and litellm model names -------------------------------------- + + +@pytest.mark.asyncio +async def test_the_process_model_alias_map_renames_the_model_in_a_no_deployments_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(model_alias_map={"g": "renamed"}) + core: Final = make_core(model_list=[deployment("g", "a")], cooldown_time=60) + assert (await core.select("g"))["model_info"]["id"] == "a" # pyright: ignore[reportIndexIssue] + core.health().record_failure("a", auth_error(), 401) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g") + assert "Passed model=renamed." in str(raised.value) + + +ACCESS_GROUPED: Final = [ + deployment("g", "in_a", model_info={"access_groups": ["grpA"]}), + deployment("g", "in_b", model_info={"access_groups": ["grpB"]}), + deployment("h", "hh"), +] + + +@pytest.mark.asyncio +async def test_an_access_group_only_caller_reaches_any_deployment_by_its_litellm_model(make_core: MakeCore) -> None: + # Oddity: the access-group narrowing looks up access groups by the requested name, and a + # litellm model string names no group, so a grpA-only caller reaches the grpB deployment. + core: Final = make_core(model_list=list(ACCESS_GROUPED), default_fallbacks=["h"]) + auth: Final = {"metadata": {"user_api_key_auth": UserAPIKeyAuth(models=["grpA"])}} # pyright: ignore[reportCallIssue] + assert (await core.select("openai/in_b", dict(auth)))["model_info"]["id"] == "in_b" # pyright: ignore[reportIndexIssue] + assert (await core.select("openai/in_a", dict(auth)))["model_info"]["id"] == "in_a" # pyright: ignore[reportIndexIssue] + assert (await core.select("unknown", dict(auth)))["model_info"]["id"] == "hh" # pyright: ignore[reportIndexIssue] + + +# --- tier litellm_params ------------------------------------------------------------------------- + + +def _pinned(**tier_params: object) -> dict[str, object]: # mutable-ok: tiers # kwargs-ok: tier litellm_params + return {**TIERS, "SIMPLE": {"model_name": "small", "litellm_params": tier_params}} + + +@pytest.mark.asyncio +async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"), + deployment("big", "b"), + _complexity_router(tiers=_pinned(logit_bias={"1": 2}, temperature=0.2)), + ] + ) + request: Final[dict[str, object]] = {} + chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] + assert request["temperature"] == 0.2 + assert "logit_bias" not in request + + +@pytest.mark.asyncio +async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"), + deployment("big", "b"), + _complexity_router(tiers=_pinned(logit_bias={"1": 2})), + ] + ) + request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["logit_bias"] == {"1": 2} + + +@pytest.mark.asyncio +async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered(tiers=_pinned(reasoning_effort="low"))) + request: Final[dict[str, object]] = { + "thinking": {"type": "enabled"}, + "output_config": {"effort": "high", "format": "json"}, + "reasoning": {"effort": "high"}, + "max_completion_tokens": 100, + } + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["reasoning_effort"] == "low" + assert "thinking" not in request + assert request["output_config"] == {"format": "json"} + assert "reasoning" not in request + assert request["max_completion_tokens"] == 100 + + +@pytest.mark.asyncio +async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered(tiers=_pinned(temperature=0.1))) + request: Final[dict[str, object]] = {"max_completion_tokens": 100} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["temperature"] == 0.1 + assert request["max_completion_tokens"] == 100 + assert "max_tokens" not in request + # The caller's ceiling is stamped on every routed pick, pinned or not. + assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100} + + +@pytest.mark.asyncio +async def test_a_later_pick_restores_the_callers_ceiling_a_tier_pin_replaced(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered(tiers=_pinned(max_tokens=7))) + request: Final[dict[str, object]] = {"max_completion_tokens": 100} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["max_tokens"] == 7 + assert "max_completion_tokens" not in request + assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100} + + chosen: Final = await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + assert request["max_completion_tokens"] == 100 + assert "max_tokens" not in request + + +@pytest.mark.asyncio +async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untracked(make_core: MakeCore) -> None: + # Oddity: forwarded marker params already on the request are not "newly forwarded" on the + # next attempt, so the tracking key is dropped while the params themselves stay behind. + core: Final = make_core(model_list=_tiered(temperature=0.3)) + request: Final[dict[str, object]] = {} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + forwarded: Final = request["_alias_marker_forwarded_params"] + assert isinstance(forwarded, tuple) + assert "temperature" in forwarded + assert request["temperature"] == 0.3 + + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert "_alias_marker_forwarded_params" not in request + assert request["temperature"] == 0.3 + + +# --- routing decision redaction ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + request: Final[dict[str, object]] = {} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + decision: Final = _metadata(request)["routing_decision"] + assert isinstance(decision, Mapping) + assert "signals" in decision + + +@pytest.mark.asyncio +async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decision_fields(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + request: Final[dict[str, object]] = {"standard_callback_dynamic_params": {"turn_off_message_logging": True}} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + decision: Final = _metadata(request)["routing_decision"] + assert isinstance(decision, Mapping) + assert "signals" not in decision + assert (decision["router_model_name"], decision["tier"], decision["routed_model"]) == ("smart", "SIMPLE", "small") + + +@pytest.mark.asyncio +async def test_routing_metadata_goes_to_litellm_metadata_when_the_request_has_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + request: Final[dict[str, object]] = {"litellm_metadata": {}} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert "routing_decision" in _metadata(request, "litellm_metadata") + assert "metadata" not in request + + +# --- the Claude Code session binding's guards ---------------------------------------------------- + + +async def _subagent_pick(core: object) -> str: + chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + return str(chosen["model_info"]["id"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("depth", "expected"), [(1, "b"), (0, "s")], ids=["fallback-attempt", "first-attempt"]) +async def test_only_a_first_attempt_binds_its_session(make_core: MakeCore, depth: int, expected: str) -> None: + core: Final = make_core(model_list=_tiered()) + first: Final = _session_request() + first["fallback_depth"] = depth + await core.select("smart", first, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert await _subagent_pick(core) == expected + + +@pytest.mark.asyncio +async def test_a_request_without_a_caller_hash_never_binds(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + first: Final = _session_request() + first["metadata"] = {} + await core.select("smart", first, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert await _subagent_pick(core) == "b" + + +@pytest.mark.asyncio +async def test_the_caller_hash_is_read_from_litellm_metadata_when_present(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + first: Final = _session_request() + first["metadata"] = {} + first["litellm_metadata"] = {"user_api_key_hash": "caller-hash"} + await core.select("smart", first, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert await _subagent_pick(core) == "s" + + +@pytest.mark.asyncio +async def test_the_session_header_name_is_matched_case_insensitively(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + first: Final = _session_request() + first["proxy_server_request"] = {"headers": {"X-Claude-Code-Session-Id": SESSION_ID}} + await core.select("smart", first, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert await _subagent_pick(core) == "s" diff --git a/tests/unit/test_router/core/test_gaps_l2800.py b/tests/unit/test_router/core/test_gaps_l2800.py new file mode 100644 index 00000000000..78e237f0696 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l2800.py @@ -0,0 +1,431 @@ +""" +Gap tests for ``router_python_core.py`` lines 2800-5600: optional pre-call check re-registration, +silent experiments, per-deployment ``num_retries`` stamped on errors, and the request kwargs +``_update_kwargs_with_deployment`` builds (Kubernetes pod routing, baggage stamps, stream usage +injection, malformed request tools, per-user GitHub Copilot credentials). + +Each test asserts what the Python core does today; oddities are named in the test's docstring. +""" + +import asyncio +import threading +import time +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.constants import BAGGAGE_MAPPED_TAGS_METADATA_KEY, KUBERNETES_POD_ROUTING_KEY +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD +from litellm.types.router import BaggageMappedTagsStamp +from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, host_callback_names +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit + +MakeCore = Callable[..., RouterCore] +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +@pytest.fixture +def host() -> ScriptedHost: + return CallbackHost() + + +def _metadata(kwargs: Mapping[str, object], name: str = "metadata") -> Mapping[str, object]: + bucket: Final = kwargs[name] + assert isinstance(bucket, Mapping) + return bucket + + +async def _settle() -> None: + for _ in range(30): + await asyncio.sleep(0) + + +# --- optional pre-call checks added again at runtime (add_optional_pre_call_checks) ------------ + + +def test_runtime_affinity_checks_merge_into_the_existing_affinity_callback( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["deployment_affinity"]) + (existing,) = core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + core.apply(ChangeSet(settings={"optional_pre_call_checks": ["deployment_affinity", "session_affinity"]})) + (merged,) = core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + assert merged is existing + assert ( + merged.enable_user_key_affinity, + merged.enable_responses_api_affinity, + merged.enable_session_id_affinity, + ) == ( + True, + False, + True, + ) + assert host_callback_names(host)["callbacks"] == ["DeploymentAffinityCheck"] + + +def test_runtime_affinity_flags_are_only_ever_turned_on(make_core: MakeCore) -> None: + """Oddity: a runtime list without ``deployment_affinity`` leaves the flag it set on, since the + existing callback's flags are OR-ed with the new list's.""" + core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["deployment_affinity"]) + core.apply(ChangeSet(settings={"optional_pre_call_checks": ["responses_api_deployment_check"]})) + (merged,) = core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + assert ( + merged.enable_user_key_affinity, + merged.enable_responses_api_affinity, + merged.enable_session_id_affinity, + ) == ( + True, + True, + False, + ) + + +def test_runtime_affinity_merge_takes_the_model_group_affinity_config(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], + optional_pre_call_checks=["deployment_affinity"], + model_group_affinity_config={"g": ["session_affinity"]}, + ) + core.apply(ChangeSet(settings={"optional_pre_call_checks": ["session_affinity"]})) + (merged,) = [cb for cb in core.optional_callbacks if type(cb).__name__ == "DeploymentAffinityCheck"] # pyright: ignore[reportAttributeAccessIssue] + assert merged.model_group_affinity_config == {"g": ["session_affinity"]} + assert merged.enable_session_id_affinity is True + + +def test_runtime_encrypted_content_affinity_reuses_the_existing_callback( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], optional_pre_call_checks=["encrypted_content_affinity"]) + (existing,) = core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + core.apply(ChangeSet(settings={"optional_pre_call_checks": ["encrypted_content_affinity"]})) + (kept,) = core.optional_callbacks # pyright: ignore[reportAttributeAccessIssue] + assert kept is existing + assert kept.enable_global_affinity is True + assert host_callback_names(host)["callbacks"] == ["EncryptedContentAffinityCheck"] + + +def test_runtime_encrypted_content_affinity_turns_global_affinity_on_for_a_group_scoped_callback( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], model_group_affinity_config={"g": ["encrypted_content_affinity"]} + ) + (existing,) = [cb for cb in core.optional_callbacks if type(cb).__name__ == "EncryptedContentAffinityCheck"] # pyright: ignore[reportAttributeAccessIssue] + assert existing.enable_global_affinity is False + core.add_optional_pre_call_checks(["encrypted_content_affinity"]) # pyright: ignore[reportAttributeAccessIssue] + assert existing.enable_global_affinity is True + assert host_callback_names(host)["callbacks"] == ["EncryptedContentAffinityCheck", "DeploymentAffinityCheck"] + + +async def test_router_budget_limiting_is_not_added_twice_beside_a_provider_budget_config(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], + provider_budget_config={"openai": {"budget_limit": 1, "time_period": "1d"}}, + optional_pre_call_checks=["router_budget_limiting"], + ) + limiters: Final = [cb for cb in core.optional_callbacks if isinstance(cb, RouterBudgetLimiting)] # pyright: ignore[reportAttributeAccessIssue] + assert len(limiters) == 1 + assert core.registry().plugins("budget_limiter") is limiters[0] + core.retire() + + +# --- silent experiments ----------------------------------------------------------------------- + + +def _wait_for_calls(host: ScriptedHost, count: int) -> None: + deadline: Final = time.monotonic() + 5 + while len(host.calls) < count and time.monotonic() < deadline: + time.sleep(0.01) + + +def test_sync_completion_mirrors_to_the_silent_model_on_a_background_thread( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + core.completion(model="g", messages=MESSAGES, metadata={"trace": "t"}) # pyright: ignore[reportAttributeAccessIssue] + _wait_for_calls(host, 2) + # the thread starts before the primary call, so the two calls can land in either order + assert sorted((call.op, call.deployment_id) for call in host.calls) == [("acompletion", "s1"), ("completion", "d1")] + primary: Final = next(call for call in host.calls if call.deployment_id == "d1") + shadow: Final = _metadata(next(call for call in host.calls if call.deployment_id == "s1").kwargs) + assert (shadow["is_silent_experiment"], shadow["model_group"], shadow["trace"]) == (True, "shadow", "t") + assert "is_silent_experiment" not in _metadata(primary.kwargs) + + +async def test_a_request_already_marked_silent_does_not_start_another_experiment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + await core.acompletion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue] + await _settle() + assert host.deployments_called() == ["d1"] + + +def test_a_sync_request_already_marked_silent_does_not_start_another_experiment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + core.completion(model="g", messages=MESSAGES, metadata={"is_silent_experiment": True}) # pyright: ignore[reportAttributeAccessIssue] + time.sleep(0.2) + assert host.deployments_called() == ["d1"] + + +async def test_silent_experiment_marks_its_own_metadata_when_the_request_metadata_cannot_be_deep_copied( + make_core: MakeCore, host: ScriptedHost +) -> None: + lock: Final = threading.Lock() + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + await core.acompletion(model="g", messages=MESSAGES, metadata={"lock": lock}) # pyright: ignore[reportAttributeAccessIssue] + await _settle() + primary: Final = next(call for call in host.calls if call.deployment_id == "d1") + shadow: Final = next(call for call in host.calls if call.deployment_id == "s1") + assert "is_silent_experiment" not in _metadata(primary.kwargs) + assert _metadata(primary.kwargs)["model_group"] == "g" + assert (_metadata(shadow.kwargs)["is_silent_experiment"], _metadata(shadow.kwargs)["lock"]) == (True, lock) + + +async def test_a_streaming_request_mirrors_a_streaming_silent_experiment( + make_core: MakeCore, host: ScriptedHost +) -> None: + from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts + + host.answer("openai/d1", chat_stream("A", model="d1")) + host.answer("openai/s1", chat_stream("S", model="s1")) + core: Final = make_core(model_list=[deployment("g", "d1", silent_model="shadow"), deployment("shadow", "s1")]) + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] + assert await collect_texts(stream) == "A" + await _settle() + shadow: Final = next(call for call in host.calls if call.deployment_id == "s1") + assert shadow.kwargs["stream"] is True + + +# --- per-deployment num_retries stamped on the error ------------------------------------------ + + +async def test_a_num_retries_already_on_the_error_wins_over_the_deployment_num_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + error: Final = rate_limit() + error.num_retries = 0 # pyright: ignore[reportAttributeAccessIssue] + host.answer("openai/d1", error) + core: Final = make_core(model_list=[deployment("g", "d1", num_retries=2)]) + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert len(host.calls) == 1 + + +async def test_a_string_deployment_num_retries_is_used_as_an_int(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/d1", rate_limit()) + core: Final = make_core(model_list=[deployment("g", "d1", num_retries="1")]) + with pytest.raises(litellm.RateLimitError) as raised: + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert len(host.calls) == 2 + assert getattr(raised.value, "num_retries") == 1 + + +async def test_a_non_numeric_deployment_num_retries_is_ignored(make_core: MakeCore, host: ScriptedHost) -> None: + """The router's own default (2 retries) applies when the deployment value cannot be read as an int.""" + host.answer("openai/d1", rate_limit()) + core: Final = make_core(model_list=[deployment("g", "d1", num_retries="abc")]) + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert len(host.calls) == 3 + + +# --- request kwargs built for the deployment -------------------------------------------------- + + +async def test_responses_replace_kubernetes_pod_routing_in_the_request_metadata( + make_core: MakeCore, host: ScriptedHost +) -> None: + """On the Responses path the routing goes to ``litellm_metadata``; a pod routing key the caller put + in ``metadata`` is overwritten with the serving deployment's (none here).""" + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] + model="g", input="hi", metadata={KUBERNETES_POD_ROUTING_KEY: {"pod": "stale"}, "keep": 1} + ) + sent: Final = host.calls[0].kwargs + assert _metadata(sent) == {KUBERNETES_POD_ROUTING_KEY: None, "keep": 1} + assert _metadata(sent, "litellm_metadata")[KUBERNETES_POD_ROUTING_KEY] is None + + +async def test_a_baggage_stamp_records_the_serving_deployment_tags(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", tags=["t1", "t2"])]) + stamp: Final = BaggageMappedTagsStamp(tags=("b1",)) + await core.acompletion( # pyright: ignore[reportAttributeAccessIssue] + model="g", messages=MESSAGES, metadata={BAGGAGE_MAPPED_TAGS_METADATA_KEY: stamp, "tags": ["b1"]} + ) + metadata: Final = _metadata(host.calls[0].kwargs) + assert metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] == BaggageMappedTagsStamp( + tags=("b1",), serving_deployment_tags=("t1", "t2") + ) + assert metadata["tags"] == ["t1", "t2", "b1"] + + +@pytest.mark.parametrize( + ("request_extra", "injected"), + [ + ({"stream": True}, True), + ({"stream": True, "stream_options": {"include_usage": False}}, False), + ({}, False), + ], +) +async def test_a_deployment_stream_options_include_usage_marks_the_streaming_request_as_injected( + make_core: MakeCore, host: ScriptedHost, request_extra: dict[str, object], injected: bool +) -> None: + from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts + + host.answer("openai/d1", chat_stream("A")) + core: Final = make_core(model_list=[deployment("g", "d1", stream_options={"include_usage": True})]) + response: Final = await core.acompletion(model="g", messages=MESSAGES, **request_extra) # pyright: ignore[reportAttributeAccessIssue] + if request_extra.get("stream"): + await collect_texts(response) + assert host.calls[0].kwargs.get(STREAM_USAGE_INJECTED_FIELD, False) is injected + + +async def test_malformed_request_tools_are_dropped_for_the_deployment_tools_when_drop_params_is_set( + make_core: MakeCore, host: ScriptedHost +) -> None: + tool: Final = {"type": "function", "function": {"name": "dep_tool", "parameters": {}}} + core: Final = make_core(model_list=[deployment("g", "d1", tools=[tool], drop_params=True)]) + await core.acompletion(model="g", messages=MESSAGES, tools="not-a-list", tool_choice="auto") # pyright: ignore[reportAttributeAccessIssue] + sent: Final = host.calls[0].kwargs + assert sent["tools"] == [tool] + assert "tool_choice" not in sent + + +async def test_a_per_user_github_copilot_deployment_rejects_another_credential_name( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[ + deployment( + "g", + "d1", + model="github_copilot/gpt-4o", + litellm_credential_name="copilot-cred", + github_copilot_auth_type="per_user_oauth", + ) + ], + num_retries=0, + ) + with pytest.raises(litellm.BadRequestError) as raised: + await core.acompletion(model="g", messages=MESSAGES, litellm_credential_name="someone-else") # pyright: ignore[reportAttributeAccessIssue] + assert "litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth" in str( + raised.value + ) + assert host.calls == [] + + +# --- timeouts --------------------------------------------------------------------------------- + + +async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + """Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's + process settings), and only when the router has a ``timeout`` of its own.""" + monkeypatch.setattr(litellm, "request_timeout", 33.0) + core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[0].kwargs["timeout"] == 33.0 + + +async def test_the_process_request_timeout_is_the_stream_timeout_without_a_stream_timeout( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts + + monkeypatch.setattr(litellm, "request_timeout", 33.0) + host.answer("openai/d1", chat_stream("A")) + core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) + await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue] + assert (host.calls[0].kwargs["stream_timeout"], host.calls[0].kwargs["timeout"]) == (33.0, 33.0) + + +async def test_default_litellm_params_timeout_applies_when_no_router_or_process_timeout_is_set( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(request_timeout=None) + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"timeout": 9}) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[0].kwargs["timeout"] == 9 + + +async def test_default_litellm_params_stream_timeout_is_the_last_stream_timeout_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts + + host.answer("openai/d1", chat_stream("A")) + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"stream_timeout": 4}) + await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[0].kwargs["stream_timeout"] == 4 + + +# --- sync completion runs the routing pre-call checks for a model group only ------------------- + + +def test_sync_completion_runs_routing_pre_call_checks_for_a_group_but_not_a_deployment_id( + make_core: MakeCore, host: ScriptedHost +) -> None: + from litellm.integrations.custom_logger import CustomLogger + + checked: Final[list[object]] = [] + + class Recorder(CustomLogger): + def pre_call_check(self, deployment: dict) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride] + checked.append(deployment["model_info"]["id"]) + return None + + host.lists["callbacks"].append(Recorder()) + core: Final = make_core(model_list=[deployment("g", "d1")]) + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + core.completion(model="d1", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert checked == ["d1"] + assert host.deployments_called() == ["d1", "d1"] + + +def test_a_sync_routing_pre_call_check_error_stops_the_call(make_core: MakeCore, host: ScriptedHost) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class OverLimit(CustomLogger): + def pre_call_check(self, deployment: dict) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride] + raise litellm.RateLimitError(message="over the rpm limit", llm_provider="openai", model="d1") + + host.lists["callbacks"].append(OverLimit()) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(litellm.RateLimitError, match="over the rpm limit"): + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + + +# --- per-model call counters ------------------------------------------------------------------ + + +async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_model( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + host.answer("openai/d1", rate_limit()) + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + counts: Final = ( + core.total_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue] + core.success_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue] + core.fail_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue] + ) + assert counts == (2, 1, 1) + + +def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None: + """Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``.""" + core: Final = make_core(model_list=[deployment("g", "d1")]) + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert core.total_calls["openai/d1"] == 0 # pyright: ignore[reportAttributeAccessIssue] diff --git a/tests/unit/test_router/core/test_gaps_l2800_streaming.py b/tests/unit/test_router/core/test_gaps_l2800_streaming.py new file mode 100644 index 00000000000..42c95c6f894 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l2800_streaming.py @@ -0,0 +1,177 @@ +""" +Gap tests for the stream wrappers of ``router_python_core.py`` (lines 2800-5600): what a mid-stream +fallback yields when the fallback answers without a stream, the usage on fallback chunks, and the +Responses stream's handling of an error that is not a mid-stream fallback error. + +Each test asserts what the Python core does today; oddities are named in the test's docstring. +""" + +from collections.abc import AsyncIterator, Callable +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.utils import ModelResponseStream, Usage +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + bad_request, + completion_response, + deployment, + rate_limit, +) +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGES, + Event, + chat_stream, + drain_into, + finish_chunk, + responses_stream, + text_chunk, +) + +MakeCore = Callable[..., RouterCore] +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] + + +def build(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(**{"model_list": GROUPS_AB, "fallbacks": [{"a": ["b"]}], "num_retries": 0, **arguments}) + + +def stream_with_usage(sync: bool = False) -> Callable[..., object]: + """A bare chunk iterator (not a ``CustomStreamWrapper``, which would strip the usage) of one text + chunk and a finish chunk that carries usage.""" + + def answer(**_request: object) -> object: # kwargs-ok: the host passes the request kwargs + chunks: Final = [ + text_chunk("B"), + finish_chunk(usage={"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}), + ] + + async def achunks() -> AsyncIterator[ModelResponseStream]: + for chunk in chunks: + yield chunk + + return iter(chunks) if sync else achunks() + + return answer + + +# --- a fallback that answers without a stream -------------------------------------------------- + + +async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place.""" + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", completion_response("not streamed", model="b")) + core: Final = build(make_core) + stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] + seen: list[object] = [] + await drain_into(stream, seen) + assert seen[-1] is None + assert host.deployments_called() == ["a", "b"] + + +def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync + wrapper streams the fallback response's fields instead of chunks.""" + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True)) + host.answer("openai/b", completion_response("not streamed", model="b")) + core: Final = build(make_core) + stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] + items: Final = list(stream) # pyright: ignore[reportArgumentType] + fields: Final = [item for item in items if isinstance(item, tuple)] + assert ("model", "b") in fields + assert {name for name, _ in fields} >= {"id", "choices", "usage"} + assert host.deployments_called() == ["a", "b"] + + +# --- usage on fallback chunks ------------------------------------------------------------------ + + +async def test_a_mid_stream_fallback_chunk_dict_usage_becomes_an_all_zero_usage( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: a fallback chunk's usage is rebuilt through ``combine_usage_objects``, which reads token + counts as attributes, so a usage given as a plain dict comes out as an all-zero ``Usage``.""" + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", stream_with_usage()) + core: Final = build(make_core) + stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] + seen: list[object] = [] + await drain_into(stream, seen) + usages: Final = [ + getattr(item, "usage") + for item in seen + if isinstance(item, ModelResponseStream) and getattr(item, "usage", None) is not None + ] + assert usages + assert isinstance(usages[-1], Usage) + assert (usages[-1].prompt_tokens, usages[-1].completion_tokens, usages[-1].total_tokens) == (0, 0, 0) + + +def test_a_sync_mid_stream_fallback_chunk_dict_usage_becomes_an_all_zero_usage( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: as in the async wrapper, a plain-dict usage comes out as an all-zero ``Usage``.""" + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True)) + host.answer("openai/b", stream_with_usage(sync=True)) + core: Final = build(make_core) + stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] + usages: Final = [ + getattr(item, "usage") + for item in stream # pyright: ignore[reportGeneralTypeIssues] + if isinstance(item, ModelResponseStream) and getattr(item, "usage", None) is not None + ] + assert usages + assert isinstance(usages[-1], Usage) + assert (usages[-1].prompt_tokens, usages[-1].completion_tokens, usages[-1].total_tokens) == (0, 0, 0) + + +# --- Responses streams ------------------------------------------------------------------------- + + +async def test_a_responses_error_that_is_not_a_mid_stream_error_flushes_held_events_then_raises( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + responses_stream(Event("response.created"), Event("response.in_progress"), error=bad_request(message="plain")), + ) + host.answer("openai/b", responses_stream(Event("response.created"), Event("response.completed"))) + core: Final = build(make_core) + stream: Final = await core.aresponses(model="a", input="hi", stream=True) # pyright: ignore[reportAttributeAccessIssue] + seen: list[object] = [] + with pytest.raises(litellm.BadRequestError, match="plain"): + await drain_into(stream, seen) + assert [getattr(event, "type") for event in seen] == ["response.created", "response.in_progress"] + assert host.deployments_called() == ["a"] + + +async def test_a_responses_stream_fallback_that_answers_without_a_stream_yields_that_response( + make_core: MakeCore, host: ScriptedHost +) -> None: + from litellm.exceptions import MidStreamFallbackError + + cut: Final = MidStreamFallbackError( + message="stream cut", + model="a", + llm_provider="anthropic", + original_exception=rate_limit(message="stream cut"), + generated_content="", + is_pre_first_chunk=True, + ) + plain: Final = completion_response("not streamed", model="b") + host.answer("openai/a", responses_stream(Event("response.created"), error=cut)) + host.answer("openai/b", plain) + core: Final = build(make_core) + stream: Final = await core.aresponses(model="a", input="hi", stream=True) # pyright: ignore[reportAttributeAccessIssue] + seen: list[object] = [] + await drain_into(stream, seen) + assert seen == [plain] + assert host.deployments_called() == ["a", "b"] diff --git a/tests/unit/test_router/core/test_gaps_l5600.py b/tests/unit/test_router/core/test_gaps_l5600.py new file mode 100644 index 00000000000..83ab8e943b9 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l5600.py @@ -0,0 +1,479 @@ +""" +Gap characterization for ``router_python_core.py`` lines 5600-8400: batch completions, the scheduler, +prompt-management groups, the image/audio/rerank/text/adapter endpoints, generic calls and their +pass-through endpoint rewrite, guardrail selection, embeddings, files and batches. + +Each test pins a behavior the earlier characterization files leave unasserted; oddities are pinned as +Python does them today and called out in the test's docstring. +""" + +import datetime +import io +from collections.abc import Callable, Mapping +from types import SimpleNamespace +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router_core import RouterCore +from litellm.types.utils import CredentialItem, LiteLLMBatch +from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + deployment, + rate_limit, + server_error, +) + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] +TWO_DEPLOYMENTS: Final = [deployment("g", "d1"), deployment("g", "d2")] + + +@pytest.fixture +def host() -> EndpointHost: + return EndpointHost() + + +def _batch() -> LiteLLMBatch: + return LiteLLMBatch( + id="b1", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="f", + object="batch", + status="completed", + ) + + +def _hidden(response: object) -> Mapping[str, object]: + return getattr(response, "_hidden_params") + + +def _headers(response: object) -> Mapping[str, object]: + found: Final = _hidden(response)["additional_headers"] + assert isinstance(found, Mapping) + return found + + +# The routed endpoints that answer through ``async_function_with_fallbacks`` with their own attempt helper. +ROUTED_ENDPOINTS: Final[dict[str, dict[str, object]]] = { + "aimage_generation": {"prompt": "p"}, + "atranscription": {"file": b"audio"}, + "aspeech": {"input": "x", "voice": "alloy"}, + "arerank": {"query": "q", "documents": ["a"]}, + "aadapter_completion": {"adapter_id": "ad", "messages": MESSAGES}, + "atext_completion": {"prompt": "p"}, + "aembedding": {"input": ["a"]}, + "acreate_batch": {"input_file_id": "f", "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + "acancel_batch": {"batch_id": "b1"}, +} + + +# --- batch completions (5565-5790) ---------------------------------------------------------- + + +async def test_abatch_completion_with_message_lists_keeps_a_failed_request_in_its_own_row( + make_core: MakeCore, host: EndpointHost +): + host.answer("openai/d1", rate_limit("d1", "a failed")) + core: Final = make_core(model_list=[deployment("a", "d1"), deployment("b", "d2")], num_retries=0) + rows: Final = await core.abatch_completion(models=["a", "b"], messages=[MESSAGES, MESSAGES]) + assert [[type(result).__name__ for result in row] for row in rows] == [ + ["RateLimitError", "ModelResponse"], + ["RateLimitError", "ModelResponse"], + ] + + +async def test_abatch_completion_with_mixed_message_shapes_returns_none_without_calling( + make_core: MakeCore, host: EndpointHost +): + """Neither every element a dict nor every element a list: no branch matches and None comes back.""" + core: Final = make_core(model_list=[deployment("a", "d1")]) + assert await core.abatch_completion(models=["a"], messages=[MESSAGES, {"role": "user", "content": "x"}]) is None + assert host.calls == [] + + +async def test_fastest_response_forwards_the_stream_flag(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("a", "d1")]) + await core.abatch_completion_fastest_response(model="a", messages=MESSAGES, stream=True) + assert host.calls[0].kwargs["stream"] is True + + +# --- scheduler (5792-5912) ------------------------------------------------------------------ + + +async def test_schedule_acompletion_forwards_the_stream_flag(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.schedule_acompletion(model="g", messages=MESSAGES, priority=1, stream=True) + assert host.calls[0].kwargs["stream"] is True + + +async def test_a_prioritized_request_for_a_deployment_id_is_queued_and_served_by_it( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + for _ in range(3): + response = await core.acompletion(model="d1", messages=MESSAGES, priority=2) + assert _headers(response)["x-litellm-request-prioritization-used"] is True + assert host.deployments_called() == ["d1", "d1", "d1"] + + +async def test_a_bad_priority_for_a_deployment_id_follows_that_deployments_drop_params( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", drop_params=True), deployment("g", "d2")]) + response: Final = await core.acompletion(model="d1", messages=MESSAGES, priority="high") + assert "x-litellm-request-prioritization-used" not in _headers(response) + assert "priority" not in host.calls[0].kwargs + with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'high'"): + await core.acompletion(model="d2", messages=MESSAGES, priority="high") + assert host.deployments_called() == ["d1"] + + +async def test_a_bad_priority_is_dropped_when_any_deployment_of_the_group_drops_params( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", drop_params=True), deployment("g", "d2")]) + response: Final = await core.acompletion(model="g", messages=MESSAGES, priority="high") + assert "x-litellm-request-prioritization-used" not in _headers(response) + assert len(host.calls) == 1 + + +# --- prompt-management groups (5893-6034) --------------------------------------------------- + + +class _PromptLogging(LiteLLMLogging): + """A logging object whose prompt template renders to ``rendered_model`` with a system message.""" + + rendered_model: str = "openai/rendered" + seen: Mapping[str, object] = {} + + def get_chat_completion_prompt(self, **kwargs: object): # kwargs-ok: Logging's own keyword signature + self.seen = kwargs + return self.rendered_model, [{"role": "system", "content": "rendered"}], {"temperature": 0.7} + + +def _prompt_logging(rendered_model: str = "openai/rendered") -> _PromptLogging: + logging_obj: Final = _PromptLogging( + model="pm", + messages=MESSAGES, + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="c1", + function_id="f1", + ) + logging_obj.rendered_model = rendered_model + return logging_obj + + +async def test_a_prompt_management_group_renders_the_deployment_prompt_and_calls_the_rendered_model( + make_core: MakeCore, host: EndpointHost +): + """The deployment's prompt_id/variables/label reach the template; the prompt-management config keys + (bitbucket_config, ...) are stripped from what is sent; the rendered model, messages and optional + params are sent directly (no routing hidden params) since the rendered model is no group.""" + core: Final = make_core( + model_list=[ + deployment( + "pm", + "p1", + model="langfuse/gpt-4o", + prompt_id="dep-pid", + prompt_variables={"a": 1}, + prompt_label="lbl", + bitbucket_config={"x": 1}, + timeout=7, + ) + ] + ) + logging_obj: Final = _prompt_logging() + response: Final = await core.acompletion(model="pm", messages=MESSAGES, litellm_logging_obj=logging_obj) + call: Final = host.calls[0] + assert (call.op, call.model, call.deployment_id, call.model_group) == ("acompletion", "openai/rendered", "p1", "pm") + sent: Final = call.kwargs + assert (sent["prompt_id"], sent["prompt_variables"], sent["prompt_label"]) == ("dep-pid", {"a": 1}, "lbl") + assert "bitbucket_config" not in sent + assert (sent["temperature"], sent["timeout"], sent["messages"]) == ( + 0.7, + 7.0, + [{"role": "system", "content": "rendered"}], + ) + assert sent["litellm_logging_obj"] is logging_obj + assert (logging_obj.seen["model"], logging_obj.seen["messages"], logging_obj.seen["prompt_id"]) == ( + "langfuse/gpt-4o", + MESSAGES, + "dep-pid", + ) + assert logging_obj.seen["injected_for_every_deployment"] is True + assert _hidden(response) == {} + + +async def test_a_prompt_rendered_to_another_group_routes_into_it_with_the_request_prompt_id_and_timeout( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[ + deployment("pm", "p1", model="langfuse/gpt-4o", prompt_id="dep-pid", timeout=9), + deployment("rendered-g", "r1"), + ] + ) + response: Final = await core.acompletion( + model="pm", + messages=MESSAGES, + litellm_logging_obj=_prompt_logging("rendered-g"), + prompt_id="req-pid", + timeout=3, + ) + call: Final = host.calls[0] + assert (call.model, call.model_group, call.kwargs["prompt_id"], call.kwargs["timeout"]) == ( + "openai/r1", + "pm", + "req-pid", + 3, + ) + assert _headers(response)["x-litellm-model-group"] == "rendered-g" + + +async def test_a_single_deployment_without_a_provider_prefix_is_not_a_prompt_management_group( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core(model_list=[deployment("g", "d1", model="gpt-4o")]) + response: Final = await core.acompletion(model="g", messages=MESSAGES) + assert (host.calls[0].model, host.calls[0].deployment_id) == ("gpt-4o", "d1") + assert _hidden(response)["model_id"] == "d1" + + +# --- routed endpoints: retries and counters (6036-6603, 7646-7789, 8026-8315) ------------- + + +@pytest.mark.parametrize("endpoint", sorted(ROUTED_ENDPOINTS)) +async def test_a_routed_endpoint_retries_a_401_on_the_other_deployment( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + for _ in range(4): + host.calls.clear() + host.answer("*", auth_error(), None) + core = make_core(model_list=TWO_DEPLOYMENTS, num_retries=1, disable_cooldowns=True) + await core.call(endpoint, {"model": "g", **ROUTED_ENDPOINTS[endpoint]}) + assert sorted(host.deployments_called()) == ["d1", "d2"] + + +@pytest.mark.parametrize("endpoint", sorted(ROUTED_ENDPOINTS)) +async def test_a_routed_endpoint_ignores_the_deployment_num_retries( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + """Unlike acompletion and the generic endpoints, these attempt helpers never stamp the deployment's + num_retries on the error, so the router's num_retries=0 governs.""" + host.answer("*", server_error()) + core: Final = make_core(model_list=[deployment("g", "d1", num_retries=2)], num_retries=0) + with pytest.raises(litellm.InternalServerError): + await core.call(endpoint, {"model": "g", **ROUTED_ENDPOINTS[endpoint]}) + assert len(host.calls) == 1 + assert host.raised[-1].num_retries is None # pyright: ignore[reportAttributeAccessIssue] # stamped by the retry loop + + +async def test_a_generic_endpoint_retries_with_the_deployment_num_retries( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost +): + patch_direct_endpoints(monkeypatch, host) + host.answer("*", server_error()) + core: Final = make_core(model_list=[deployment("g", "d1", num_retries=2)], num_retries=0) + with pytest.raises(litellm.InternalServerError) as raised: + await core.aresponses(model="g", input="hi") + assert len(host.calls) == 3 + assert raised.value.num_retries == 2 # pyright: ignore[reportAttributeAccessIssue] # stamped from the deployment + + +async def test_sync_image_generation_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost): + host.answer("*", rate_limit()) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=2) + with pytest.raises(litellm.RateLimitError): + core.image_generation(model="g", prompt="p") + assert len(host.calls_to("image_generation")) == 3 + + +@pytest.mark.parametrize("endpoint", ["aimage_generation", "image_generation"]) +async def test_image_generation_for_an_unknown_group_raises_without_calling_the_host( + make_core: MakeCore, host: EndpointHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(litellm.BadRequestError, match="There are no healthy deployments for this model"): + await core.call(endpoint, {"model": "nope", "prompt": "p"}) + assert host.calls == [] + + +@pytest.mark.parametrize( + ("endpoint", "total_key", "fail_key"), + [ + ("aimage_generation", "openai/d1", "openai/d1"), + ("atranscription", "g", "g"), + ("aspeech", "g", "g"), + ("arerank", "openai/d1", "openai/d1"), + ("aadapter_completion", "openai/d1", "g"), + ("atext_completion", "openai/d1", "g"), + ("aembedding", "openai/d1", "openai/d1"), + ("acreate_batch", "openai/d1", "g"), + ("acancel_batch", "openai/d1", "g"), + ("image_generation", "", ""), + ], +) +async def test_call_counters_are_keyed_differently_per_endpoint( + make_core: MakeCore, host: EndpointHost, endpoint: str, total_key: str, fail_key: str +): + """total_calls/fail_calls are keyed by the deployment model for some endpoints and by the model group + for others (fail_calls even differs from total_calls for some); sync image_generation counts under "".""" + host.answer("*", rate_limit()) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1) + request: Final = ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"}) + with pytest.raises(litellm.RateLimitError): + await core.call(endpoint, {"model": "g", **request}) + assert dict(core.total_calls) == {total_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter + assert dict(core.fail_calls) == {fail_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter + assert dict(core.success_calls) == {} # pyright: ignore[reportAttributeAccessIssue] # a public counter + + +async def test_success_counters_count_the_deployment_model(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.aspeech(model="g", input="x", voice="alloy") + await core.aembedding(model="g", input=["a"]) + assert dict(core.total_calls) == {"g": 1, "openai/d1": 1} # pyright: ignore[reportAttributeAccessIssue] # a public counter + assert dict(core.success_calls) == {"g": 1, "openai/d1": 1} # pyright: ignore[reportAttributeAccessIssue] # a public counter + + +# --- guardrails (6692-6721) ----------------------------------------------------------------- + + +async def test_aguardrail_picks_among_same_named_guardrails_by_weight(make_core: MakeCore, host: ScriptedHost): + picked: Final[list[object]] = [] + + async def guard(**kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + selected: Final = kwargs["selected_guardrail"] + assert isinstance(selected, Mapping) + picked.append(selected["litellm_params"]["guardrail"]) + return "verdict" + + guardrails: Final = [ + {"guardrail_name": "gr", "litellm_params": {"guardrail": "never", "weight": 0}}, + {"guardrail_name": "gr", "litellm_params": {"guardrail": "always", "weight": 1}}, + ] + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=guardrails) + for _ in range(10): + await core.aguardrail(guardrail_name="gr", original_function=guard) + assert set(picked) == {"always"} + + +async def test_aguardrail_without_weights_spreads_over_same_named_guardrails(make_core: MakeCore, host: ScriptedHost): + picked: Final[set[object]] = set() + + async def guard(**kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + selected: Final = kwargs["selected_guardrail"] + assert isinstance(selected, Mapping) + picked.add(selected["litellm_params"]["guardrail"]) + return "verdict" + + guardrails: Final = [ + {"guardrail_name": "gr", "litellm_params": {"guardrail": "a"}}, + {"guardrail_name": "gr", "litellm_params": {"guardrail": "b"}}, + ] + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=guardrails) + for _ in range(60): + await core.aguardrail(guardrail_name="gr", original_function=guard) + assert picked == {"a", "b"} + + +# --- generic calls: pass-through endpoint rewrite (6754-6781, 7561-7627) -------------------- + + +async def test_async_passthrough_rewrites_only_whole_path_segments_naming_the_group( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost +): + patch_direct_endpoints(monkeypatch, host) + core: Final = make_core(model_list=[deployment("g", "d1", model="bedrock/us.anthropic.claude")]) + await core.allm_passthrough_route(model="g", method="POST", endpoint="/model/g/invoke/gg/g:stream") + assert host.calls[0].kwargs["endpoint"] == "/model/us.anthropic.claude/invoke/gg/us.anthropic.claude:stream" + + +async def test_sync_generic_call_replaces_every_substring_of_the_group_name_in_the_endpoint( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost +): + """The sync path uses str.replace with the full deployment model (provider prefix kept), so a group + name inside other path segments is rewritten too, unlike the async path's segment-bounded rewrite.""" + patch_direct_endpoints(monkeypatch, host) + core: Final = make_core(model_list=[deployment("g", "d1")]) + core.responses(model="g", input="hi", endpoint="/gg/g") + call: Final = host.calls[0] + assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/openai/d1openai/d1/openai/d1") + + +# --- batches (8112-8362) -------------------------------------------------------------------- + + +async def test_aretrieve_batch_sends_the_deployment_provider_over_the_request_one( + make_core: MakeCore, host: EndpointHost +): + host.answer("*", _batch()) + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("h", "d2", custom_llm_provider="azure")]) + await core.aretrieve_batch(model="g", batch_id="b1", custom_llm_provider="azure") + await core.aretrieve_batch(model="h", batch_id="b1") + assert [call.kwargs["custom_llm_provider"] for call in host.calls] == ["openai", "azure"] + + +async def test_alist_batches_takes_the_first_set_first_id_and_the_last_answer_last_id( + make_core: MakeCore, host: EndpointHost +): + """last_id is overwritten by every answer in deployment order, even with None.""" + host.answer("openai/d1", SimpleNamespace(first_id=None, last_id="b", data=[1], has_more=False)) + host.answer("openai/d2", SimpleNamespace(first_id="c", last_id=None, data=[2], has_more=False)) + core: Final = make_core(model_list=TWO_DEPLOYMENTS) + assert await core.alist_batches(model="g") == { + "object": "list", + "data": [1, 2], + "first_id": "c", + "last_id": None, + "has_more": False, + } + + +async def test_acancel_batch_resolves_the_deployment_credential_but_acreate_batch_does_not( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost +): + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="cred1", + credential_values={"api_key": "sk-cred", "api_base": "https://cred.example"}, + credential_info={}, + ) + ], + ) + host.answer("*", _batch()) + core: Final = make_core(model_list=[deployment("g", "d1", litellm_credential_name="cred1")]) + await core.acancel_batch(model="g", batch_id="b1") + await core.acreate_batch(model="g", **ROUTED_ENDPOINTS["acreate_batch"]) + cancel, create = host.calls[0].kwargs, host.calls[1].kwargs + assert (cancel["api_key"], cancel["api_base"], "litellm_credential_name" in cancel) == ( + "sk-cred", + "https://cred.example", + False, + ) + assert (create["api_key"], create.get("api_base"), create["litellm_credential_name"]) == ("sk-test", None, "cred1") + + +# --- files (7814-7934) ---------------------------------------------------------------------- + + +async def test_acreate_file_passthrough_keeps_the_batch_file_model(make_core: MakeCore, host: EndpointHost): + upload: Final = io.BytesIO(b'{"custom_id": "1", "body": {"model": "g", "messages": []}}\n') + core: Final = make_core(model_list=[deployment("g", "d1")]) + await core.acreate_file(model="g", file=upload, purpose="batch", passthrough=True) + sent: Final = host.calls[0].kwargs["file"] + assert isinstance(sent, io.BytesIO) + assert b'"model": "g"' in sent.getvalue() diff --git a/tests/unit/test_router/core/test_gaps_l5600_anthropic.py b/tests/unit/test_router/core/test_gaps_l5600_anthropic.py new file mode 100644 index 00000000000..e2b19058a50 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l5600_anthropic.py @@ -0,0 +1,103 @@ +""" +Gap characterization for /v1/messages streaming recovery (``router_python_core.py`` 7106-7455): a +``MidStreamFallbackError`` raised by the source stream itself, a same-group stream retry whose own +attempt raises before opening, and a request-level retry policy that is not a ``RetryPolicy``. +""" + +from collections.abc import Callable +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit, server_error +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGE_START, + MESSAGE_STOP, + MESSAGES, + anthropic_stream, + collect_bytes, + drain_into, + error_frame, + install_orjson_stub_if_missing, + text_delta, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] + + +async def _open(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.aanthropic_messages(model="a", messages=MESSAGES, max_tokens=5, stream=True, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def _mid_stream(*, pre_first_chunk: bool) -> MidStreamFallbackError: + return MidStreamFallbackError( + message="bridge dropped", + model="a", + llm_provider="openai", + original_exception=rate_limit("a", "bridge original"), + is_pre_first_chunk=pre_first_chunk, + ) + + +async def test_a_raised_mid_stream_error_not_before_the_first_chunk_flushes_held_frames_and_raises_the_original( + make_core: MakeCore, host: ScriptedHost +): + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=_mid_stream(pre_first_chunk=False))) + host.answer("openai/b", anthropic_stream(MESSAGE_START, text_delta("B"), MESSAGE_STOP)) + core: Final = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream: Final = await _open(core) + seen: Final[list[object]] = [] + with pytest.raises(litellm.RateLimitError, match="bridge original"): + await drain_into(stream, seen) + assert seen == [MESSAGE_START] + assert host.deployments_called() == ["a"] + + +async def test_a_raised_mid_stream_error_before_the_first_chunk_falls_back(make_core: MakeCore, host: ScriptedHost): + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=_mid_stream(pre_first_chunk=True))) + host.answer("openai/b", anthropic_stream(MESSAGE_START, text_delta("B"), MESSAGE_STOP)) + core: Final = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames: Final = await collect_bytes(await _open(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_a_stream_retry_that_raises_before_opening_spends_a_retry_and_the_next_one_serves( + make_core: MakeCore, host: ScriptedHost +): + host.answer( + "openai/a", + anthropic_stream(MESSAGE_START, error_frame()), + server_error("a"), + anthropic_stream(MESSAGE_START, text_delta("A"), MESSAGE_STOP), + ) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=2) + stream: Final = await _open(core) + frames: Final = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "a", "a"] + assert len(host.sleeps) == 2 + headers: Final = getattr(stream, "_hidden_params")["additional_headers"] + assert (headers["x-litellm-attempted-retries"], headers["x-litellm-max-retries"]) == (2, 2) + + +async def test_a_request_retry_policy_that_is_not_a_retry_policy_streams_with_the_plain_budget( + make_core: MakeCore, host: ScriptedHost +): + host.answer( + "openai/a", + anthropic_stream(MESSAGE_START, error_frame("rate_limit_error", "slow")), + anthropic_stream(MESSAGE_START, text_delta("A"), MESSAGE_STOP), + ) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=1) + stream: Final = await _open(core, model_group_retry_policy={"a": 5}) + frames: Final = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "a"] + assert getattr(stream, "_hidden_params")["additional_headers"]["x-litellm-max-retries"] == 1 diff --git a/tests/unit/test_router/core/test_gaps_l8400_precall.py b/tests/unit/test_router/core/test_gaps_l8400_precall.py new file mode 100644 index 00000000000..1e64257bac8 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l8400_precall.py @@ -0,0 +1,178 @@ +""" +Gaps in the routing-strategy pre-call checks and the Responses-websocket turn admission of ``PythonCore`` +(router_python_core.py ``routing_strategy_pre_call_checks``, ``async_routing_strategy_pre_call_checks``, +``async_admit_deployment_turn``). A ``CustomLogger`` on the host's ``callbacks`` list (``litellm.callbacks``) +takes part in those checks through ``pre_call_check`` / ``async_pre_call_check``. +""" + +from collections.abc import Callable +from typing import Final + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_core import RouterCore +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.retries_health_helpers import attempt, attempt_sync, cooled_down +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit + +MakeCore = Callable[..., RouterCore] + + +class PreCallGate(CustomLogger): + """Records the deployment ids its pre-call checks saw and raises ``error`` (when set) from them.""" + + def __init__(self, error: Exception | None = None) -> None: + super().__init__() + self.error = error + self.seen: Final[list[tuple[str, str]]] = [] + + async def async_pre_call_check(self, deployment: dict, parent_otel_span: object) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride] + self.seen.append(("async", deployment["model_info"]["id"])) + if self.error is not None: + raise self.error + return None + + def pre_call_check(self, deployment: dict) -> dict | None: + self.seen.append(("sync", deployment["model_info"]["id"])) + if self.error is not None: + raise self.error + return None + + +def gated(host: ScriptedHost, error: Exception | None = None) -> PreCallGate: + gate: Final = PreCallGate(error) + host.lists["callbacks"].append(gate) + return gate + + +GROUP: Final = [deployment("g", "a"), deployment("g", "b")] + + +# --- async pre-call checks ---------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_rate_limit_from_a_pre_call_check_cools_the_deployment_down_and_is_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33) + gate: Final = gated(host, rate_limit(message="gate says no")) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert str(error) == "litellm.RateLimitError: gate says no" + assert host.calls == [] + ((kind, checked),) = gate.seen + assert kind == "async" + cooled: Final = cooled_down(core, ["a", "b"]) + assert list(cooled) == [checked] + assert cooled[checked]["cooldown_time"] == 33 + + +@pytest.mark.asyncio +async def test_any_other_error_from_a_pre_call_check_is_raised_without_a_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33) + gate: Final = gated(host, ValueError("gate broke")) + + error: Final = await attempt(core) + + assert type(error) is ValueError + assert str(error) == "gate broke" + assert host.calls == [] + assert len(gate.seen) == 1 + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.asyncio +async def test_a_passing_pre_call_check_sees_the_deployment_that_is_then_called( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUP, num_retries=0) + gate: Final = gated(host) + + await attempt(core) + + assert gate.seen == [("async", host.deployments_called("acompletion")[0])] + + +def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the sync twin raises the gate's error but, unlike the async path, sets no cooldown.""" + core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33) + gate: Final = gated(host, rate_limit(message="gate says no")) + + error: Final = attempt_sync(core) + + assert isinstance(error, litellm.RateLimitError) + assert host.calls == [] + assert [kind for kind, _ in gate.seen] == ["sync"] + assert cooled_down(core, ["a", "b"]) == {} + + +# --- Responses websocket turn admission ---------------------------------------------------------- + + +class TestResponsesWebsocketTurnAdmission: + """The core hands the websocket handler a ``turn_admission`` callable that re-runs the deployment's + admission (budget limiter, routing strategy, pre-call checks) before every later turn.""" + + @pytest.fixture + def host(self) -> EndpointHost: + return EndpointHost() + + @staticmethod + async def open_socket(core: RouterCore, host: EndpointHost) -> Callable[[], object]: + await core.call("_aresponses_websocket", {"model": "g"}) + (call,) = host.calls_to("_aresponses_websocket") + admission: Final = call.kwargs["turn_admission"] + assert callable(admission) + return admission + + @pytest.mark.asyncio + async def test_simple_shuffle_admits_every_turn_and_runs_the_pre_call_checks( + self, make_core: MakeCore, host: EndpointHost + ) -> None: + core: Final = make_core(model_list=[deployment("g", "a", rpm=1)]) + gate: Final = gated(host) + admit: Final = await self.open_socket(core, host) + + admitted: Final = [dict(await admit()) for _ in range(3)] # pyright: ignore[reportGeneralTypeIssues] + + assert admitted == [{}, {}, {}] + assert gate.seen == [("async", "a")] * 4 # the socket opening, then each turn + + @pytest.mark.asyncio + async def test_a_rate_limit_from_a_pre_call_check_refuses_the_turn( + self, make_core: MakeCore, host: EndpointHost + ) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + gate: Final = gated(host) + admit: Final = await self.open_socket(core, host) + gate.error = rate_limit(message="turn refused") + + with pytest.raises(litellm.RateLimitError, match="turn refused"): + await admit() # pyright: ignore[reportGeneralTypeIssues] + + @pytest.mark.asyncio + async def test_usage_based_routing_admits_turns_until_the_deployment_rpm_is_used_up( + self, make_core: MakeCore, host: EndpointHost + ) -> None: + """Opening the socket already counts two requests (its pre-call check and its success), so an + rpm of 5 leaves room for three turns.""" + core: Final = make_core(model_list=[deployment("g", "a", rpm=5)], routing_strategy="usage-based-routing-v2") + admit: Final = await self.open_socket(core, host) + + admitted: Final = [dict(await admit()) for _ in range(3)] # pyright: ignore[reportGeneralTypeIssues] + with pytest.raises(litellm.RateLimitError) as refused: + await admit() # pyright: ignore[reportGeneralTypeIssues] + + assert admitted == [{}, {}, {}] + assert str(refused.value).startswith( + "litellm.RateLimitError: No deployments available for selected model. Passed model=g. Deployments=" + ) diff --git a/tests/unit/test_router/core/test_gaps_l8400_pricing.py b/tests/unit/test_router/core/test_gaps_l8400_pricing.py new file mode 100644 index 00000000000..4c0df5a17b6 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l8400_pricing.py @@ -0,0 +1,240 @@ +""" +Gaps in the built-in pricing a custom-priced deployment inherits when the core registers it in +``litellm.model_cost`` (router_python_core.py ``_inherit_builtin_cache_pricing``, ``_cost_map_backend_model``, +``_inherit_builtin_service_tier_pricing``, ``_inherit_builtin_base_rates_for_off_peak``, +``_inherit_builtin_tiered_output_rate``), and the deployment checks of ``_create_deployment`` +(access windows, PTU configuration). Expected rates are read from the built-in cost map, not hardcoded. +""" + +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from tests.unit.test_router.core.registry_helpers import MakeCore, unique_id +from tests.unit.test_router.core.scripted_host import deployment + +OFF_PEAK: Final = {"hours_utc": "01:00-02:00", "input_cost_per_token": 1e-9} + + +def registered(make_core: MakeCore, model: str, **params: object) -> Mapping[str, object]: # kwargs-ok: litellm_params + """The ``litellm.model_cost`` entry registered under a fresh deployment id.""" + model_id: Final = unique_id("l8400") + make_core(model_list=[deployment("g", model_id, model=model, **params)]) + return litellm.model_cost[model_id] + + +def builtin(key: str) -> Mapping[str, object]: + return litellm.model_cost[key] + + +# --- service-tier rates --------------------------------------------------------------------------- + + +def test_custom_input_pricing_inherits_the_backend_service_tier_rates(make_core: MakeCore) -> None: + entry: Final = registered(make_core, "openai/gpt-4o", input_cost_per_token=1e-3) + + for field in ( + "input_cost_per_token_priority", + "output_cost_per_token_priority", + "cache_read_input_token_cost_priority", + ): + assert entry[field] == builtin("gpt-4o")[field] + assert entry.get("input_cost_per_token_batches") is None # batch rates are not a service tier + + +def test_a_user_set_service_tier_rate_wins_over_the_inherited_one(make_core: MakeCore) -> None: + entry: Final = registered(make_core, "openai/gpt-4o", input_cost_per_token=1e-3, input_cost_per_token_priority=9e-3) + + assert entry["input_cost_per_token_priority"] == 9e-3 + assert entry["output_cost_per_token_priority"] == builtin("gpt-4o")["output_cost_per_token_priority"] + + +def test_output_only_custom_pricing_inherits_service_tiers_but_not_cache_pricing(make_core: MakeCore) -> None: + entry: Final = registered(make_core, "openai/gpt-4o", output_cost_per_token=1e-3) + + assert entry["input_cost_per_token_priority"] == builtin("gpt-4o")["input_cost_per_token_priority"] + assert entry.get("cache_read_input_token_cost") is None + + +def test_a_backend_missing_from_the_cost_map_contributes_nothing(make_core: MakeCore) -> None: + entry: Final = registered(make_core, "openai/l8400-not-a-real-model", input_cost_per_token=1e-3) + + assert entry["input_cost_per_token"] == 1e-3 + assert entry.get("cache_read_input_token_cost") is None + assert entry.get("input_cost_per_token_priority") is None + + +# --- which backend model the rates come from ------------------------------------------------------ + + +@pytest.mark.parametrize( + "params", + [{"model_info": {"base_model": "gpt-5"}}, {"base_model": "gpt-5"}], + ids=["model-info-base-model", "litellm-params-base-model"], +) +def test_the_base_model_names_the_backend_the_rates_are_inherited_from( + make_core: MakeCore, params: dict[str, object] +) -> None: + entry: Final = registered(make_core, "openai/my-fine-tune", input_cost_per_token=1e-3, **params) + + assert entry["cache_read_input_token_cost"] == builtin("gpt-5")["cache_read_input_token_cost"] + assert entry["input_cost_per_token_flex"] == builtin("gpt-5")["input_cost_per_token_flex"] + + +# --- explicit cache-read rate ----------------------------------------------------------------------- + + +def test_explicit_cache_read_rate_is_inherited_only_without_a_user_cache_read_rate(make_core: MakeCore) -> None: + backend: Final = builtin("dashscope/qwen3.7-max") + inherited: Final = registered(make_core, "dashscope/qwen3.7-max", input_cost_per_token=1e-3) + user_set: Final = registered( + make_core, "dashscope/qwen3.7-max", input_cost_per_token=1e-3, cache_read_input_token_cost=1e-8 + ) + + assert inherited["explicit_cache_read_input_token_cost"] == backend["explicit_cache_read_input_token_cost"] + assert inherited["cache_read_input_token_cost"] == backend["cache_read_input_token_cost"] + assert user_set.get("explicit_cache_read_input_token_cost") is None + assert user_set["cache_read_input_token_cost"] == 1e-8 + + +# --- off-peak only ---------------------------------------------------------------------------------- + + +def test_an_off_peak_only_deployment_copies_the_whole_backend_entry(make_core: MakeCore) -> None: + backend: Final = builtin("deepseek-v4-flash") + entry: Final = registered(make_core, "deepseek/deepseek-v4-flash", model_info={"off_peak_pricing": OFF_PEAK}) + + assert entry["off_peak_pricing"] == OFF_PEAK + for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", "max_input_tokens"): + assert entry[field] == backend[field] + assert entry["source"] == backend["source"] + + +def test_an_off_peak_deployment_keeps_its_own_fields(make_core: MakeCore) -> None: + entry: Final = registered( + make_core, "deepseek/deepseek-v4-flash", model_info={"off_peak_pricing": OFF_PEAK, "max_input_tokens": 7} + ) + + assert entry["max_input_tokens"] == 7 + assert entry["input_cost_per_token"] == builtin("deepseek-v4-flash")["input_cost_per_token"] + + +def test_an_off_peak_deployment_with_a_base_rate_copies_nothing_more(make_core: MakeCore) -> None: + entry: Final = registered( + make_core, + "deepseek/deepseek-v4-flash", + model_info={"off_peak_pricing": OFF_PEAK}, + input_cost_per_token=5e-3, + ) + + assert entry["input_cost_per_token"] == 5e-3 + assert entry.get("output_cost_per_token") is None + assert entry.get("max_input_tokens") is None + + +def test_off_peak_pricing_given_in_litellm_params_is_not_registered(make_core: MakeCore) -> None: + """Oddity: only ``model_info.off_peak_pricing`` counts; the same key in litellm_params is dropped.""" + entry: Final = registered(make_core, "deepseek/deepseek-v4-flash", off_peak_pricing=OFF_PEAK) + + assert entry.get("off_peak_pricing") is None + assert entry.get("input_cost_per_token") is None + + +# --- tiered pricing output rate --------------------------------------------------------------------- + + +def test_a_tier_table_without_output_rates_inherits_the_backend_flat_output_rate(make_core: MakeCore) -> None: + entry: Final = registered( + make_core, "openai/gpt-4o", tiered_pricing=[{"input_cost_per_token": 1e-9, "range": [0, 1000]}] + ) + + assert entry["output_cost_per_token"] == builtin("gpt-4o")["output_cost_per_token"] + + +@pytest.mark.parametrize( + ("model", "params"), + [ + ("dashscope/qwen-flash", {"tiered_pricing": [{"input_cost_per_token": 1e-9, "range": [0, 1000]}]}), + ( + "openai/gpt-4o", + {"tiered_pricing": [{"input_cost_per_token": 1e-9, "output_cost_per_token": 2e-9, "range": [0, 1000]}]}, + ), + ( + "openai/gpt-4o", + {"tiered_pricing": [{"input_cost_per_token": 1e-9, "range": [0, 1000]}], "output_cost_per_token": 3e-9}, + ), + ], + ids=["tiered-only-backend", "every-tier-has-an-output-rate", "user-output-rate"], +) +def test_the_tiered_output_rate_is_not_inherited(make_core: MakeCore, model: str, params: dict[str, object]) -> None: + entry: Final = registered(make_core, model, **params) + + assert entry.get("output_cost_per_token") == params.get("output_cost_per_token") + + +# --- _create_deployment checks ------------------------------------------------------------------------ + + +def test_an_invalid_access_windows_config_raises_naming_the_model(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match=r"^model 'gpt': invalid model_info\.access_windows: "): + make_core(model_list=[deployment("gpt", "a", model_info={"access_windows": "bad"})]) + + +@pytest.mark.parametrize( + "model_info", + [ + {"access_windows": "bad"}, + {"ptu_shares": {"team-a": 1}}, + ], + ids=["access-windows", "ptu-shares"], +) +def test_ignore_invalid_deployments_drops_a_deployment_with_an_invalid_config( + make_core: MakeCore, model_info: dict[str, object] +) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "a", model_info=model_info), deployment("gpt", "b")], + ignore_invalid_deployments=True, + ) + + assert core.registry().names(kind="ids") == ["b"] + + +def test_ptu_shares_without_a_count_and_rate_raise(make_core: MakeCore) -> None: + with pytest.raises( + ValueError, + match=r"^PTU configuration on model 'gpt' is invalid: ptu_count and cost_per_ptu_per_hour are required " + r"when ptu_shares is set", + ): + make_core(model_list=[deployment("gpt", "a", model_info={"ptu_shares": {"team-a": 1}})]) + + +def test_an_inverted_ptu_window_with_shares_raises(make_core: MakeCore) -> None: + with pytest.raises( + ValueError, + match=r"^PTU configuration on model 'gpt' is invalid: ptu_effective_to must be after ptu_effective_from", + ): + make_core( + model_list=[ + deployment( + "gpt", + "a", + model_info={ + "ptu_effective_from": "2026-01-02T00:00:00Z", + "ptu_effective_to": "2026-01-01T00:00:00Z", + "ptu_shares": {"team-a": 1}, + "ptu_count": 1, + "cost_per_ptu_per_hour": 1, + }, + ) + ] + ) + + +def test_a_db_sourced_deployment_skips_the_ptu_config_check(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "a", model_info={"ptu_shares": {"team-a": 1}, "db_model": True})] + ) + + assert core.registry().names(kind="ids") == ["a"] diff --git a/tests/unit/test_router/core/test_gaps_l8400_retries.py b/tests/unit/test_router/core/test_gaps_l8400_retries.py new file mode 100644 index 00000000000..34373f6e03e --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_l8400_retries.py @@ -0,0 +1,138 @@ +""" +Gaps in the retry loop and the per-attempt bookkeeping of ``PythonCore`` (router_python_core.py +lines 8400-11100): the ``UnsupportedEndpointError`` skip-and-stop rule, the retry breadcrumb cap and +``base_model``, and the request the async failure callback counts toward group usage. +""" + +from typing import Final + +import pytest + +import litellm +from tests.unit.test_router.core.retries_health_helpers import ( + BackgroundSuccessLoggingHost, + MakeCore, + MetadataRecorder, + attempt, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit, server_error + + +def unsupported_endpoint() -> litellm.UnsupportedEndpointError: + return litellm.UnsupportedEndpointError(message="endpoint not supported", llm_provider="openai", model="gpt-4o") + + +# --- UnsupportedEndpointError: retried on the other deployments, stopped once every one refused --------- + + +@pytest.mark.asyncio +async def test_unsupported_endpoint_is_retried_once_per_deployment_then_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + """A 400 is normally never retried; ``UnsupportedEndpointError`` is, but each deployment that refused + is skipped and the loop stops as soon as every deployment of the group refused (num_retries=3 unused).""" + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=3) + host.answer("*", lambda **_: unsupported_endpoint()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.UnsupportedEndpointError) + called: Final = host.deployments_called("acompletion") + assert sorted(called) == ["a", "b"] + assert host.sleeps == [0] + + +@pytest.mark.asyncio +async def test_unsupported_endpoint_on_a_lone_deployment_is_raised_without_a_retry( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + host.answer("*", lambda **_: unsupported_endpoint()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.UnsupportedEndpointError) + assert host.deployments_called("acompletion") == ["a"] + assert host.sleeps == [] + + +# --- retry breadcrumbs ---------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_previous_models_keeps_only_the_four_most_recent_failures( + make_core: MakeCore, host: ScriptedHost +) -> None: + recorder: Final = MetadataRecorder() + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=6) + host.answer("*", *([recorder.then(server_error)] * 6), recorder.then(None)) + + response: Final = await attempt(core) + + assert not isinstance(response, Exception) + trails: Final = [snapshot["previous_models"] for snapshot in recorder.snapshots] + assert trails[0] is None + assert [len(trail) for trail in trails[1:]] == [1, 2, 3, 4, 4, 4] # pyright: ignore[reportArgumentType] + assert [entry["attempted_retries"] for entry in trails[-1]] == [2, 3, 4, 5] # pyright: ignore[reportGeneralTypeIssues, reportOptionalIterable] + assert [snapshot["request_retry_count"] for snapshot in recorder.snapshots] == [None, 1, 2, 3, 4, 5, 6] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("params", "expected"), + [ + ({"base_model": "azure/gpt-4o"}, "azure/gpt-4o"), + ({"model_info": {"base_model": "azure/gpt-4o-mini"}}, "azure/gpt-4o-mini"), + ({}, None), + ], + ids=["litellm-params-base-model", "model-info-base-model", "none"], +) +async def test_a_retry_breadcrumb_names_the_deployments_base_model( + make_core: MakeCore, host: ScriptedHost, params: dict[str, object], expected: str | None +) -> None: + recorder: Final = MetadataRecorder() + core: Final = make_core(model_list=[deployment("g", "a", **params)], num_retries=1) + host.answer("*", recorder.then(server_error), recorder.then(None)) + + await attempt(core) + + (entry,) = recorder.snapshots[1]["previous_models"] # pyright: ignore[reportGeneralTypeIssues] + assert entry["base_model"] == expected # pyright: ignore[reportIndexIssue] + + +# --- the async failure callback counts the failed request --------------------------------------- + + +@pytest.mark.asyncio +async def test_a_failed_call_counts_a_request_but_no_tokens_in_group_usage( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", rpm=5)], num_retries=0) + host.answer("*", rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert await core.health().usage("g") == (None, 1) + + +class TestUsageWhenTheHostLogsSuccessAfterTheResponse: + """litellm runs the async success callbacks after the endpoint returned (``ScriptedHost`` runs them + before, which lets the success callback and the router's own count both add the same response).""" + + @pytest.fixture + def host(self) -> BackgroundSuccessLoggingHost: + return BackgroundSuccessLoggingHost() + + @pytest.mark.asyncio + @pytest.mark.parametrize(("failures", "requests"), [(0, 1), (1, 2), (2, 3)]) + async def test_every_failed_attempt_and_the_final_success_count_as_requests( + self, make_core: MakeCore, host: BackgroundSuccessLoggingHost, failures: int, requests: int + ) -> None: + core: Final = make_core(model_list=[deployment("g", "a", rpm=50)], num_retries=2) + host.answer("*", *([server_error()] * failures), None) + + await attempt(core) + await host.drain() + + assert await core.health().usage("g") == (10, requests) diff --git a/tests/unit/test_router/core/test_gaps_pick_input_and_compaction.py b/tests/unit/test_router/core/test_gaps_pick_input_and_compaction.py new file mode 100644 index 00000000000..ded4b50c45f --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_pick_input_and_compaction.py @@ -0,0 +1,253 @@ +""" +Parity gaps differential fuzzing found (regress5 R2, U4, N10): the ``messages`` and ``input`` each +endpoint hands its pick, which decide whether the pre-call checks run and what they count; +``print_deployment`` on a ``Deployment``; and the context compaction checks of an auto-router. +""" + +import io +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_api import RoutingContext +from litellm.router_core import RouterCore +from litellm.router_strategy.complexity_router.context_compaction import CompactionState +from litellm.types.router import Deployment, LiteLLM_Params, RouterRateLimitError +from tests.unit.test_router.core.construction_precall_helpers import windowed +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.scripted_host import deployment + +MakeCore = Callable[..., RouterCore] + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +LONG_TEXT: Final = "word " * 200 +REGIONAL: Final = [deployment("g", "eu", region_name="eu"), deployment("g", "ap", region_name="ap")] +EU_ONLY: Final = [deployment("g", "eu", region_name="eu")] +BATCH: Final = {"input_file_id": "f", "endpoint": "/v1/chat/completions", "completion_window": "24h"} + + +@pytest.fixture +def host() -> EndpointHost: + return EndpointHost() + + +# --- R2: the pre-call checks of a prompt-only request --------------------------------------- + + +@pytest.mark.asyncio +async def test_async_text_completion_is_refused_outside_the_allowed_region( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=EU_ONLY, enable_pre_call_checks=True, num_retries=0) + with pytest.raises(RouterRateLimitError): + await core.call("atext_completion", {"model": "g", "prompt": "hi", "allowed_model_region": "ap"}) + assert host.calls_to("atext_completion") == [] + + +def test_sync_text_completion_is_refused_outside_the_allowed_region(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=EU_ONLY, enable_pre_call_checks=True, num_retries=0) + with pytest.raises(RouterRateLimitError): + core.text_completion(model="g", prompt="hi", allowed_model_region="ap") # pyright: ignore[reportAttributeAccessIssue] + assert host.calls_to("text_completion") == [] + + +@pytest.mark.asyncio +async def test_async_text_completion_keeps_the_deployment_in_the_allowed_region( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=REGIONAL, enable_pre_call_checks=True, num_retries=0) + await core.call("atext_completion", {"model": "g", "prompt": "hi", "allowed_model_region": "ap"}) + assert host.deployments_called("atext_completion") == ["ap"] + + +def test_sync_text_completion_keeps_the_deployment_in_the_allowed_region( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=REGIONAL, enable_pre_call_checks=True, num_retries=0) + core.text_completion(model="g", prompt="hi", allowed_model_region="ap") # pyright: ignore[reportAttributeAccessIssue] + # The sync text path sends no ``model_info``: the params name the deployment. + assert [call.kwargs["region_name"] for call in host.calls_to("text_completion")] == ["ap"] + + +@pytest.mark.asyncio +async def test_async_text_completion_counts_the_prompt_against_the_context_window( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True, num_retries=0) + with pytest.raises(litellm.ContextWindowExceededError, match="Context Window exceeded for given call"): + await core.call("atext_completion", {"model": "g", "prompt": LONG_TEXT}) + assert host.calls_to("atext_completion") == [] + + +def test_sync_text_completion_counts_the_prompt_against_the_context_window( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 5)], enable_pre_call_checks=True, num_retries=0) + with pytest.raises(litellm.ContextWindowExceededError, match="Context Window exceeded for given call"): + core.text_completion(model="g", prompt=LONG_TEXT) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls_to("text_completion") == [] + + +@pytest.mark.parametrize( + ("endpoint", "request_fields"), + [ + ("aimage_generation", {"prompt": "a cat"}), + ("image_generation", {"prompt": "a cat"}), + ("atranscription", {"file": io.BytesIO(b"x")}), + ("aspeech", {"input": "hi", "voice": "alloy"}), + ("aadapter_completion", {"adapter_id": "a", "messages": MESSAGES}), + ("acreate_batch", BATCH), + ], +) +@pytest.mark.asyncio +async def test_endpoints_picking_with_a_stand_in_message_run_the_region_check( + make_core: MakeCore, host: EndpointHost, endpoint: str, request_fields: Mapping[str, object] +) -> None: + core: Final = make_core(model_list=EU_ONLY, enable_pre_call_checks=True, num_retries=0) + with pytest.raises(RouterRateLimitError): + await core.call(endpoint, {"model": "g", "allowed_model_region": "ap", **request_fields}) + assert host.calls == [] + + +@pytest.mark.parametrize( + ("endpoint", "request_fields"), + [("arerank", {"query": "q", "documents": ["a"]}), ("amoderation", {"input": "text"})], +) +@pytest.mark.asyncio +async def test_endpoints_picking_with_no_input_skip_the_region_check( + make_core: MakeCore, host: EndpointHost, endpoint: str, request_fields: Mapping[str, object] +) -> None: + core: Final = make_core(model_list=EU_ONLY, enable_pre_call_checks=True, num_retries=0) + await core.call(endpoint, {"model": "g", "allowed_model_region": "ap", **request_fields}) + assert host.deployments_called(endpoint) == ["eu"] + + +@pytest.mark.asyncio +async def test_speech_counts_its_stand_in_message_and_not_its_input(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=[windowed("g", "small", 50)], enable_pre_call_checks=True, num_retries=0) + await core.call("aspeech", {"model": "g", "input": LONG_TEXT, "voice": "alloy"}) + assert host.deployments_called("aspeech") == ["small"] + + +# --- U4: print_deployment -------------------------------------------------------------------- + + +def test_print_deployment_masks_the_key_of_a_deployment_and_returns_a_deployment(make_core: MakeCore) -> None: + given: Final = Deployment(model_name="g", litellm_params=LiteLLM_Params(model="openai/x", api_key="sk-obj-123456")) + core: Final = make_core(model_list=[deployment("g", "a")]) + printed: Final = core.print_deployment(given) # pyright: ignore[reportAttributeAccessIssue] + assert isinstance(printed, Deployment) + assert printed.litellm_params.api_key == "sk**********" + assert given.litellm_params.api_key == "sk-obj-123456" + + +def test_print_deployment_refuses_a_list_of_pairs(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(TypeError): + core.print_deployment([["litellm_params", {"api_key": "sk-pairs-1234"}]]) # pyright: ignore[reportAttributeAccessIssue] + + +# --- N10: context compaction ----------------------------------------------------------------- + + +def _complexity_router_over_a_small_window() -> list[dict[str, object]]: # mutable-ok: a model_list + return [ + windowed("mini", "mini", 60), + deployment("o47", "o47", model="anthropic/claude-opus-4-7"), + { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": "mini", "REASONING": "o47"}, + "enable_context_window_escalation": True, + }, + "complexity_router_default_model": "mini", + }, + "model_info": {"id": "smart"}, + }, + ] + + +@pytest.mark.asyncio +async def test_an_auto_router_refuses_a_prompt_compaction_cannot_fit(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=_complexity_router_over_a_small_window(), num_retries=0) + messages: Final = [ + {"role": "user", "content": "context line about the project history and many details " * 30}, + {"role": "assistant", "content": "earlier answer"}, + {"role": "user", "content": "and next?"}, + ] + with pytest.raises(litellm.ContextWindowExceededError, match="Context compaction"): + await core.call("acompletion", {"model": "smart", "messages": messages}) + assert host.calls_to("acompletion") == [] + + +@pytest.mark.asyncio +async def test_the_async_chat_attempt_carries_the_compaction_state(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0) + await core.call("acompletion", {"model": "g", "messages": MESSAGES}) + state: Final = host.calls_to("acompletion")[0].kwargs["_context_compaction_state"] + assert isinstance(state, CompactionState) + assert state.surface == "chat" + assert state.config is None + + +@pytest.mark.parametrize( + ("endpoint", "request_fields"), + [("completion", {"messages": MESSAGES}), ("atext_completion", {"prompt": "hi"})], +) +@pytest.mark.asyncio +async def test_other_attempts_carry_no_compaction_state( + make_core: MakeCore, host: EndpointHost, endpoint: str, request_fields: Mapping[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0) + await core.call(endpoint, {"model": "g", **request_fields}) + assert "_context_compaction_state" not in host.calls_to(endpoint)[0].kwargs + + +def test_the_routing_context_counts_messages_and_tools(make_core: MakeCore) -> None: + ctx: Final = RoutingContext(make_core(model_list=[deployment("g", "a")])) # pyright: ignore[reportArgumentType] + tool: Final = {"type": "function", "function": {"name": "lookup", "description": "look it up", "parameters": {}}} + plain: Final = ctx.count_pre_call_check_tokens(messages=MESSAGES, input=None) + with_tools: Final = ctx.count_pre_call_check_tokens(messages=MESSAGES, input=None, request_kwargs={"tools": [tool]}) + assert 0 < plain < with_tools + + +def test_the_routing_context_counts_responses_input(make_core: MakeCore) -> None: + ctx: Final = RoutingContext(make_core(model_list=[deployment("g", "a")])) # pyright: ignore[reportArgumentType] + assert ctx.count_pre_call_check_tokens(messages=None, input="hello there") > 0 + + +def test_the_routing_context_needs_messages_or_input_to_count(make_core: MakeCore) -> None: + ctx: Final = RoutingContext(make_core(model_list=[deployment("g", "a")])) # pyright: ignore[reportArgumentType] + with pytest.raises(ValueError, match="Either messages or input must be provided to count tokens"): + ctx.count_pre_call_check_tokens(messages=None, input=None) + + +def test_the_routing_context_pre_call_checks_refuse_a_count_over_the_window(make_core: MakeCore) -> None: + small: Final = windowed("g", "small", 5) + ctx: Final = RoutingContext(make_core(model_list=[small])) # pyright: ignore[reportArgumentType] + with pytest.raises(litellm.ContextWindowExceededError, match="Context Window exceeded for given call"): + ctx.pre_call_checks( + model="g", + healthy_deployments=[small], + messages=MESSAGES, # pyright: ignore[reportArgumentType] + request_kwargs={}, + input_token_count=100, + skip_inline_token_count=True, + ) + + +def test_the_routing_context_pre_call_checks_drop_a_deployment_outside_the_region(make_core: MakeCore) -> None: + ctx: Final = RoutingContext(make_core(model_list=REGIONAL)) # pyright: ignore[reportArgumentType] + kept: Final = ctx.pre_call_checks( + model="g", + healthy_deployments=list(REGIONAL), + messages=MESSAGES, # pyright: ignore[reportArgumentType] + request_kwargs={"allowed_model_region": "ap"}, + input_token_count=1, + skip_inline_token_count=True, + ) + assert [record["model_info"]["id"] for record in kept] == ["ap"] # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType] diff --git a/tests/unit/test_router/core/test_gaps_request_overrides.py b/tests/unit/test_router/core/test_gaps_request_overrides.py new file mode 100644 index 00000000000..3793dd5f898 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_request_overrides.py @@ -0,0 +1,227 @@ +""" +Per-request routing overrides a router core reads (or does not read) on each call: + +- ``default_fallbacks`` and ``retry_policy`` in a request are no routing input: the router neither + falls back on the request's ``default_fallbacks`` nor retries on its ``retry_policy``. Both reach + the deployment call as given (litellm's own ``completion`` reads ``retry_policy`` there), on every + attempt, sync and async, streamed or not. +- ``model_group_retry_policy`` that is not a mapping of policies, in a request or as the router's + setting: the retry loop looks the model group up in it (``group in value``, then ``value[group]``), + so a value that does not hold the group leaves the router's ``retry_policy`` in charge, a ``None`` + entry is no policy at all, and an explicit request ``None`` replaces the router's per-group policies. +""" + +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import RetryPolicy +from tests.unit.test_router.core.retries_health_helpers import attempt, attempt_sync +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts, collect_texts_sync + +MakeCore = Callable[..., RouterCore] +MESSAGES: Final = [{"role": "user", "content": "hi"}] +GROUPS: Final = [deployment("a", "a"), deployment("b", "b"), deployment("c", "c")] +OVERRIDES: Final[Mapping[str, object]] = {"default_fallbacks": ["c"], "retry_policy": {"RateLimitErrorRetries": 3}} + + +def sent_overrides(host: ScriptedHost) -> list[tuple[str | None, object, object]]: # mutable-ok: a fresh list + return [ + (call.deployment_id, call.kwargs.get("default_fallbacks"), call.kwargs.get("retry_policy")) + for call in host.calls_to() + ] + + +# --- default_fallbacks and retry_policy: passed to the deployment, never routed on ---------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "request_body"), + [ + ("acompletion", {"messages": MESSAGES}), + ("atext_completion", {"prompt": "p"}), + ("aembedding", {"input": ["x"]}), + ("aresponses", {"input": "hi"}), + ], +) +async def test_request_default_fallbacks_and_retry_policy_reach_the_deployment_as_given( + make_core: MakeCore, host: ScriptedHost, endpoint: str, request_body: Mapping[str, object] +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=0) + + await core.call(endpoint, {"model": "a", **request_body, **OVERRIDES}) + + assert sent_overrides(host) == [("a", ["c"], {"RateLimitErrorRetries": 3})] + + +@pytest.mark.asyncio +async def test_a_failed_call_neither_falls_back_on_nor_retries_by_the_request_overrides( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=0) + host.answer("openai/a", rate_limit()) + + error: Final = await attempt(core, "a", **OVERRIDES) + + assert isinstance(error, litellm.RateLimitError) + assert sent_overrides(host) == [("a", ["c"], {"RateLimitErrorRetries": 3})] + + +def test_a_failed_sync_call_neither_falls_back_on_nor_retries_by_the_request_overrides( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=0) + host.answer("openai/a", rate_limit()) + + error: Final = attempt_sync(core, "a", **OVERRIDES) + + assert isinstance(error, litellm.RateLimitError) + assert sent_overrides(host) == [("a", ["c"], {"RateLimitErrorRetries": 3})] + + +@pytest.mark.asyncio +async def test_the_router_retries_and_fallbacks_carry_the_request_overrides_on_every_attempt( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=1, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit(), rate_limit()) + + response: Final = await attempt(core, "a", **OVERRIDES) + + assert not isinstance(response, Exception) + assert sent_overrides(host) == [("a", ["c"], {"RateLimitErrorRetries": 3})] * 2 + [ + ("b", ["c"], {"RateLimitErrorRetries": 3}) + ] + + +@pytest.mark.asyncio +async def test_a_stream_that_fails_before_content_falls_back_with_the_request_overrides( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=0, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", chat_stream("x", error=rate_limit(), fail_after=0)) + host.answer("openai/b", chat_stream("B")) + + stream: Final = await core.call("acompletion", {"model": "a", "messages": MESSAGES, "stream": True, **OVERRIDES}) + + assert await collect_texts(stream) == "B" + assert sent_overrides(host) == [ + ("a", ["c"], {"RateLimitErrorRetries": 3}), + ("b", ["c"], {"RateLimitErrorRetries": 3}), + ] + + +def test_a_sync_stream_carries_the_request_overrides(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=0) + host.answer("openai/a", chat_stream("A", sync=True)) + + completion: Final[Callable[..., object]] = getattr(core, "completion") # pyright: ignore[reportAny] # an endpoint method + stream: Final = completion(model="a", messages=MESSAGES, stream=True, **OVERRIDES) + + assert collect_texts_sync(stream) == "A" + assert sent_overrides(host) == [("a", ["c"], {"RateLimitErrorRetries": 3})] + + +# --- model_group_retry_policy that is not a mapping of policies -------------------------------------- + + +def calls_after_a_rate_limit( + core: RouterCore, host: ScriptedHost, **request: object +) -> int: # kwargs-ok: request kwargs + host.answer("openai/a", rate_limit()) + error: Final = attempt_sync(core, "a", **request) + assert isinstance(error, litellm.RateLimitError) + return len(host.calls_to()) + + +@pytest.mark.parametrize( + "policies", + [None, {"a": None}, "xyz", [], ("b",)], + ids=["none", "none-entry", "string-without-the-group", "empty-list", "tuple-without-the-group"], +) +def test_a_request_policy_that_holds_no_policy_for_the_group_keeps_the_plain_retries( + make_core: MakeCore, host: ScriptedHost, policies: object +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=2) + + assert calls_after_a_rate_limit(core, host, model_group_retry_policy=policies) == 3 + + +@pytest.mark.parametrize( + ("policies", "calls"), + [ + ("xyz", 2), # the group is not in it: the router's retry_policy applies + ({"a": None}, 3), # a None entry is no policy: the plain retries apply + (None, 2), # an explicit None replaces the router's per-group policies + ], + ids=["string-without-the-group", "none-entry", "none"], +) +def test_a_request_policy_without_the_group_leaves_the_router_retry_policy_in_charge( + make_core: MakeCore, host: ScriptedHost, policies: object, calls: int +) -> None: + core: Final = make_core( + model_list=GROUPS, + num_retries=2, + retry_policy=RetryPolicy(RateLimitErrorRetries=1), + model_group_retry_policy={"a": RetryPolicy(RateLimitErrorRetries=0)}, + ) + + assert calls_after_a_rate_limit(core, host, model_group_retry_policy=policies) == calls + + +@pytest.mark.parametrize("policies", [5, "abc", {"a": 3}], ids=["int", "string-holding-the-group", "int-entry"]) +def test_a_request_that_opts_out_of_retries_never_reads_its_policy( + make_core: MakeCore, host: ScriptedHost, policies: object +) -> None: + core: Final = make_core(model_list=GROUPS, num_retries=2) + + assert calls_after_a_rate_limit(core, host, model_group_retry_policy=policies, num_retries=0) == 1 + + +@pytest.mark.parametrize( + "policies", [{"a": None}, "xyz", []], ids=["none-entry", "string-without-the-group", "empty-list"] +) +@pytest.mark.parametrize("how", ["constructor", "apply"]) +def test_a_router_policy_that_holds_no_policy_for_the_group_keeps_the_plain_retries( + make_core: MakeCore, host: ScriptedHost, policies: object, how: str +) -> None: + if how == "constructor": + core = make_core(model_list=GROUPS, num_retries=2, model_group_retry_policy=policies) + else: + core = make_core(model_list=GROUPS, num_retries=2) + core.apply(ChangeSet(settings={"model_group_retry_policy": policies})) + + stored: Final[object] = getattr(core, "model_group_retry_policy") # pyright: ignore[reportAny] # the router attribute + assert stored == policies + assert calls_after_a_rate_limit(core, host) == 3 + + +@pytest.mark.parametrize("how", ["constructor", "apply"]) +def test_a_router_policy_that_is_no_mapping_is_not_read_when_the_request_opts_out( + make_core: MakeCore, host: ScriptedHost, how: str +) -> None: + if how == "constructor": + core = make_core(model_list=GROUPS, num_retries=2, model_group_retry_policy=5) + else: + core = make_core(model_list=GROUPS, num_retries=2) + core.apply(ChangeSet(settings={"model_group_retry_policy": 5})) + + assert calls_after_a_rate_limit(core, host, num_retries=0) == 1 + + +def test_get_num_retries_from_retry_policy_reads_a_router_policy_that_is_no_mapping(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=GROUPS, retry_policy=RetryPolicy(RateLimitErrorRetries=4), model_group_retry_policy="xyz" + ) + error: Final = rate_limit() + retries_for: Final[Callable[..., object]] = getattr(core, "get_num_retries_from_retry_policy") # pyright: ignore[reportAny] # the router member + + assert retries_for(error, model_group="a") == 4 + with pytest.raises(TypeError, match="string indices must be integers"): + retries_for(error, model_group="x") diff --git a/tests/unit/test_router/core/test_gaps_router_core_crate.py b/tests/unit/test_router/core/test_gaps_router_core_crate.py new file mode 100644 index 00000000000..7f00d26c944 --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_router_core_crate.py @@ -0,0 +1,283 @@ +""" +Gaps the Rust core closes inside router-core, pinned to ``PythonCore``'s behaviour (oddities included): +``default_deployment``, a ``model_group_retry_policy`` whose lookup raises, the deprecated registry +index writes, a raw ``routing_groups`` write against the stale member map, and ``cooldown_time=0`` +assigned after construction. +""" + +import warnings +from typing import Final + +import pytest +from pydantic import ValidationError + +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router import Router +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import RoutingGroup +from tests.unit.test_router.core.registry_helpers import team_deployment +from tests.unit.test_router.core.retries_health_helpers import MakeCore, attempt, cooled_down, headers_of +from tests.unit.test_router.core.scripted_host import ScriptedHost, completion_response, deployment, rate_limit + +DEFAULT: Final[dict[str, object]] = { # mutable-ok: a deployment dict, copied per test + "model_name": "dm", + "litellm_params": {"model": "openai/other-model"}, + "model_info": {"id": "dd"}, +} + + +def as_router(core: RouterCore) -> Router: + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it + return router + + +def write(router: Router, name: str, value: object) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + setattr(router, name, value) + + +async def picked(router: Router, model: str) -> str: + """The id ``select`` picks, or the error's class and message.""" + try: + chosen: Final[object] = await router.select(model, {}) + except Exception as error: + return f"{type(error).__name__}: {error}" + return str((as_str_mapping((as_str_mapping(chosen) or {}).get("model_info")) or {}).get("id")) + + +# --- default_deployment -------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_unknown_model_routes_to_the_default_deployment_without_retries_or_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the default deployment skips cooldown and the strategy, and the retry count reads no healthy + deployment, so ``num_retries`` is unused; its id is never cooled while it names no configured deployment.""" + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=1, allowed_fails=0) + _ = core.apply(ChangeSet(assign={"default_deployment": dict(DEFAULT)})) + host.answer("other-model", rate_limit(), completion_response()) + + failed: Final = await attempt(core, model="other-model") + assert isinstance(failed, litellm.RateLimitError) + assert cooled_down(core, ["dd", "a"]) == {} + served: Final = await attempt(core, model="other-model") + + assert headers_of(served)["x-litellm-model-group"] == "other-model" + assert [call.deployment_id for call in host.calls] == ["dd", "dd"] + assert host.calls[0].kwargs["model"] == "other-model" + assert host.sleeps == [] + assert core.settings().default_deployment == DEFAULT + + +@pytest.mark.asyncio +async def test_a_known_model_never_reaches_the_default_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + _ = core.apply(ChangeSet(assign={"default_deployment": dict(DEFAULT)})) + + _ = await attempt(core) + + assert host.deployments_called("acompletion") == ["a"] + + +# --- model_group_retry_policy lookups that raise ----------------------------------------------------- + + +RAISING_POLICIES: Final = [ + (3, TypeError, "argument of type 'int' is not iterable"), + ("g", TypeError, "string indices must be integers"), + (["g"], TypeError, "list indices must be integers or slices"), + ({"g": 3}, AttributeError, "'int' object has no attribute 'DefaultRetries'"), + ({"g": {"DefaultRetries": "x"}}, ValidationError, "validation error for RetryPolicy"), +] +"""Each policy, the error its lookup raises for group ``g``, and a fragment of that error's message.""" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("policy", "raised", "message"), RAISING_POLICIES) +async def test_a_raising_retry_policy_lookup_replaces_the_error_and_stops_the_hop( + make_core: MakeCore, host: ScriptedHost, policy: object, raised: type[Exception], message: str +) -> None: + """Oddity: the lookup runs on the hop's first failure; its error replaces the attempt's, and no retry + follows. The error is raised while the attempt's is handled, so it chains to it, except pydantic's + ``ValidationError``, which pydantic raises with no context.""" + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=2, model_group_retry_policy=policy + ) + host.answer("*", rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, raised) + assert message in str(error) + if raised is ValidationError: + assert error.__context__ is None + else: + assert isinstance(error.__context__, litellm.RateLimitError) + assert len(host.calls) == 1 + assert host.sleeps == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("policy", "raised", "message"), RAISING_POLICIES) +async def test_a_raising_retry_policy_lookup_takes_the_generic_fallbacks( + make_core: MakeCore, host: ScriptedHost, policy: object, raised: type[Exception], message: str +) -> None: + del raised, message + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c")], + num_retries=2, + model_group_retry_policy=policy, + fallbacks=[{"g": ["h"]}], + ) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + served: Final = await attempt(core) + + assert headers_of(served)["x-litellm-attempted-fallbacks"] == 1 + assert host.deployments_called("acompletion")[-1] == "c" + assert len(host.calls) == 2 + + +@pytest.mark.asyncio +async def test_a_policy_naming_another_group_leaves_this_group_retrying( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=2, model_group_retry_policy={"h": 3} + ) + host.answer("*", rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert len(host.calls) == 2 + + +# --- deprecated registry index writes ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_written_model_name_index_decides_which_names_resolve(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("claude", "c")])) + write(router, "model_name_to_deployment_indices", {"gpt": [0], "claude": []}) + + assert await picked(router, "gpt") == "a" + assert (await picked(router, "claude")).startswith("BadRequestError") + + +@pytest.mark.asyncio +async def test_a_model_name_index_entry_pointing_at_another_name_resolves_nothing(make_core: MakeCore) -> None: + """Each indexed deployment is still checked against the requested name.""" + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("claude", "c")])) + write(router, "model_name_to_deployment_indices", {"gpt": [1], "claude": [1]}) + + assert (await picked(router, "gpt")).startswith("BadRequestError") + assert await picked(router, "claude") == "c" + + +@pytest.mark.asyncio +async def test_a_written_model_names_set_decides_which_names_reach_the_default_deployment( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=[deployment("gpt", "a"), deployment("claude", "c")]) + _ = core.apply(ChangeSet(assign={"default_deployment": dict(DEFAULT)})) + router: Final = as_router(core) + + write(router, "model_names", set[str]()) + assert [await picked(router, name) for name in ("gpt", "claude", "other-model")] == ["dd", "dd", "dd"] + + write(router, "model_names", {"gpt", "claude", "other-model"}) + assert await picked(router, "gpt") == "a" + assert (await picked(router, "other-model")).startswith("BadRequestError") + + +@pytest.mark.asyncio +async def test_an_emptied_team_index_still_finds_team_deployments_by_scanning(make_core: MakeCore) -> None: + router: Final = as_router( + make_core(model_list=[deployment("gpt", "a"), team_deployment("int-t", "t1", "TA", "pub")]) + ) + write(router, "team_model_to_deployment_indices", {}) + write(router, "model_name_to_deployment_indices", {"gpt": [0]}) + + chosen: Final = await router.select("pub", {"metadata": {"user_api_key_team_id": "TA"}}) + assert chosen["model_info"]["id"] == "t1" + assert (await picked(router, "int-t")).startswith("BadRequestError") + + +# --- a raw routing_groups write: the member map stays as constructed ------------------------------------ + + +def _grouped(make_core: MakeCore) -> Router: + return as_router( + make_core( + model_list=[deployment("gpt", "a"), deployment("gpt", "b"), deployment("claude", "c")], + routing_groups=[{"group_name": "G", "models": ["gpt"], "routing_strategy": "least-busy"}], + ) + ) + + +@pytest.mark.asyncio +async def test_dropping_a_routing_group_raw_leaves_its_members_raising_a_key_error(make_core: MakeCore) -> None: + """Oddity: the member-to-group map is built once, so a member of a dropped group looks it up and fails.""" + router: Final = _grouped(make_core) + write(router, "routing_groups", {}) + + assert await picked(router, "gpt") == "KeyError: 'G'" + assert await picked(router, "claude") == "c" + + +@pytest.mark.asyncio +async def test_a_routing_group_added_raw_leaves_its_members_on_the_default_strategy(make_core: MakeCore) -> None: + router: Final = _grouped(make_core) + write( + router, + "routing_groups", + { + "G": RoutingGroup(group_name="G", models=["gpt"], routing_strategy="least-busy"), + "K": RoutingGroup(group_name="K", models=["claude"], routing_strategy="latency-based-routing"), + }, + ) + + assert await picked(router, "gpt") == "a" + assert await picked(router, "claude") == "c" + assert (await picked(router, "K")).startswith("RouterRateLimitError: No deployments available") + + +# --- cooldown_time assigned after construction ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_assigned_zero_cooldown_time_never_keeps_a_deployment_cooled( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: an assigned 0 is a zero cooldown ttl (only the constructor reads 0 as the default).""" + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, allowed_fails=0) + _ = core.apply(ChangeSet(assign={"cooldown_time": 0})) + host.answer("*", rate_limit()) + + for _ in range(3): + assert isinstance(await attempt(core), litellm.RateLimitError) + assert cooled_down(core, ["a", "b"]) == {} + + assert core.settings().cooldown_time == 0 + assert len(host.calls) == 3 + + +@pytest.mark.asyncio +async def test_an_assigned_none_cooldown_time_cools_for_the_default(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, allowed_fails=0) + _ = core.apply(ChangeSet(assign={"cooldown_time": None})) + host.answer("*", rate_limit()) + + _ = await attempt(core) + _ = await attempt(core) + + assert sorted(cooled_down(core, ["a", "b"])) == ["a", "b"] + assert {entry["cooldown_time"] for entry in cooled_down(core, ["a", "b"]).values()} == {5} + assert isinstance(await attempt(core), litellm.RouterRateLimitError) diff --git a/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py b/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py new file mode 100644 index 00000000000..255d731142d --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py @@ -0,0 +1,372 @@ +""" +Pick errors the Rust core serves inside router-core, pinned to ``PythonCore``'s behaviour (oddities +included): a pick's error retried by the retry loop, a ``default_deployment`` Python cannot serve, a +written index position past the model list, a stale routing group at a deeper fallback level or in +weighted failover, and a strategy change through ``update_settings`` after a raw ``routing_groups`` write. +""" + +import warnings +from typing import Final + +import pytest +from pydantic import ValidationError + +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router import Router +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import RoutingGroup +from tests.unit.test_router.core.registry_helpers import team_deployment +from tests.unit.test_router.core.retries_health_helpers import MakeCore, attempt, attempt_sync +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.test_fallbacks_characterization_outcomes import FallbackEvents + +TEAM: Final = {"metadata": {"user_api_key_team_id": "TA"}} + + +def as_router(core: RouterCore) -> Router: + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it + return router + + +def write(router: Router, name: str, value: object) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + setattr(router, name, value) + + +async def selected(router: Router, model: str, **request: object) -> object: # kwargs-ok: the request kwargs + """The deployment ``select`` picks, or the error it raised.""" + try: + return await router.select(model, dict(request)) + except Exception as error: + return error + + +def picked_id(picked: object) -> object: + return (as_str_mapping((as_str_mapping(picked) or {}).get("model_info")) or {}).get("id") + + +# --- a pick's error in the retry loop ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_retry_policy_retries_a_group_whose_deployments_all_cool_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The pick's ``RouterRateLimitError`` is a plain ``ValueError``: ``DefaultRetries`` applies to it.""" + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + num_retries=2, + allowed_fails=0, + model_group_retry_policy={"g": {"DefaultRetries": 1}}, + ) + host.answer("*", rate_limit()) + for _ in range(2): + _ = await attempt(core, num_retries=0) + host.calls.clear() + host.sleeps.clear() + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RouterRateLimitError) + assert isinstance(error.__context__, litellm.RouterRateLimitError) + assert host.calls == [] + assert len(host.sleeps) == 2 + assert all(sleep > 0 for sleep in host.sleeps) + + +@pytest.mark.asyncio +async def test_a_retry_policy_retries_an_unknown_model_and_stamps_the_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2, retry_policy={"DefaultRetries": 1}) + + error: Final = await attempt(core, model="zz") + + assert isinstance(error, litellm.BadRequestError) + assert (getattr(error, "num_retries", None), getattr(error, "max_retries", None)) == (1, 1) + assert isinstance(error.__context__, litellm.BadRequestError) + assert len(host.sleeps) == 2 + + +@pytest.mark.asyncio +async def test_invalid_router_weights_are_retried_while_a_deployment_is_healthy( + make_core: MakeCore, host: ScriptedHost +) -> None: + """No policy: ``should_retry_this_error`` retries a plain exception while a deployment is healthy.""" + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=2) + + error: Final = await attempt(core, _router_weights={"a": "x"}) + + assert isinstance(error, ValidationError) + assert isinstance(error.__context__, ValidationError) + assert host.calls == [] + assert host.sleeps == [0, 0, 0] + assert all(type(sleep) is int for sleep in host.sleeps) + + +@pytest.mark.asyncio +async def test_without_a_policy_an_unknown_model_is_raised_at_once(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + + error: Final = await attempt(core, model="zz") + + assert isinstance(error, litellm.BadRequestError) + assert error.__context__ is None + assert host.sleeps == [] + + +@pytest.mark.asyncio +async def test_the_last_retry_failure_chains_to_the_hops_first_failure(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + num_retries=2, + model_group_retry_policy={"g": {"DefaultRetries": 1}}, + ) + host.answer("*", rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert isinstance(error.__context__, litellm.RateLimitError) + assert error.__context__ is not error + assert host.sleeps[0] == 0 + assert type(host.sleeps[0]) is int + + +# --- a default_deployment Python cannot serve --------------------------------------------------------- + + +UNCOPYABLE_DEFAULTS: Final[list[tuple[object, type[Exception], str]]] = [ # mutable-ok: pytest parameters + ({}, KeyError, "'litellm_params'"), + ({"model_name": "dm"}, KeyError, "'litellm_params'"), + ({"model_name": "dm", "litellm_params": "s", "model_info": {"id": "dd"}}, AttributeError, "no attribute 'copy'"), + ("str", AttributeError, "'str' object has no attribute 'copy'"), + (5, AttributeError, "'int' object has no attribute 'copy'"), + (["a"], TypeError, "list indices must be integers or slices, not str"), +] +"""Each value, and the error its copy raises for any name.""" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("value", "raised", "message"), UNCOPYABLE_DEFAULTS) +async def test_a_default_deployment_python_cannot_copy_fails_each_pick_of_an_unknown_name( + make_core: MakeCore, host: ScriptedHost, value: object, raised: type[Exception], message: str +) -> None: + """Oddity: assigning it succeeds; every request for a name nothing else serves raises the copy's error.""" + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=1) + _ = core.apply(ChangeSet(assign={"default_deployment": value})) + + for error in ( + await attempt(core, model="zz"), + attempt_sync(core, model="zz"), + await selected(as_router(core), "zz"), + ): + assert isinstance(error, raised) + assert message in str(error) + assert host.deployments_called("acompletion") == [] + _ = await attempt(core) + assert host.deployments_called("acompletion") == ["a"] + + +FAILING_DEFAULTS: Final[list[tuple[object, str, object]]] = [ # mutable-ok: pytest parameters + ({"litellm_params": {"model": "x"}}, "'model_name'", None), + ({"model_name": "dm", "litellm_params": {"model": "x"}}, "'model_info'", None), + ({"model_name": "dm", "litellm_params": {"model": "x"}, "model_info": {}}, "'id'", {}), +] +"""Each value, the ``KeyError`` every call to it raises, and the ``model_info`` its pick returns.""" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("value", "missing", "model_info"), FAILING_DEFAULTS) +async def test_a_default_deployment_missing_a_field_is_picked_but_every_call_fails( + make_core: MakeCore, host: ScriptedHost, value: object, missing: str, model_info: object +) -> None: + """Oddity: the pick returns Python's copy; the call's first reads of it raise ``KeyError``, with no retry.""" + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=1) + _ = core.apply(ChangeSet(assign={"default_deployment": value})) + + called: Final = await attempt(core, model="zz") + called_sync: Final = attempt_sync(core, model="zz") + chosen: Final = as_str_mapping(await selected(as_router(core), "zz")) or {} + + assert isinstance(called, KeyError) + assert isinstance(called_sync, KeyError) + assert str(called) == missing + assert chosen.get("model_info") == model_info + assert (as_str_mapping(chosen.get("litellm_params")) or {}).get("model") == "zz" + assert host.calls == [] + assert host.sleeps == [] + + +# --- a written index position past the model list ----------------------------------------------------- + + +def _teams(make_core: MakeCore) -> tuple[RouterCore, Router]: + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("claude", "c"), team_deployment("int-t", "t1", "TA", "pub")], + num_retries=1, + fallbacks=[{"gpt": ["claude"]}, {"pub": ["claude"]}], + ) + return core, as_router(core) + + +@pytest.mark.asyncio +async def test_a_name_index_position_past_the_list_raises_index_error_for_that_name( + make_core: MakeCore, host: ScriptedHost +) -> None: + core, router = _teams(make_core) + write(router, "model_name_to_deployment_indices", {"gpt": [0, 7], "claude": [1], "int-t": [2]}) + + picked: Final = await selected(router, "gpt") + called: Final = await attempt(core, model="gpt") + + assert isinstance(picked, IndexError) + assert isinstance(called, IndexError) + assert str(called) == "list index out of range" + assert picked_id(await selected(router, "claude")) == "c" + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_a_team_index_position_past_the_list_raises_from_the_fallback_handling( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: the level's fallback handling lists the group again and raises a second ``IndexError`` + (chained to the pick's), before any fallback.""" + core, router = _teams(make_core) + write(router, "team_model_to_deployment_indices", {("TA", "pub"): [9]}) + + picked: Final = await selected(router, "pub", **TEAM) + called: Final = await attempt(core, model="pub", **TEAM) + + assert isinstance(picked, IndexError) + assert isinstance(called, IndexError) + assert isinstance(called.__context__, IndexError) + assert host.calls == [] + assert picked_id(await selected(router, "gpt")) == "a" + + +# --- a stale routing group deeper in the walk --------------------------------------------------------- + + +def _stale(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: extra router arguments + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("gpt", "b"), deployment("claude", "c"), deployment("x", "x1")], + routing_groups=[{"group_name": "G", "models": ["gpt"], "routing_strategy": "simple-shuffle"}], + num_retries=0, + fallbacks=[{"x": ["gpt"]}, {"gpt": ["claude"]}], + **arguments, + ) + write(as_router(core), "routing_groups", {}) + return core + + +@pytest.mark.asyncio +async def test_a_stale_routing_group_at_a_fallback_level_is_that_levels_error( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + events: Final = FallbackEvents() + monkeypatch.setattr(litellm, "callbacks", [events]) + core: Final = _stale(make_core) + host.answer("*", rate_limit()) + + error: Final = await attempt(core, model="x") + + assert isinstance(error, litellm.RateLimitError) + assert events.events == [("failure", "gpt", "claude", "KeyError"), ("failure", "x", "gpt", "RateLimitError")] + + +@pytest.mark.asyncio +async def test_weighted_failover_raises_the_stale_groups_key_error_before_any_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Oddity: ``_maybe_run_weighted_failover`` reads the routing context first, so the stale member map's + ``KeyError`` escapes the fallback handling (chained to the pick's) and the regular fallback never runs.""" + core: Final = _stale(make_core, enable_weighted_failover=True) + + error: Final = await attempt(core, model="gpt") + + assert isinstance(error, KeyError) + assert str(error) == "'G'" + assert isinstance(error.__context__, KeyError) + assert host.calls == [] + + +# --- update_settings(routing_strategy) after a raw routing_groups write ----------------------------------- + + +@pytest.mark.asyncio +async def test_a_strategy_change_re_inits_the_routing_groups_from_their_last_input(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("gpt", "b"), deployment("claude", "c")], + routing_groups=[{"group_name": "G", "models": ["gpt"], "routing_strategy": "least-busy"}], + ) + router: Final = as_router(core) + write( + router, "routing_groups", {"K": RoutingGroup(group_name="K", models=["claude"], routing_strategy="least-busy")} + ) + assert isinstance(await selected(router, "gpt"), KeyError) + + _ = core.apply(ChangeSet(settings={"routing_strategy": "usage-based-routing-v2"})) + + assert sorted(core.registry().routing_groups()) == ["G"] + assert picked_id(await selected(router, "G")) == "a" + assert picked_id(await selected(router, "gpt")) == "a" + assert isinstance(await selected(router, "K"), litellm.BadRequestError) + + +@pytest.mark.asyncio +async def test_an_unchanged_strategy_keeps_the_raw_routing_groups(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "a"), deployment("claude", "c")], + routing_groups=[{"group_name": "G", "models": ["gpt"], "routing_strategy": "least-busy"}], + ) + write( + as_router(core), + "routing_groups", + {"K": RoutingGroup(group_name="K", models=["claude"], routing_strategy="least-busy")}, + ) + + _ = core.apply(ChangeSet(settings={"routing_strategy": "simple-shuffle"})) + + assert sorted(core.registry().routing_groups()) == ["K"] + + +@pytest.mark.asyncio +async def test_a_negative_index_position_counts_from_the_end_of_the_list( + make_core: MakeCore, host: ScriptedHost +) -> None: + """As Python's ``model_list[index]`` reads it; one before the start raises ``IndexError``.""" + core, router = _teams(make_core) + write(router, "model_name_to_deployment_indices", {"gpt": [-3], "claude": [-2], "int-t": [-4]}) + + assert picked_id(await selected(router, "gpt")) == "a" + assert picked_id(await selected(router, "claude")) == "c" + before_the_start: Final = await attempt(core, model="int-t") + assert isinstance(before_the_start, IndexError) + assert str(before_the_start) == "list index out of range" + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_a_default_deployment_whose_id_is_not_a_string_is_picked_and_called_as_python_copies_it( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + value: Final = { + "model_name": "dm", + "litellm_params": {"model": "openai/dm", "api_key": "k"}, + "model_info": {"id": 5}, + } + _ = core.apply(ChangeSet(assign={"default_deployment": value})) + + served: Final = await attempt(core, model="zz") + chosen: Final = await selected(as_router(core), "zz") + + assert isinstance(served, litellm.ModelResponse) + assert host.calls[0].kwargs["model"] == "zz" + assert chosen == {"model_name": "dm", "litellm_params": {"model": "zz", "api_key": "k"}, "model_info": {"id": 5}} diff --git a/tests/unit/test_router/core/test_health_registry_views.py b/tests/unit/test_router/core/test_health_registry_views.py new file mode 100644 index 00000000000..0180f991ccb --- /dev/null +++ b/tests/unit/test_router/core/test_health_registry_views.py @@ -0,0 +1,174 @@ +""" +The ``health()`` priority helpers and the ``registry()`` members that read router state beyond the +deployment list: ``client(client_type="max_parallel_requests")``, ``plugins("routing")`` and +``zero_cost()``. Each test pins what the Python core does; every core must answer the same. +""" + +from typing import Final + +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit +from litellm.types.router import Deployment, LiteLLM_Params, RoutingContext +from tests.unit.test_router.core.scripted_host import deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore + +# --- health().split_priority ---------------------------------------------------------------------- + + +def test_split_priority_gives_the_request_priority_and_the_kwargs_without_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().split_priority("g", {"priority": 5, "user": "u"}) == (5, {"user": "u"}) + + +def test_split_priority_falls_back_to_the_default_priority_when_the_request_sets_none(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], default_priority=2) + assert core.health().split_priority("g", {"user": "u"}) == (2, {"user": "u"}) + + +def test_split_priority_without_a_default_leaves_the_request_unqueued(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().split_priority("g", {"priority": None}) == (None, {}) + + +def test_split_priority_refuses_a_priority_that_is_no_integer(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], default_priority=2) + with pytest.raises(litellm.BadRequestError, match="priority must be an integer, got str 'high'"): + core.health().split_priority("g", {"priority": "high"}) + + +def test_split_priority_drops_a_bad_priority_for_a_group_that_drops_params(make_core: MakeCore) -> None: + """A deployment of the group with ``drop_params=True`` turns the bad priority into the default.""" + core: Final = make_core(model_list=[deployment("g", "a", drop_params=True)], default_priority=2) + assert core.health().split_priority("g", {"priority": "high"}) == (2, {}) + + +def test_split_priority_reads_a_pinned_deployment_s_params_for_its_id(make_core: MakeCore) -> None: + """A deployment id as the model reads that deployment's ``drop_params``, not a group's.""" + core: Final = make_core( + model_list=[deployment("g", "a", drop_params=True), deployment("g", "b")], default_priority=1 + ) + assert core.health().split_priority("a", {"priority": "high"}) == (1, {}) + with pytest.raises(litellm.BadRequestError): + core.health().split_priority("b", {"priority": "high"}) + + +# --- health().priority_header_fields -------------------------------------------------------------- + + +def test_priority_header_fields_read_an_integer_header(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().priority_header_fields("g", {}, "3") == {"priority": 3} + + +@pytest.mark.parametrize("header", [None, ""]) +def test_priority_header_fields_are_empty_without_a_header(make_core: MakeCore, header: str | None) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().priority_header_fields("g", {}, header) == {} + + +def test_priority_header_fields_pass_a_bad_header_on_for_the_call_to_refuse(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().priority_header_fields("g", {}, "urgent") == {"priority": "urgent"} + + +def test_priority_header_fields_drop_a_bad_header_when_the_request_drops_params(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.health().priority_header_fields("g", {"drop_params": True}, "urgent") == {} + + +# --- registry().client(client_type="max_parallel_requests") --------------------------------------- + + +def test_the_max_parallel_requests_client_is_the_deployment_s_cached_limit(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", max_parallel_requests=2)]) + entry: Final = deployment("g", "a", max_parallel_requests=2) + limit: Final = core.registry().client(entry, client_type="max_parallel_requests") + assert isinstance(limit, MaxParallelRequestsLimit) + assert (limit.max_parallel_requests, limit.model_id, limit.model_group, limit.in_flight) == (2, "a", "g", 0) + assert core.registry().client(entry, client_type="max_parallel_requests") is limit + + +def test_the_max_parallel_requests_limit_comes_from_rpm_or_the_router_default(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", rpm=7), deployment("g", "b")], default_max_parallel_requests=4 + ) + by_rpm: Final = core.registry().client(deployment("g", "a", rpm=7), client_type="max_parallel_requests") + by_default: Final = core.registry().client(deployment("g", "b"), client_type="max_parallel_requests") + assert isinstance(by_rpm, MaxParallelRequestsLimit) + assert isinstance(by_default, MaxParallelRequestsLimit) + assert (by_rpm.max_parallel_requests, by_default.max_parallel_requests) == (7, 4) + + +def test_a_deployment_without_a_limit_has_no_max_parallel_requests_client(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + assert core.registry().client(deployment("g", "a"), client_type="max_parallel_requests") is None + + +def test_the_client_lookup_reads_the_deployment_id_first(make_core: MakeCore) -> None: + """``deployment["model_info"]["id"]``: a deployment without model_info raises, whatever the client type.""" + core: Final = make_core(model_list=[deployment("g", "a")]) + entry: Final = {"model_name": "g", "litellm_params": {"model": "openai/a"}} + for client_type in ("max_parallel_requests", "async", None): + with pytest.raises(KeyError, match="model_info"): + core.registry().client(entry, client_type=client_type) + + +@pytest.mark.asyncio +async def test_a_full_max_parallel_requests_client_turns_the_next_call_away(make_core: MakeCore) -> None: + """The limit ``client`` returns is the one calls take their slot from.""" + core: Final = make_core(model_list=[deployment("g", "a", max_parallel_requests=1)], num_retries=0) + entry: Final = deployment("g", "a", max_parallel_requests=1) + limit: Final = core.registry().client(entry, client_type="max_parallel_requests") + assert isinstance(limit, MaxParallelRequestsLimit) + with limit, pytest.raises(litellm.RateLimitError, match="already has max_parallel_requests=1"): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # every core serves acompletion + assert limit.in_flight == 0 + + +# --- registry().plugins("routing") and zero_cost() ------------------------------------------------ + + +class _Keep: + async def run(self, context: RoutingContext) -> RoutingContext: + return context + + +def test_the_routing_plugins_are_the_ones_the_router_was_built_with(make_core: MakeCore) -> None: + plugin: Final = _Keep() + core: Final = make_core(model_list=[deployment("g", "a")], plugins=[plugin]) + assert core.registry().plugins("routing") == [plugin] + + +def test_a_router_without_plugins_lists_no_routing_plugins(make_core: MakeCore) -> None: + assert make_core(model_list=[deployment("g", "a")]).registry().plugins("routing") == [] + + +def test_zero_cost_is_an_empty_lookup_the_auth_checks_fill(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + lookup: Final = core.registry().zero_cost() + assert lookup == {} + assert isinstance(lookup, dict) + lookup["g"] = True + assert core.registry().zero_cost() == {"g": True} + + +def test_a_deployment_change_empties_the_zero_cost_lookup(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + lookup: Final = core.registry().zero_cost() + assert isinstance(lookup, dict) + lookup["g"] = True + core.apply( + ChangeSet( + upsert=[ + Deployment( + model_name="g", + litellm_params=LiteLLM_Params(model="openai/b", api_key="sk-test"), + model_info={"id": "b"}, + ) + ] + ) + ) + assert core.registry().zero_cost() == {} diff --git a/tests/unit/test_router/core/test_host_settings_changes_characterization.py b/tests/unit/test_router/core/test_host_settings_changes_characterization.py new file mode 100644 index 00000000000..fa1446aa51a --- /dev/null +++ b/tests/unit/test_router/core/test_host_settings_changes_characterization.py @@ -0,0 +1,428 @@ +""" +Characterization of the settings only the host reads, changed after construction: ``search_tools``, +``guardrail_list``, ``default_max_parallel_requests``, ``provider_budget_config``, +``health_check_ignore_transient_errors`` and ``router_general_settings``. + +``apply(ChangeSet(assign=...))`` stores each as given, ``settings()`` reports it, and whatever reads +it afterwards sees it from then on. None of them is runtime-updatable, so ``ChangeSet(settings=...)`` +(``update_settings``) ignores them. +""" + +import asyncio +import time +from collections.abc import Awaitable, Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery +from litellm.types.router import Deployment, RouterGeneralSettings +from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints +from tests.unit.test_router.core.retries_health_helpers import cooled_down +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.selection_helpers import ids_of +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts + +MakeCore = Callable[..., RouterCore] +MESSAGES: Final = [{"role": "user", "content": "hi"}] +SEARCH_TOOLS: Final = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily", "api_key": "tk"}}] +GUARDRAILS: Final = [{"guardrail_name": "gr", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}] +PROVIDER_BUDGET: Final = {"openai": {"budget_limit": 1, "time_period": "1d"}} + +ASSIGNED: Final[list[tuple[str, object]]] = [ # mutable-ok: pytest parameters + ("search_tools", SEARCH_TOOLS), + ("guardrail_list", GUARDRAILS), + ("default_max_parallel_requests", 3), + ("provider_budget_config", PROVIDER_BUDGET), + ("health_check_ignore_transient_errors", True), + ("router_general_settings", RouterGeneralSettings(pass_through_all_models=True)), + ("provider_default_deployment_ids", ["d1"]), + ("enable_health_check_routing", True), + ("background_health_check_model_groups", ["g"]), +] + + +@pytest.fixture +def host(monkeypatch: pytest.MonkeyPatch) -> EndpointHost: + endpoint_host: Final = EndpointHost() + patch_direct_endpoints(monkeypatch, endpoint_host) + return endpoint_host + + +def _endpoint(core: RouterCore, name: str) -> Callable[..., Awaitable[object]]: + """An async endpoint method, which the core serves by name.""" + method: Final[Callable[..., Awaitable[object]]] = getattr(core, name) # pyright: ignore[reportAny] # endpoint methods are served by name + return method + + +async def _open_stream(core: RouterCore) -> object: + return await _endpoint(core, "acompletion")(model="a", messages=MESSAGES, stream=True) + + +# --- settings() --------------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("name", "value"), ASSIGNED) +def test_an_assigned_host_setting_is_reported_by_settings(make_core: MakeCore, name: str, value: object) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")]) + before: Final = core.settings() + _ = core.apply(ChangeSet(assign={name: value})) + assert getattr(core.settings(), name) == value + assert core.settings() is not before + + +@pytest.mark.parametrize(("name", "value"), [("search_tools", SEARCH_TOOLS), ("guardrail_list", GUARDRAILS)]) +def test_an_assigned_host_setting_is_copied_from_the_callers_list( + make_core: MakeCore, name: str, value: list[dict[str, object]] +) -> None: + given: Final = [dict(entry) for entry in value] + core: Final = make_core(model_list=[deployment("g", "d1")]) + _ = core.apply(ChangeSet(assign={name: given})) + given.clear() + assert getattr(core.settings(), name) == value # pyright: ignore[reportAny] # a settings field named by the parameter + + +@pytest.mark.parametrize(("name", "value"), ASSIGNED) +def test_update_settings_ignores_a_host_setting(make_core: MakeCore, name: str, value: object) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")]) + before: Final[object] = getattr(core.settings(), name) # pyright: ignore[reportAny] # a settings field named by the parameter + _ = core.apply(ChangeSet(settings={name: value})) + assert getattr(core.settings(), name) == before + + +def test_a_router_attribute_write_assigns_a_host_setting(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")]) + core.search_tools = SEARCH_TOOLS # pyright: ignore[reportAttributeAccessIssue] # the deprecated attribute write + assert core.settings().search_tools == SEARCH_TOOLS + + +# --- router_general_settings as a constructor argument ------------------------------------------ + + +def test_router_general_settings_passed_to_the_constructor_are_reported(make_core: MakeCore) -> None: + general: Final = RouterGeneralSettings(async_only_mode=True, pass_through_all_models=True) + core: Final = make_core(model_list=[deployment("g", "d1")], router_general_settings=general) + assert core.settings().router_general_settings == general + + +def test_router_general_settings_of_none_become_the_defaults(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], router_general_settings=None) + assert core.settings().router_general_settings == RouterGeneralSettings() + + +def test_routers_built_without_general_settings_share_one_default_object(make_core: MakeCore) -> None: + """The Python core's default argument is one ``RouterGeneralSettings()`` every router shares.""" + first: Final = make_core(model_list=[deployment("g", "d1")]) + second: Final = make_core(model_list=[deployment("g", "d1")]) + assert first.settings().router_general_settings is second.settings().router_general_settings + + +async def test_router_general_settings_leave_routing_unchanged(make_core: MakeCore, host: EndpointHost) -> None: + general: Final = RouterGeneralSettings(async_only_mode=True, pass_through_all_models=True) + core: Final = make_core(model_list=[deployment("g", "d1")], router_general_settings=general) + _ = await _endpoint(core, "acompletion")(model="g", messages=MESSAGES) + completion: Final[Callable[..., object]] = getattr(core, "completion") # pyright: ignore[reportAny] # see _endpoint + _ = completion(model="g", messages=MESSAGES) + assert host.deployments_called() == ["d1", "d1"] + + +# --- readers ------------------------------------------------------------------------------------ + + +async def test_asearch_reads_search_tools_assigned_after_construction(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + _ = core.apply(ChangeSet(assign={"search_tools": SEARCH_TOOLS})) + _ = await _endpoint(core, "asearch")(model="s", query="q") + call: Final = host.calls[0] + assert (call.op, call.kwargs["search_provider"], call.kwargs["api_key"]) == ("asearch", "tavily", "tk") + + +async def test_asearch_no_longer_finds_a_tool_dropped_by_a_changeset(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1")], search_tools=SEARCH_TOOLS, num_retries=0) + _ = core.apply(ChangeSet(assign={"search_tools": []})) + with pytest.raises(ValueError, match=r"Search tool 's' not found in router\.search_tools"): + _ = await _endpoint(core, "asearch")(model="s", query="q") + assert host.calls == [] + + +async def test_aguardrail_reads_a_guardrail_list_assigned_after_construction(make_core: MakeCore, host: ScriptedHost): + seen: Final[list[Mapping[str, object]]] = [] + + async def guard(**kwargs: object) -> str: # kwargs-ok: the guardrail's keyword payload + seen.append(kwargs) + return "verdict" + + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + _ = core.apply(ChangeSet(assign={"guardrail_list": GUARDRAILS})) + assert await _endpoint(core, "aguardrail")(guardrail_name="gr", original_function=guard) == "verdict" + assert seen[0]["selected_guardrail"] == GUARDRAILS[0] + + +async def test_a_default_max_parallel_requests_assigned_later_limits_the_next_requests( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x")) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=0) + _ = core.apply(ChangeSet(assign={"default_max_parallel_requests": 1})) + first: Final = await _open_stream(core) + with pytest.raises(litellm.RateLimitError): + await asyncio.wait_for(_open_stream(core), 5) + assert await collect_texts(first) == "x" + + +async def test_a_deployment_keeps_the_slot_limit_it_was_given_before_a_change( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The limit is cached per deployment on first use: a later default does not replace it.""" + host.answer("openai/a", chat_stream("x")) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=0, default_max_parallel_requests=1) + assert await collect_texts(await _open_stream(core)) == "x" + _ = core.apply(ChangeSet(assign={"default_max_parallel_requests": 5})) + first: Final = await _open_stream(core) + with pytest.raises(litellm.RateLimitError): + await asyncio.wait_for(_open_stream(core), 5) + assert await collect_texts(first) == "x" + + +async def test_a_budget_config_assigned_beside_the_budget_check_builds_the_limiter_with_it(make_core: MakeCore) -> None: + """``apply_changes`` assigns before ``update_settings`` adds the check.""" + core: Final = make_core(model_list=[deployment("g", "a")]) + _ = core.apply( + ChangeSet( + assign={"provider_budget_config": PROVIDER_BUDGET}, + settings={"optional_pre_call_checks": ["router_budget_limiting"]}, + ) + ) + limiter: Final = core.registry().plugins("budget_limiter") + assert isinstance(limiter, RouterBudgetLimiting) + assert limiter.provider_budget_config is core.settings().provider_budget_config + assert list(limiter.provider_budget_config or {}) == ["openai"] + core.retire() + + +async def test_a_budget_limiter_built_already_keeps_its_config(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], provider_budget_config=PROVIDER_BUDGET) + limiter: Final = core.registry().plugins("budget_limiter") + assert isinstance(limiter, RouterBudgetLimiting) + _ = core.apply(ChangeSet(assign={"provider_budget_config": {"azure": {"budget_limit": 2, "time_period": "1d"}}})) + assert core.registry().plugins("budget_limiter") is limiter + assert list(limiter.provider_budget_config or {}) == ["openai"] + assert list(core.settings().provider_budget_config or {}) == ["azure"] + core.retire() + + +# --- settings().guardrail(name) ----------------------------------------------------------------- + + +def test_settings_guardrail_returns_the_one_configured_under_the_name(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS) + assert core.settings().guardrail("gr") == GUARDRAILS[0] + + +def test_settings_guardrail_with_an_unknown_name_raises_value_error(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS) + with pytest.raises(ValueError, match="No guardrail found with name: nope"): + _ = core.settings().guardrail("nope") + + +def test_settings_guardrail_picks_among_same_named_guardrails_by_weight(make_core: MakeCore) -> None: + guardrails: Final = [ + {"guardrail_name": "gr", "litellm_params": {"guardrail": "never", "weight": 0}}, + {"guardrail_name": "gr", "litellm_params": {"guardrail": "always", "weight": 1}}, + ] + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=guardrails) + assert {core.settings().guardrail("gr")["litellm_params"]["guardrail"] for _ in range(20)} == {"always"} + + +def test_settings_guardrail_reads_a_guardrail_list_assigned_after_construction(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")]) + _ = core.apply(ChangeSet(assign={"guardrail_list": GUARDRAILS})) + assert core.settings().guardrail("gr") == GUARDRAILS[0] + + +# --- a default_max_parallel_requests that is no int --------------------------------------------- + + +@pytest.mark.parametrize("assigned", [False, True]) +async def test_a_float_default_max_parallel_requests_limits_the_slots( + make_core: MakeCore, host: ScriptedHost, assigned: bool +) -> None: + host.answer("openai/a", chat_stream("x")) + given: Final = {} if assigned else {"default_max_parallel_requests": 1.0} + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=0, **given) + if assigned: + _ = core.apply(ChangeSet(assign={"default_max_parallel_requests": 1.0})) + first: Final = await _open_stream(core) + with pytest.raises(litellm.RateLimitError): + await asyncio.wait_for(_open_stream(core), 5) + assert await collect_texts(first) == "x" + + +# --- health-check routing assigned after construction ------------------------------------------- + + +async def _healthy_ids(core: RouterCore, model: str) -> list[str]: # mutable-ok: a fresh list + select: Final[Callable[..., Awaitable[object]]] = getattr(core, "select") # pyright: ignore[reportAny] # the core's select, which takes kind + return ids_of(await select(model, kind="healthy")) + + +def _unhealthy(*ids: str) -> dict[str, dict[str, object]]: # mutable-ok: health states + return {i: {"is_healthy": False, "timestamp": time.time(), "reason": "probe failed"} for i in ids} + + +async def test_health_check_routing_turned_on_later_drops_unhealthy_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + core.health().health_checks.set_deployment_health_states(_unhealthy("a")) # pyright: ignore[reportArgumentType] # health states + assert await _healthy_ids(core, "g") == ["a", "b"] + _ = core.apply(ChangeSet(assign={"enable_health_check_routing": True})) + assert await _healthy_ids(core, "g") == ["b"] + _ = core.apply(ChangeSet(assign={"enable_health_check_routing": False})) + assert await _healthy_ids(core, "g") == ["a", "b"] + + +async def test_background_health_check_model_groups_assigned_later_scope_the_filter(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c"), deployment("h", "d")], + enable_health_check_routing=True, + ) + core.health().health_checks.set_deployment_health_states(_unhealthy("a", "c")) # pyright: ignore[reportArgumentType] # health states + assert await _healthy_ids(core, "h") == ["d"] + _ = core.apply(ChangeSet(assign={"background_health_check_model_groups": ["g"]})) + assert await _healthy_ids(core, "g") == ["b"] + assert await _healthy_ids(core, "h") == ["c", "d"] + + +# --- provider_default_deployment_ids assigned after construction -------------------------------- + + +@pytest.mark.parametrize(("default_ids", "cooled"), [([], ["a"]), (["a"], [])]) +async def test_an_assigned_provider_default_deployment_never_cools_down( + make_core: MakeCore, host: ScriptedHost, default_ids: list[str], cooled: list[str] +) -> None: + host.answer("openai/a", rate_limit()) + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0) + _ = core.apply(ChangeSet(assign={"provider_default_deployment_ids": default_ids})) + with pytest.raises(litellm.RateLimitError): + await core.call("acompletion", {"model": "a", "messages": MESSAGES}) + assert sorted(cooled_down(core, ["a", "b"])) == cooled + + +async def test_wildcard_deployments_never_cool_down(make_core: MakeCore, host: ScriptedHost) -> None: + """Each is a provider default deployment, which the cooldown check skips.""" + host.answer("openai/x", rate_limit()) + core: Final = make_core( + model_list=[deployment("openai/*", "w", model="openai/*"), deployment("openai/*", "w2", model="openai/*")], + num_retries=0, + ) + for _ in range(3): + with pytest.raises(litellm.RateLimitError): + await core.call("acompletion", {"model": "openai/x", "messages": MESSAGES}) + assert cooled_down(core, ["w", "w2"]) == {} + assert len(host.calls) == 3 + + +async def test_an_added_wildcard_deployment_never_cools_down(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/x", rate_limit()) + core: Final = make_core(model_list=[deployment("openai/*", "w", model="openai/*")], num_retries=0) + _ = core.apply(ChangeSet(upsert=[Deployment.model_validate(deployment("openai/*", "w2", model="openai/*"))])) + assert core.settings().provider_default_deployment_ids == ["w", "w2"] + for _ in range(3): + with pytest.raises(litellm.RateLimitError): + await core.call("acompletion", {"model": "openai/x", "messages": MESSAGES}) + assert cooled_down(core, ["w", "w2"]) == {} + + +# --- Kubernetes pod discovery ------------------------------------------------------------------- + + +class _RecordingDiscovery(KubernetesPodDiscovery): + """Resolves every pod-discovery deployment to one pod IP, and records the retry counts seen.""" + + def __init__(self) -> None: + super().__init__(proxy_environment=lambda: {}) + self.seen: list[object] = [] # mutable-ok: filled as picks resolve + + def _pod(self, deployment: object) -> object: + record: Final = dict(as_str_mapping(deployment) or {}) + params: Final = as_str_mapping(record.get("litellm_params")) or {} + if params.get("kubernetes_pod_discovery") is not True: + return deployment + record["litellm_params"] = {**params, "api_base": "http://10.0.0.7:4000"} + return record + + def resolve_deployment(self, deployment: object, request_kwargs: Mapping[str, object] | None = None) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # the resolver's generic signature, read as objects + self.seen.append(self._request_retry_count(request_kwargs)) + return self._pod(deployment) + + async def async_resolve_deployment( # pyright: ignore[reportIncompatibleMethodOverride] # see resolve_deployment + self, deployment: object, request_kwargs: Mapping[str, object] | None = None + ) -> object: + self.seen.append(self._request_retry_count(request_kwargs)) + return self._pod(deployment) + + +def _pod_deployment(deployment_id: str) -> dict[str, object]: # mutable-ok: a model_list entry + return deployment("g", deployment_id, api_base="http://svc.ns:4000", kubernetes_pod_discovery=True) + + +def test_settings_report_the_routers_pod_discovery(make_core: MakeCore) -> None: + settings: Final = make_core(model_list=[deployment("g", "d1")]).settings() + assert settings.is_set("kubernetes_pod_discovery") + assert isinstance(settings.kubernetes_pod_discovery, KubernetesPodDiscovery) + + +async def test_an_assigned_pod_discovery_resolves_each_attempt(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", rate_limit()) + discovery: Final = _RecordingDiscovery() + core: Final = make_core(model_list=[_pod_deployment("a")], num_retries=1) + _ = core.apply(ChangeSet(assign={"kubernetes_pod_discovery": discovery})) + assert core.settings().kubernetes_pod_discovery is discovery + with pytest.raises(litellm.RateLimitError): + await core.call("acompletion", {"model": "g", "messages": MESSAGES}) + assert [call.kwargs["api_base"] for call in host.calls] == ["http://10.0.0.7:4000"] * 2 + assert discovery.seen == [0, 1] + + +def test_an_assigned_pod_discovery_resolves_a_sync_attempt(make_core: MakeCore, host: ScriptedHost) -> None: + discovery: Final = _RecordingDiscovery() + core: Final = make_core(model_list=[_pod_deployment("a")]) + _ = core.apply(ChangeSet(assign={"kubernetes_pod_discovery": discovery})) + completion: Final[Callable[..., object]] = getattr(core, "completion") # pyright: ignore[reportAny] # see _endpoint + _ = completion(model="g", messages=MESSAGES) + assert host.calls[0].kwargs["api_base"] == "http://10.0.0.7:4000" + + +def _api_base(record: object) -> object: + return (as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {}).get("api_base") + + +async def test_a_pick_is_resolved_to_a_pod_and_a_healthy_listing_is_not(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[_pod_deployment("a")]) + _ = core.apply(ChangeSet(assign={"kubernetes_pod_discovery": _RecordingDiscovery()})) + assert _api_base(await core.select("g")) == "http://10.0.0.7:4000" + select: Final[Callable[..., Awaitable[object]]] = getattr(core, "select") # pyright: ignore[reportAny] # see _healthy_ids + listed: Final[object] = await select("g", kind="healthy") + assert isinstance(listed, list) + assert [_api_base(record) for record in listed] == ["http://svc.ns:4000"] # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # the listing + + +def test_a_sync_pick_is_resolved_to_a_pod(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[_pod_deployment("a")]) + _ = core.apply(ChangeSet(assign={"kubernetes_pod_discovery": _RecordingDiscovery()})) + pick: Final[Callable[..., object]] = getattr(core, "get_available_deployment") # pyright: ignore[reportAny] # the deprecated sync pick + assert _api_base(pick(model="g")) == "http://10.0.0.7:4000" + + +async def test_the_default_pod_discovery_resolves_a_service_host(make_core: MakeCore, host: ScriptedHost) -> None: + """``localhost`` resolves to a loopback address, which takes the service host's place.""" + core: Final = make_core( + model_list=[deployment("g", "a", api_base="http://localhost:4000", kubernetes_pod_discovery=True)] + ) + _ = core.apply(ChangeSet(assign={"kubernetes_pod_discovery": KubernetesPodDiscovery(proxy_environment=lambda: {})})) + await core.call("acompletion", {"model": "g", "messages": MESSAGES}) + assert str(host.calls[0].kwargs["api_base"]) in ("http://127.0.0.1:4000", "http://[::1]:4000") diff --git a/tests/unit/test_router/core/test_legacy_members.py b/tests/unit/test_router/core/test_legacy_members.py new file mode 100644 index 00000000000..1158a44e7fd --- /dev/null +++ b/tests/unit/test_router/core/test_legacy_members.py @@ -0,0 +1,711 @@ +""" +The old public Router members outside the entry points, on each core: registry lookups, the +routing-state lifecycle, the routing callbacks' checks, one provider call's accounting and the +pre-routing strategy builders (``RustLegacyMembers``, litellm/router_rust/surface/legacy_members.py). +""" + +import asyncio +import inspect +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +from litellm.containers.endpoint_factory import ( + _generated_endpoints, # pyright: ignore[reportPrivateUsage] # the names the routers register +) +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.token_counter import served_openai_models +from litellm.router_api import RoutingContext +from litellm.router_core import RouterCore +from litellm.router_python_core import PythonCore +from litellm.router_rust.surface.legacy_members import ( + _ASSISTANTS_ENDPOINTS, # pyright: ignore[reportPrivateUsage] # pinned against the Python core below + _ROUTER_ENDPOINTS, # pyright: ignore[reportPrivateUsage] # see above +) +from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_strategy.auto_router.auto_router_model_naming import HEURISTIC_V2_CAPABILITY +from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy +from litellm.router_strategy.least_busy import LeastBusyLoggingHandler +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck +from litellm.types.router import CustomRoutingStrategyBase, Deployment +from tests.unit.test_router.core.construction_precall_helpers import CallbackHost +from tests.unit.test_router.core.gaps_l11100_helpers import ( + TIERS, + adaptive_marker, + complexity_marker, + marker, + registered_tags, +) +from tests.unit.test_router.core.registry_helpers import MESSAGES, MakeCore, team_deployment +from tests.unit.test_router.core.retries_health_helpers import cooled_down +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + completion_response, + deployment, + python_core, + rate_limit, + rust_core, + server_error, +) + +GROUP: Final = [deployment("g", "a"), deployment("g", "b")] +PLAIN: Final = [deployment("small", "s"), deployment("big", "b")] + + +@pytest.fixture +def host() -> ScriptedHost: + """Switching an optional pre-call check off removes it from the host's lists.""" + return CallbackHost() + + +class Gate(CustomLogger): + """Records what its pre-call checks saw and raises ``error`` (when set) from them.""" + + def __init__(self, error: Exception | None = None) -> None: + super().__init__() + self.error = error + self.seen: Final[list[tuple[object, object]]] = [] + + def pre_call_check(self, deployment: dict) -> dict | None: + self.seen.append((deployment, None)) + if self.error is not None: + raise self.error + return None + + async def async_pre_call_check(self, deployment: dict, parent_otel_span: object) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride] + self.seen.append((deployment, parent_otel_span)) + if self.error is not None: + raise self.error + return None + + +class Narrow(CustomLogger): + """Keeps the deployments other than ``dropped``, or raises ``error``.""" + + def __init__(self, dropped: str = "", error: Exception | None = None) -> None: + super().__init__() + self.dropped = dropped + self.error = error + self.calls: Final[list[Mapping[str, object]]] = [] + + async def async_filter_deployments( + self, + model: str, + healthy_deployments: list, + messages: object, + request_kwargs: object = None, + parent_otel_span: object = None, + ) -> list: + self.calls.append( + { + "model": model, + "messages": messages, + "request_kwargs": request_kwargs, + "parent_otel_span": parent_otel_span, + } + ) + if self.error is not None: + raise self.error + return [d for d in healthy_deployments if d["model_info"]["id"] != self.dropped] + + +class RecordingLogging: + """The logging object's failure dispatch, recorded.""" + + def __init__(self) -> None: + self.dispatched: Final[list[Mapping[str, object]]] = [] + + async def dispatch_failure_handlers(self, **kwargs: object) -> None: # kwargs-ok: recorded as passed + self.dispatched.append(kwargs) + + +def counters(core: RouterCore, deployment_id: str, kind: str) -> list[object]: # mutable-ok: a fresh list + """The per-minute ``kind`` (``tpm``, ``rpm``) counters of ``deployment_id`` in the routing cache.""" + memory: Final = core.health().store.in_memory_cache + return [ + memory.get_cache(key) + for key in tuple(memory.cache_dict) + if key.startswith(f"global_router:{deployment_id}:") and f":{kind}:" in key + ] + + +def call_metadata(deployment_id: str = "a") -> dict[str, object]: # mutable-ok: the request's own bucket + return {"model_info": {"id": deployment_id}, "deployment": f"openai/{deployment_id}", "model_group": "g"} + + +# --- registry lookups -------------------------------------------------------------------------- + + +def test_get_routing_group_names_a_group_a_deployment_does_not_shadow(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m2", "b")], + routing_groups=[{"group_name": "grp", "models": ["m1", "m2"], "routing_strategy": "simple-shuffle"}], + ) + group: Final = core.get_routing_group("grp") # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert group is not None + assert group.group_name == "grp" + assert core.get_routing_group("m1") is None # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.get_routing_group("nope") is None # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_team_public_model_names_of_lists_one_teams_public_names(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + team_deployment("int-1", "t1", "TA", "pub-a"), + team_deployment("int-2", "t2", "TB", "pub-b"), + ] + ) + + assert core.team_public_model_names_of("TA") == frozenset({"pub-a"}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + assert core.team_public_model_names_of("nobody") == frozenset() # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_served_openai_models_reads_the_group_and_follows_a_model_list_change(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o")]) + before: Final = core.served_openai_models("g") # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert before == served_openai_models(core.registry().deployments("g", source="named") or ()) + assert core.served_openai_models("nope") == () # pyright: ignore[reportAttributeAccessIssue] # see above + + core.set_model_list([deployment("g", "b", model="openai/gpt-4o-mini")]) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert core.served_openai_models("g") == served_openai_models( + core.registry().deployments("g", source="named") or () + ) # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_the_tier_params_never_dropped_constant_matches(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP)) + + assert core.TIER_PARAMS_NEVER_DROPPED == PythonCore.TIER_PARAMS_NEVER_DROPPED # pyright: ignore[reportAttributeAccessIssue] # a class constant + + +def test_auto_router_capability_violation_resolves_the_limit_on_every_call(make_core: MakeCore) -> None: + limits: Final[list[int | None]] = [None] # mutable-ok: what the resolver answers + + core: Final = make_core( + model_list=[*PLAIN, complexity_marker(config={"classifier_type": "heuristic_v2", "tiers": dict(TIERS)})], + auto_router_capability_limit=lambda: limits[0], + ) + + assert core.auto_router_capability_violation(HEURISTIC_V2_CAPABILITY) is None # pyright: ignore[reportAttributeAccessIssue] # an old public member + limits[0] = 1 + violation: Final = core.auto_router_capability_violation(HEURISTIC_V2_CAPABILITY) # pyright: ignore[reportAttributeAccessIssue] # see above + assert violation is not None + assert violation.startswith("At most 1 auto-router(s) using the heuristic_v2 classifier can be registered") + + +# --- routing state ------------------------------------------------------------------------------ + + +def test_retire_routing_state_retires_the_selectors_and_keeps_them_registered( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + core: Final = make_core(model_list=list(GROUP), routing_strategy="usage-based-routing-v2") + selector: Final = core.lowesttpm_logger_v2 # pyright: ignore[reportAttributeAccessIssue] # an old public attribute + assert isinstance(selector, BaseRoutingStrategy) + retired: Final[list[object]] = [] # mutable-ok: what retire() saw + monkeypatch.setattr(selector, "retire", lambda: retired.append(selector)) + + core.retire_routing_state() # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert retired == [selector] + assert core.lowesttpm_logger_v2 is selector # pyright: ignore[reportAttributeAccessIssue] # see above + assert selector in host.lists["callbacks"] + + +def test_routing_strategy_init_refuses_an_unknown_strategy_and_changes_nothing(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP), routing_strategy="least-busy") + selector: Final = core.leastbusy_logger # pyright: ignore[reportAttributeAccessIssue] # an old public attribute + + with pytest.raises(ValueError, match="not-a-real-strategy"): + core.routing_strategy_init(routing_strategy="not-a-real-strategy", routing_strategy_args={}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert core.leastbusy_logger is selector # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_routing_strategy_init_replaces_the_selector_and_leaves_the_setting( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(GROUP), routing_strategy="least-busy") + old: Final = core.leastbusy_logger # pyright: ignore[reportAttributeAccessIssue] # an old public attribute + + core.routing_strategy_init("usage-based-routing-v2", {}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert core.leastbusy_logger is None # pyright: ignore[reportAttributeAccessIssue] # see above + selector: Final = core.lowesttpm_logger_v2 # pyright: ignore[reportAttributeAccessIssue] # see above + assert isinstance(selector, LowestTPMLoggingHandler_v2) + assert selector in host.lists["callbacks"] + assert old not in host.lists["callbacks"] + assert core.settings().routing_strategy == "least-busy" + + +@pytest.mark.asyncio +async def test_a_request_override_of_the_initialized_strategy_is_not_the_routers_own(make_core: MakeCore) -> None: + """``_get_override_strategy_selector`` compares with the router's ``routing_strategy``, which + ``routing_strategy_init`` leaves, so the request gets its own selector.""" + core: Final = make_core(model_list=list(GROUP)) + core.routing_strategy_init("least-busy", {}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + initialized: Final = core.leastbusy_logger # pyright: ignore[reportAttributeAccessIssue] # an old public attribute + assert isinstance(initialized, LeastBusyLoggingHandler) + + picked: Final = await core.select("g", {"routing_strategy": "least-busy", "messages": list(MESSAGES)}) + + assert picked["model_info"]["id"] in {"a", "b"} # pyright: ignore[reportIndexIssue] # the pick + assert core.leastbusy_logger is initialized # pyright: ignore[reportAttributeAccessIssue] # see above + + +@pytest.mark.asyncio +async def test_routing_strategy_init_drops_a_custom_strategy(make_core: MakeCore) -> None: + class Custom(CustomRoutingStrategyBase): + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> dict: # kwargs-ok: any pick + return dict(GROUP[0]) + + def get_available_deployment(self, *args: object, **kwargs: object) -> dict: # kwargs-ok: any pick + return dict(GROUP[0]) + + core: Final = make_core(model_list=list(GROUP)) + core.register_plugin(Custom()) + assert {(await core.select("g", {"messages": list(MESSAGES)}))["model_info"]["id"] for _ in range(20)} == {"a"} # pyright: ignore[reportIndexIssue] # the pick + + core.routing_strategy_init("simple-shuffle", {}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + picks: Final = {(await core.select("g", {"messages": list(MESSAGES)}))["model_info"]["id"] for _ in range(60)} # pyright: ignore[reportIndexIssue] # the pick + assert picks == {"a", "b"} + + +def test_set_optional_pre_call_checks_adds_and_drops_the_toggleable_checks( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(GROUP)) + + core.set_optional_pre_call_checks(["prompt_caching"]) # pyright: ignore[reportAttributeAccessIssue] # an old public member + assert [type(callback) for callback in core.optional_callbacks] == [PromptCachingDeploymentCheck] # pyright: ignore[reportAttributeAccessIssue] # see above + assert any(isinstance(callback, PromptCachingDeploymentCheck) for callback in host.lists["callbacks"]) + + core.set_optional_pre_call_checks(None) # pyright: ignore[reportAttributeAccessIssue] # see above + assert [type(callback) for callback in core.optional_callbacks] == [PromptCachingDeploymentCheck] # pyright: ignore[reportAttributeAccessIssue] # see above + + core.set_optional_pre_call_checks([]) # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue] # see above + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in host.lists["callbacks"]) + + +def test_initialize_assistants_endpoint_replaces_an_assigned_endpoint(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP)) + assigned: Final = object() + core.aget_assistants = assigned # pyright: ignore[reportAttributeAccessIssue] # an old public endpoint + core.amoderation = assigned # pyright: ignore[reportAttributeAccessIssue] # see above + + core.initialize_assistants_endpoint() # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert core.aget_assistants is not assigned # pyright: ignore[reportAttributeAccessIssue] # see above + assert callable(core.aget_assistants) # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.amoderation is assigned # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_initialize_router_endpoints_replaces_only_what_it_binds(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP)) + assigned: Final = object() + for name in ("amoderation", "aresponses", "aupload_container_file", "aget_assistants", "acreate_file"): + setattr(core, name, assigned) + + core.initialize_router_endpoints() # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert core.amoderation is not assigned # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.aresponses is not assigned # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.aupload_container_file is not assigned # pyright: ignore[reportAttributeAccessIssue] # see above + # `initialize_router_endpoints` binds neither of these: the assistants have their own, and + # `acreate_file` is a method an assignment shadows. + assert core.aget_assistants is assigned # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.acreate_file is assigned # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_the_endpoint_names_are_the_ones_the_python_core_binds() -> None: + bound: Final = vars(python_core(ScriptedHost(), model_list=[])) + listed: Final = {*_ROUTER_ENDPOINTS, *_ASSISTANTS_ENDPOINTS, *_generated_endpoints} + + assert listed <= set(bound) + wrappers: Final = { + name + for name, value in bound.items() + if inspect.isfunction(value) and value.__qualname__.startswith("PythonCore.factory_function.") + } + assert wrappers <= listed + + +# --- pre-routing strategies ---------------------------------------------------------------------- + + +def test_init_complexity_router_deployment_registers_it_once(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PLAIN)) + router: Final = Deployment(**complexity_marker()) + + core.init_complexity_router_deployment(router) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert registered_tags(core, "complexity") == {"smart": [()]} + with pytest.raises(ValueError, match="already exists"): + core.init_complexity_router_deployment(router) # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_init_quality_router_deployment_needs_a_default_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PLAIN)) + + with pytest.raises(ValueError, match="quality_router_default_model is required"): + core.init_quality_router_deployment(Deployment(**marker("q", "m", "auto_router/quality_router"))) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + core.init_quality_router_deployment( # pyright: ignore[reportAttributeAccessIssue] # see above + Deployment(**marker("q", "m", "auto_router/quality_router", quality_router_default_model="big")) + ) + assert registered_tags(core, "quality") == {"q": [()]} + + +def test_init_adaptive_router_deployment_registers_it_and_its_hook(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=list(PLAIN)) + + core.init_adaptive_router_deployment(Deployment(**adaptive_marker("ad", "m", ["big"]))) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert registered_tags(core, "adaptive") == {"ad": [()]} + assert any(isinstance(callback, AdaptiveRouterPostCallHook) for callback in host.lists["callbacks"]) + + +def test_init_auto_router_deployment_needs_its_config(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PLAIN)) + + with pytest.raises(ValueError, match="auto_router_config_path or auto_router_config is required"): + core.init_auto_router_deployment(Deployment(**marker("auto", "m", "auto_router/semantic"))) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + +# --- routing callbacks --------------------------------------------------------------------------- + + +def test_routing_strategy_pre_call_checks_hands_each_callback_the_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + gate: Final = Gate() + host.lists["callbacks"].append(gate) + core: Final = make_core(model_list=list(GROUP)) + picked: Final = dict(GROUP[0]) + + core.routing_strategy_pre_call_checks(picked) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert gate.seen[0][0] is picked + + gate.error = rate_limit(message="sync gate") + with pytest.raises(litellm.RateLimitError, match="sync gate"): + core.routing_strategy_pre_call_checks(picked) # pyright: ignore[reportAttributeAccessIssue] # see above + assert cooled_down(core, ["a"]) == {} + + +@pytest.mark.asyncio +async def test_a_rate_limited_async_pre_call_check_cools_down_and_logs_the_failure( + make_core: MakeCore, host: ScriptedHost +) -> None: + gate: Final = Gate(rate_limit(message="gate says no")) + host.lists["callbacks"].append(gate) + core: Final = make_core(model_list=list(GROUP), cooldown_time=33) + logging_obj: Final = RecordingLogging() + span: Final = object() + + with pytest.raises(litellm.RateLimitError, match="gate says no") as raised: + await core.async_routing_strategy_pre_call_checks( # pyright: ignore[reportAttributeAccessIssue] # an old public member + deployment=dict(GROUP[0]), parent_otel_span=span, logging_obj=logging_obj + ) + await asyncio.sleep(0) + + assert gate.seen[0][1] is span + assert logging_obj.dispatched[0]["exception"] is raised.value + assert logging_obj.dispatched[0]["prefer_async_handlers"] is True + assert set(cooled_down(core, ["a", "b"])) == {"a"} + + +@pytest.mark.asyncio +async def test_another_async_pre_call_check_error_logs_the_failure_without_a_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.lists["callbacks"].append(Gate(ValueError("no"))) + core: Final = make_core(model_list=list(GROUP)) + logging_obj: Final = RecordingLogging() + + with pytest.raises(ValueError, match="no"): + await core.async_routing_strategy_pre_call_checks( # pyright: ignore[reportAttributeAccessIssue] # an old public member + deployment=dict(GROUP[0]), parent_otel_span=None, logging_obj=logging_obj + ) + await asyncio.sleep(0) + + assert len(logging_obj.dispatched) == 1 + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.asyncio +async def test_async_callback_filter_deployments_narrows_through_each_callback( + make_core: MakeCore, host: ScriptedHost +) -> None: + first: Final = Narrow(dropped="a") + second: Final = Narrow() + host.lists["callbacks"].extend((first, second)) + core: Final = make_core(model_list=list(GROUP)) + span: Final = object() + request: Final = {"metadata": {}} + + remaining: Final = await core.async_callback_filter_deployments( # pyright: ignore[reportAttributeAccessIssue] # an old public member + model="g", + healthy_deployments=[dict(entry) for entry in GROUP], + messages=list(MESSAGES), + parent_otel_span=span, + request_kwargs=request, + ) + + assert [entry["model_info"]["id"] for entry in remaining] == ["b"] + assert second.calls[0]["parent_otel_span"] is span + assert second.calls[0]["request_kwargs"] is request + + +@pytest.mark.asyncio +async def test_a_failing_callback_filter_logs_the_failure_and_raises(make_core: MakeCore, host: ScriptedHost) -> None: + host.lists["callbacks"].append(Narrow(error=ValueError("filter refused"))) + core: Final = make_core(model_list=list(GROUP)) + logging_obj: Final = RecordingLogging() + + with pytest.raises(ValueError, match="filter refused"): + await core.async_callback_filter_deployments( # pyright: ignore[reportAttributeAccessIssue] # an old public member + model="g", + healthy_deployments=[dict(entry) for entry in GROUP], + messages=None, + parent_otel_span=None, + logging_obj=logging_obj, + ) + await asyncio.sleep(0) + + assert len(logging_obj.dispatched) == 1 + + +@pytest.mark.asyncio +async def test_async_admit_deployment_turn_runs_the_checks_on_the_deployment_as_configured_now( + make_core: MakeCore, host: ScriptedHost +) -> None: + gate: Final = Gate() + host.lists["callbacks"].append(gate) + core: Final = make_core(model_list=list(GROUP)) + + admission: Final = await core.async_admit_deployment_turn("g", {"model_info": {"id": "a"}}, {}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert dict(admission) == {} + assert gate.seen[0][0]["litellm_params"]["model"] == "openai/a" # pyright: ignore[reportIndexIssue] # the deployment dict + + +@pytest.mark.asyncio +async def test_a_rate_limited_turn_cools_the_deployment_down(make_core: MakeCore, host: ScriptedHost) -> None: + host.lists["callbacks"].append(Gate(rate_limit(message="turn refused"))) + core: Final = make_core(model_list=list(GROUP)) + + with pytest.raises(litellm.RateLimitError, match="turn refused"): + await core.async_admit_deployment_turn("g", {"model_info": {"id": "a"}}, {}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert set(cooled_down(core, ["a", "b"])) == {"a"} + + +# --- one provider call --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_make_call_counts_a_tracked_deployments_usage_and_notes_it_on_the_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o", rpm=5)]) + metadata: Final = call_metadata() + + async def endpoint(**kwargs: object) -> object: # kwargs-ok: the request, as make_call passes it + return completion_response(total_tokens=10) + + await core.make_call(endpoint, model="g", metadata=metadata) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert counters(core, "a", "tpm") == [10] + assert counters(core, "a", "rpm") == [1] + assert metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] == 10 + + +@pytest.mark.asyncio +async def test_make_call_runs_a_sync_endpoint_and_stamps_the_response(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o")]) + answer: Final = completion_response() + + def endpoint(**kwargs: object) -> object: # kwargs-ok: the request, as make_call passes it + return answer + + response: Final = await core.make_call(endpoint, model="g", metadata=call_metadata()) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert response is answer + assert answer._hidden_params["model_id"] == "a" # pyright: ignore[reportPrivateUsage] # what set_response_headers stamps + assert counters(core, "a", "tpm") == [] + + +@pytest.mark.asyncio +async def test_make_call_raises_the_endpoints_error(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o", rpm=5)]) + metadata: Final = call_metadata() + + async def endpoint(**kwargs: object) -> object: # kwargs-ok: the request, as make_call passes it + raise server_error() + + with pytest.raises(litellm.InternalServerError): + await core.make_call(endpoint, model="g", metadata=metadata) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert counters(core, "a", "rpm") == [] + assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in metadata + + +@pytest.mark.asyncio +async def test_increment_usage_notes_tokens_even_for_a_deployment_it_does_not_track(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o")]) + metadata: Final = call_metadata() + + await core.increment_deployment_usage_for_response( # pyright: ignore[reportAttributeAccessIssue] # an old public member + response=completion_response(total_tokens=7), request_kwargs={"model": "g", "metadata": metadata} + ) + + assert metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] == 7 + assert counters(core, "a", "rpm") == [] + + +@pytest.mark.asyncio +async def test_increment_usage_takes_the_note_back_when_the_count_fails( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o", rpm=5)]) + metadata: Final = call_metadata() + + async def refused(**kwargs: object) -> None: # kwargs-ok: the pipeline's arguments + raise ConnectionError("the usage store is down") + + monkeypatch.setattr(core.health().store, "async_increment_cache_pipeline", refused) + + await core.increment_deployment_usage_for_response(completion_response(), {"model": "g", "metadata": metadata}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in metadata + assert counters(core, "a", "rpm") == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_kwargs", + [{"model": "g", "metadata": {}}, {"metadata": call_metadata()}, {"model": "g"}], + ids=["no-model-info", "no-model", "no-metadata"], +) +async def test_increment_usage_skips_a_request_with_nothing_to_count_under( + make_core: MakeCore, request_kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o", rpm=5)]) + + await core.increment_deployment_usage_for_response(completion_response(), dict(request_kwargs)) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert counters(core, "a", "rpm") == [] + + +@pytest.mark.asyncio +async def test_increment_usage_counts_nothing_for_no_response(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model="openai/gpt-4o", rpm=5)]) + metadata: Final = call_metadata() + + await core.increment_deployment_usage_for_response(None, {"model": "g", "metadata": metadata}) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in metadata + + +@pytest.mark.asyncio +async def test_async_deployment_callback_on_failure_counts_one_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP)) + logged: Final = {"litellm_params": {"metadata": call_metadata(), "model_info": {"id": "a"}}} + + await core.async_deployment_callback_on_failure(logged, None, None, None) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert counters(core, "a", "rpm") == [1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "logged", + [ + {"call_type": "aretrieve_batch", "litellm_params": {"metadata": call_metadata(), "model_info": {"id": "a"}}}, + {"litellm_params": {"metadata": {"deployment": "x"}, "model_info": {"id": "a"}}}, + {"litellm_params": {"metadata": call_metadata()}}, + ], + ids=["batch-retrieve", "no-group", "no-id"], +) +async def test_async_deployment_callback_on_failure_skips(make_core: MakeCore, logged: dict[str, object]) -> None: + core: Final = make_core(model_list=list(GROUP)) + + await core.async_deployment_callback_on_failure(logged, None, None, None) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + assert counters(core, "a", "rpm") == [] + + +@pytest.mark.asyncio +async def test_async_deployment_callback_on_failure_needs_the_params(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP)) + + with pytest.raises(KeyError, match="litellm_params"): + await core.async_deployment_callback_on_failure({}, None, None, None) # pyright: ignore[reportAttributeAccessIssue] # an old public member + + +# --- the plugin context -------------------------------------------------------------------------- + + +def test_deployment_params_merge_in_the_order_the_deployment_receives_them(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", temperature=0.1, tool_choice="auto", top_p=0.5)], + default_litellm_params={"temperature": 0.9, "tool_choice": "none"}, + ) + target: Final = core.registry().deployment("a") + + params: Final = RoutingContext(core).deployment_params({"top_p": 0.7}, target) # pyright: ignore[reportArgumentType] # the core a context wraps + + assert params["model"] == "openai/a" + assert params["temperature"] == 0.9 + assert params["tool_choice"] == "auto" + assert params["top_p"] == 0.7 + assert RoutingContext(core).deployment_params({}, None).get("temperature") == 0.9 # pyright: ignore[reportArgumentType] # see above + + +def test_update_kwargs_before_fallbacks_writes_the_group_into_the_named_bucket(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP), model_group_alias={"al": "g"}) + request: Final[dict[str, object]] = {"litellm_trace_id": "t", "litellm_metadata": {"k": "v"}} + + RoutingContext(core).update_kwargs_before_fallbacks("al", request, "litellm_metadata") # pyright: ignore[reportArgumentType] # see above + + assert request == { + "litellm_trace_id": "t", + "litellm_metadata": {"k": "v", "model_group": "al", "model_group_alias": "al"}, + } + + plain: Final[dict[str, object]] = {} # mutable-ok: the request the core writes into + RoutingContext(core).update_kwargs_before_fallbacks("g", plain) # pyright: ignore[reportArgumentType] # see above + + assert isinstance(plain["litellm_trace_id"], str) + assert plain["metadata"] == {"model_group": "g", "model_group_alias": None} + + +# --- what stays refused -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name", + [ + "async_function_with_fallbacks", + "async_function_with_fallbacks_common_utils", + "async_function_with_retries", + "function_with_fallbacks", + "factory_function", + "model_id_to_deployment_index_map", + ], +) +def test_the_rust_core_refuses_what_it_cannot_serve(name: str) -> None: + from litellm.router_rust.core import RustCoreUnsupported + + core: Final = rust_core(ScriptedHost(), model_list=list(GROUP)) + + with pytest.raises(RustCoreUnsupported): + getattr(core, name) diff --git a/tests/unit/test_router/core/test_parity_cooldown_service_model.py b/tests/unit/test_router/core/test_parity_cooldown_service_model.py new file mode 100644 index 00000000000..4204ebeeca3 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_cooldown_service_model.py @@ -0,0 +1,901 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on three things the logging pipeline and the +fuzzer read: the cooldown event Prometheus counts (``router_cooldown_event_callback``, scheduled by +``set_cooldown_deployments``), the ``ServiceTypes.ROUTER`` service events, and a ``model`` that is no +string, which the native calls cannot take. + +``tests/unit/test_router/core/conftest.py`` skips a test that reaches ``RustCoreUnsupported``. These +members are served, so ``served`` turns that refusal into a failure. + +The scripted host runs a failure callback where the test puts it. ``litellm`` runs it on the loop's +thread for an async call, inline for a sync one, and on a thread with no loop for a stream; in every +case ``CustomLogger.log_event`` swallows what the callback raises. ``logged_like_litellm`` and +``streams_log_off_the_loop`` give the host those two behaviours. +""" + +import asyncio +import contextlib +import functools +import inspect +import re +import time +from collections.abc import Awaitable, Callable, Coroutine, Generator, Mapping +from dataclasses import dataclass, field +from typing import Final, TypeVar + +import pydantic +import pytest + +import litellm +from litellm._service_logger import ServiceLogging +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_core import RouterCore +from litellm.router_utils import cooldown_callbacks, cooldown_handlers +from litellm.types.services import ServiceTypes +from tests.unit.test_router.core.retries_health_helpers import ( + MESSAGES, + api_connection, + attempt, + attempt_sync, + cooled_down, + fresh, + not_found, + rate_limited, + unprocessable, +) +from tests.unit.test_router.core.scripted_host import Call, ScriptedHost, deployment, rate_limit, server_error +from tests.unit.test_router.core.streaming_helpers import HostLogging, chat_stream, collect_texts, collect_texts_sync + +MakeCore = Callable[..., RouterCore] +_Test = TypeVar("_Test", bound=Callable[..., object]) + +ACQUIRE: Final = "async_get_available_deployment" +STRATEGY: Final = ".async_get_available_deployments" + + +def _refused(error: BaseException) -> None: + from litellm.router_rust.core import RustCoreUnsupported + + if isinstance(error, RustCoreUnsupported) or isinstance(error.__cause__, RustCoreUnsupported): + pytest.fail(f"the Rust core stopped serving this: {error}") + + +def served(test: _Test) -> _Test: + """Fail, rather than skip, when the test reaches ``RustCoreUnsupported``.""" + + @functools.wraps(test) + def run(*args: object, **kwargs: object) -> object: # kwargs-ok: the test's fixtures + try: + return test(*args, **kwargs) + except Exception as error: + _refused(error) + raise + + @functools.wraps(test) + async def arun(*args: object, **kwargs: object) -> object: # kwargs-ok: the test's fixtures + try: + return await test(*args, **kwargs) # pyright: ignore[reportGeneralTypeIssues] # an async test + except Exception as error: + _refused(error) + raise + + return arun if inspect.iscoroutinefunction(test) else run # pyright: ignore[reportReturnType] # the test, wrapped + + +async def settle() -> None: + """Let the tasks a call scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +# --- cooldown events ------------------------------------------------------------------------------ + + +@dataclass +class CooldownEvents: + """``router_cooldown_event_callback`` calls, as ``(deployment id, status, cooldown time)``.""" + + fired: list[tuple[str, object, object]] = field(default_factory=list) # mutable-ok: appended as they fire + + +@pytest.fixture +def events(monkeypatch: pytest.MonkeyPatch) -> CooldownEvents: + recorded: Final = CooldownEvents() + + async def record(ctx: object, deployment_id: str, exception_status: object, cooldown_time: object) -> None: + recorded.fired.append((deployment_id, exception_status, cooldown_time)) + + monkeypatch.setattr(cooldown_handlers, "router_cooldown_event_callback", record) + return recorded + + +@pytest.fixture +def logged_like_litellm(monkeypatch: pytest.MonkeyPatch) -> None: + """``CustomLogger.log_event``: what a sync failure callback raises stops at the logging.""" + log_sync: Final = ScriptedHost.log_sync + + def swallowed(host: ScriptedHost, call: Call, error: BaseException | None, response: object) -> None: + with contextlib.suppress(RuntimeError): + log_sync(host, call, error, response) + + monkeypatch.setattr(ScriptedHost, "log_sync", swallowed) + + +@pytest.fixture +def streams_log_off_the_loop(monkeypatch: pytest.MonkeyPatch, logged_like_litellm: None) -> None: + """``dispatch_failure_handlers``: a stream's sync callbacks run in ``executor.submit``, off the loop.""" + + def dispatch(logging: HostLogging, exception: Exception, _traceback: str) -> Coroutine[None, None, None]: + async def run() -> None: + await asyncio.to_thread(logging.failure_handler, exception) + + return run() + + monkeypatch.setattr(HostLogging, "dispatch_attempt_failure_handlers", dispatch) + + +def fired_exactly(events: CooldownEvents) -> list[str]: # mutable-ok: a fresh list + """The events as written: ``5 == 5.0``, and the cooldown's type is part of the event.""" + return [repr(fired) for fired in events.fired] + + +def lone(**arguments: object) -> Mapping[str, object]: # kwargs-ok: the router's constructor arguments + return {"model_list": [deployment("g", "a")], "num_retries": 0, **arguments} + + +def ordered(**arguments: object) -> Mapping[str, object]: # kwargs-ok: the router's constructor arguments + return { + "model_list": [deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + "num_retries": 0, + "allowed_fails": 0, + **arguments, + } + + +@served +async def test_a_failed_attempt_fires_one_event_with_the_status_and_the_cooldown( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents +) -> None: + core: Final = make_core(**lone(cooldown_time=7)) + host.answer("openai/a", fresh(not_found)) + + assert isinstance(await attempt(core), litellm.NotFoundError) + await settle() + + assert fired_exactly(events) == [repr(("a", 404, 7))] + + +@served +async def test_a_second_call_refused_by_the_cooldown_fires_nothing_more( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents +) -> None: + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(not_found)) + + _ = await attempt(core) + assert isinstance(await attempt(core), litellm.RouterRateLimitError) + await settle() + + assert [fired[0] for fired in events.fired] == ["a"] + + +@served +async def test_each_retry_that_cools_down_fires_its_own_event( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=1, retry_after=0, allowed_fails=0 + ) + host.answer("*", fresh(server_error)) + + _ = await attempt(core) + await settle() + + assert sorted(fired[0] for fired in events.fired) == ["a", "b"] + + +@pytest.mark.parametrize("builder", [unprocessable, api_connection]) +@served +async def test_a_failure_that_sets_no_cooldown_fires_nothing( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents, builder: Callable[[], Exception] +) -> None: + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(builder)) + + _ = await attempt(core) + await settle() + + assert cooled_down(core, ["a"]) == {} + assert events.fired == [] + + +@served +async def test_disabled_cooldowns_fire_nothing(make_core: MakeCore, host: ScriptedHost, events: CooldownEvents) -> None: + core: Final = make_core(**lone(disable_cooldowns=True)) + host.answer("openai/a", fresh(not_found)) + + _ = await attempt(core) + await settle() + + assert cooled_down(core, ["a"]) == {} + assert events.fired == [] + + +@served +async def test_a_sync_call_on_a_running_loop_fires_on_that_loop( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents +) -> None: + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(not_found)) + + assert isinstance(attempt_sync(core), litellm.NotFoundError) + await settle() + + assert fired_exactly(events) == [repr(("a", 404, DEFAULT_COOLDOWN_TIME_SECONDS))] + + +@served +def test_a_sync_call_with_no_loop_cools_down_and_fires_on_its_own_loop( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents, logged_like_litellm: None +) -> None: + """``function_with_fallbacks`` runs the sync walk on a private loop, which the event's task lands on.""" + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(not_found)) + + assert isinstance(attempt_sync(core), litellm.NotFoundError) + + assert set(cooled_down(core, ["a"])) == {"a"} + assert fired_exactly(events) == [repr(("a", 404, DEFAULT_COOLDOWN_TIME_SECONDS))] + + +SYNC_ENDPOINTS: Final[Mapping[str, Callable[[RouterCore], object]]] = { + "completion": lambda core: core.completion(model="g", messages=MESSAGES), # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownLambdaType] # endpoint method + "embedding": lambda core: core.embedding(model="g", input=["hi"]), # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownLambdaType] # endpoint method + "image_generation": lambda core: core.image_generation(model="g", prompt="hi"), # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownLambdaType] # endpoint method +} + + +@served +@pytest.mark.parametrize("endpoint", sorted(SYNC_ENDPOINTS)) +def test_every_sync_endpoint_fires_the_default_cooldown_as_the_int_it_is( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents, logged_like_litellm: None, endpoint: str +) -> None: + host.answer("openai/a", fresh(not_found)) + + with pytest.raises(litellm.NotFoundError, match="no such model"): + _ = SYNC_ENDPOINTS[endpoint](make_core(**lone())) + + assert fired_exactly(events) == [repr(("a", 404, DEFAULT_COOLDOWN_TIME_SECONDS))] + + +@served +@pytest.mark.parametrize( + ("arguments", "request_fields", "expected"), + [ + pytest.param({"cooldown_time": 7}, {}, 7, id="router-int"), + pytest.param({"cooldown_time": 7.0}, {}, 7.0, id="router-whole-float"), + pytest.param({"cooldown_time": 7.5}, {}, 7.5, id="router-float"), + pytest.param({"cooldown_time": 0}, {}, DEFAULT_COOLDOWN_TIME_SECONDS, id="router-zero-is-unset"), + pytest.param({"model_list": [deployment("g", "a", cooldown_time=7)]}, {}, 7, id="deployment-int"), + pytest.param({"model_list": [deployment("g", "a", cooldown_time=7.0)]}, {}, 7.0, id="deployment-float"), + pytest.param( + {"cooldown_time": 7, "model_list": [deployment("g", "a", cooldown_time=7.0)]}, + {}, + 7.0, + id="deployment-float-over-router-int", + ), + pytest.param( + {"cooldown_time": 7.0, "model_list": [deployment("g", "a", cooldown_time=7)]}, + {}, + 7, + id="deployment-int-over-router-float", + ), + pytest.param({}, {"cooldown_time": 7}, 7, id="request-int"), + pytest.param({}, {"cooldown_time": 7.0}, 7.0, id="request-float"), + pytest.param( + {"model_list": [deployment("g", "a", cooldown_time=7.0)]}, + {"cooldown_time": 7}, + 7, + id="request-int-over-deployment-float", + ), + pytest.param( + {"model_list": [deployment("g", "a", model_info={"cooldown_time": 7})]}, + {"cooldown_time": 7.0}, + 7, + id="model-info-int-over-request-float", + ), + ], +) +def test_the_event_carries_the_number_python_chose_as_it_was_written( + make_core: MakeCore, + host: ScriptedHost, + events: CooldownEvents, + logged_like_litellm: None, + arguments: Mapping[str, object], + request_fields: Mapping[str, object], + expected: float, +) -> None: + host.answer("openai/a", fresh(not_found)) + + assert isinstance(attempt_sync(make_core(**lone(**arguments)), **request_fields), litellm.NotFoundError) + + assert fired_exactly(events) == [repr(("a", 404, expected))] + + +@served +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + pytest.param({}, 9, id="over-the-default"), + pytest.param({"cooldown_time": 9.0}, 9, id="an-int-over-an-equal-router-float"), + ], +) +def test_a_retry_after_header_fires_as_the_int_python_reads( + make_core: MakeCore, + host: ScriptedHost, + events: CooldownEvents, + logged_like_litellm: None, + arguments: Mapping[str, object], + expected: int, +) -> None: + for name in ("a", "b"): + host.answer(f"openai/{name}", fresh(lambda: rate_limited("9"))) + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, allowed_fails=0, **arguments + ) + + assert isinstance(attempt_sync(core), litellm.RateLimitError) + + assert {repr(fired[1:]) for fired in events.fired} == {repr((429, expected))} + + +@served +async def test_an_async_stream_failure_cools_down_and_fires_nothing( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents, streams_log_off_the_loop: None +) -> None: + host.answer("openai/a", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True)) + core: Final = make_core(**ordered()) + + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + with pytest.raises(litellm.RateLimitError, match="mid stream"): + await collect_texts(stream) + await asyncio.sleep(0.05) + await settle() + + assert set(cooled_down(core, ["a", "b"])) == {"a"} + assert events.fired == [] + + +@served +async def test_a_sync_stream_failure_on_a_running_loop_fires_nothing( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents, logged_like_litellm: None +) -> None: + host.answer("openai/a", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True, sync=True)) + core: Final = make_core(**ordered()) + + stream: Final = core.completion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + with pytest.raises(litellm.RateLimitError, match="mid stream"): + collect_texts_sync(stream) + await asyncio.sleep(0.05) + await settle() + + assert events.fired == [] + + +class _RefusingCheck(CustomLogger): + async def async_pre_call_check(self, deployment: dict, parent_otel_span: object) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride, reportMissingTypeArgument, reportUnknownParameterType] # CustomLogger's own signature + raise rate_limit(message="refused") + + +@served +async def test_a_refusing_pre_call_check_fires_the_cooldown_it_sets( + make_core: MakeCore, host: ScriptedHost, events: CooldownEvents +) -> None: + host.lists["callbacks"].append(_RefusingCheck()) + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, allowed_fails=0, cooldown_time=7 + ) + + assert isinstance(await attempt(core), litellm.RateLimitError) + await settle() + + assert len(events.fired) == 1 + assert repr(events.fired[0][1:]) == repr((429, 7)) + assert events.fired[0][0] in cooled_down(core, ["a", "b"]) + + +@served +@pytest.mark.parametrize( + ("params", "cooldown_time", "expected"), + [ + pytest.param({}, 7.0, 7.0, id="router-float"), + pytest.param({}, None, DEFAULT_COOLDOWN_TIME_SECONDS, id="default"), + pytest.param({"cooldown_time": 9}, 7, 7, id="the-deployment-is-not-read"), + ], +) +async def test_a_refusing_pre_call_check_fires_the_routers_own_cooldown( + make_core: MakeCore, + host: ScriptedHost, + events: CooldownEvents, + params: Mapping[str, object], + cooldown_time: float | None, + expected: float, +) -> None: + host.lists["callbacks"].append(_RefusingCheck()) + core: Final = make_core( + model_list=[deployment("g", "a", **params), deployment("g", "b", **params)], + num_retries=0, + allowed_fails=0, + cooldown_time=cooldown_time, + ) + + assert isinstance(await attempt(core), litellm.RateLimitError) + await settle() + + assert [repr(fired[1:]) for fired in events.fired] == [repr((429, expected))] + + +@served +async def test_a_recorded_failure_fires_with_the_status_it_was_given( + make_core: MakeCore, events: CooldownEvents +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0) + + assert core.health().record_failure("a", server_error(), 500) is True + await settle() + + assert fired_exactly(events) == [repr(("a", 500, DEFAULT_COOLDOWN_TIME_SECONDS))] + + +@served +@pytest.mark.parametrize( + ("params", "cooldown_time", "error", "expected"), + [ + pytest.param({}, 7, server_error, 7, id="router-int"), + pytest.param({}, 7.0, server_error, 7.0, id="router-float"), + pytest.param({"cooldown_time": 9}, 7.0, server_error, 7.0, id="the-deployment-is-not-read"), + pytest.param({}, 7.0, lambda: rate_limited("9"), 7.0, id="the-header-is-not-read"), + ], +) +async def test_a_recorded_failure_fires_the_routers_own_cooldown( + make_core: MakeCore, + events: CooldownEvents, + params: Mapping[str, object], + cooldown_time: float, + error: Callable[[], Exception], + expected: float, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", **params), deployment("g", "b", **params)], + allowed_fails=0, + cooldown_time=cooldown_time, + ) + + assert core.health().record_failure("a", error(), 500) is True + await settle() + + assert fired_exactly(events) == [repr(("a", 500, expected))] + + +@served +@pytest.mark.parametrize( + ("model_info", "litellm_params", "cooldown_time", "error", "expected"), + [ + pytest.param({}, {}, 7, server_error, 7, id="router-int"), + pytest.param({}, {}, 7.0, server_error, 7.0, id="router-float"), + pytest.param({}, {"cooldown_time": 7}, 7.0, server_error, 7, id="params-int-over-router-float"), + pytest.param({}, {"cooldown_time": 7.0}, 7, server_error, 7.0, id="params-float-over-router-int"), + pytest.param({"cooldown_time": 7}, {"cooldown_time": 7.0}, None, server_error, 7, id="info-over-params"), + pytest.param({"cooldown_time": -1}, {"cooldown_time": 7}, 7.0, server_error, 7.0, id="a-negative-is-unset"), + pytest.param({}, {}, 9.0, lambda: rate_limited("9"), 9, id="header-int-over-router-float"), + pytest.param({}, {"cooldown_time": 9.0}, None, lambda: rate_limited("9"), 9.0, id="params-over-header"), + ], +) +async def test_a_logged_failure_fires_the_callbacks_precedence( + make_core: MakeCore, + events: CooldownEvents, + model_info: Mapping[str, object], + litellm_params: Mapping[str, object], + cooldown_time: float | None, + error: Callable[[], Exception], + expected: float, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, cooldown_time=cooldown_time + ) + logged: Final = { + "exception": error(), + "litellm_params": { + "metadata": {"model_group": "g"}, + "model_info": {"id": "a", **model_info}, + **litellm_params, + }, + } + + assert core.deployment_callback_on_failure(logged, None, 0.0, 0.0) is True # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the failure callback + await settle() + + assert [repr(fired[2]) for fired in events.fired] == [repr(expected)] + + +@served +def test_a_recorded_failure_with_no_loop_raises_and_fires_nothing(make_core: MakeCore, events: CooldownEvents) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0) + + with pytest.raises(RuntimeError, match="no running event loop"): + core.health().record_failure("a", server_error(), 500) + + assert events.fired == [] + + +class _Prometheus: + def __init__(self) -> None: + self.cooled: Final[list[Mapping[str, object]]] = [] # mutable-ok: appended as the event runs + self.outages: Final[list[Mapping[str, object]]] = [] # mutable-ok: same + + def set_deployment_complete_outage(self, **labels: object) -> None: # kwargs-ok: the metric's labels + self.outages.append(labels) + + def increment_deployment_cooled_down(self, **labels: object) -> None: # kwargs-ok: the metric's labels + self.cooled.append(labels) + + +@served +async def test_the_event_reaches_prometheus_with_the_deployments_labels( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + prometheus: Final = _Prometheus() + monkeypatch.setattr(cooldown_callbacks, "get_prometheus_logger_from_callbacks", lambda: prometheus) + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(not_found)) + + _ = await attempt(core) + await settle() + + assert [(labels["model_id"], labels["exception_status"]) for labels in prometheus.cooled] == [("a", "404")] + assert [labels["model_id"] for labels in prometheus.outages] == ["a"] + + +# --- service events ------------------------------------------------------------------------------- + + +class _ServiceEvents: + """Stands in for ``service_logger_obj``.""" + + def __init__(self) -> None: + self.seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: appended as they run + + async def async_service_success_hook(self, **event: object) -> None: # kwargs-ok: the hook's arguments + self.seen.append(event) + + def call_types(self) -> list[object]: # mutable-ok: a fresh list + return [event["call_type"] for event in self.seen] + + +def service_events(core: RouterCore) -> _ServiceEvents: + recorded: Final = _ServiceEvents() + core.service_logger_obj = recorded # pyright: ignore[reportAttributeAccessIssue] # the attribute both cores keep + return recorded + + +@served +def test_the_core_serves_its_service_logger(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + + assert isinstance(core.service_logger_obj, ServiceLogging) # pyright: ignore[reportAttributeAccessIssue] # see service_events + + +@served +async def test_a_served_acompletion_logs_the_pick_and_the_call(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + recorded: Final = service_events(core) + span: Final = object() + + _ = await attempt(core, metadata={"litellm_parent_otel_span": span}) + await settle() + + assert sorted(map(str, recorded.call_types())) == sorted([ACQUIRE, "acompletion"]) + for event in recorded.seen: + assert event["service"] is ServiceTypes.ROUTER + assert event["parent_otel_span"] is span + duration: object = event["duration"] + start_time: object = event["start_time"] + end_time: object = event["end_time"] + assert isinstance(duration, float) and isinstance(start_time, float) and isinstance(end_time, float) + assert duration == pytest.approx(end_time - start_time) + assert abs(end_time - time.time()) < 60 + + +@served +async def test_a_retry_logs_a_pick_per_attempt_and_one_call(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=1, retry_after=0) + host.answer("*", server_error(), None) + recorded: Final = service_events(core) + + _ = await attempt(core) + await settle() + + assert recorded.call_types().count(ACQUIRE) == 2 + assert recorded.call_types().count("acompletion") == 1 + + +@served +async def test_a_failed_acompletion_logs_the_pick_alone(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(**lone()) + host.answer("openai/a", fresh(not_found)) + recorded: Final = service_events(core) + + _ = await attempt(core) + await settle() + + assert recorded.call_types() == [ACQUIRE] + + +@served +async def test_a_refused_pick_logs_nothing(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + recorded: Final = service_events(core) + + assert isinstance(await attempt(core, model="nope"), litellm.BadRequestError) + await settle() + + assert recorded.call_types() == [] + + +@served +async def test_a_strategy_pick_logs_its_own_event(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, routing_strategy="least-busy" + ) + recorded: Final = service_events(core) + + _ = await attempt(core) + await settle() + + assert sorted(map(str, recorded.call_types())) == sorted([STRATEGY, ACQUIRE, "acompletion"]) + + +@served +async def test_another_endpoint_logs_only_the_strategy_pick(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, routing_strategy="least-busy" + ) + recorded: Final = service_events(core) + + _ = await core.aembedding(model="g", input=["hi"]) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + await settle() + + assert recorded.call_types() == [STRATEGY] + + +@served +async def test_a_shuffled_endpoint_logs_nothing(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + recorded: Final = service_events(core) + + _ = await core.aembedding(model="g", input=["hi"]) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + await settle() + + assert recorded.call_types() == [] + + +@served +async def test_a_sync_call_logs_nothing(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + recorded: Final = service_events(core) + + _ = attempt_sync(core) + await settle() + + assert recorded.call_types() == [] + + +@served +async def test_a_pass_through_strategy_pick_runs_on_the_perf_counter(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", use_in_pass_through=True), + deployment("g", "b", use_in_pass_through=True), + ], + num_retries=0, + routing_strategy="least-busy", + ) + recorded: Final = service_events(core) + + _ = await core.async_get_available_deployment_for_pass_through(model="g", request_kwargs={}) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the legacy member + await settle() + + assert recorded.call_types() == [STRATEGY] + start_time: object = recorded.seen[0]["start_time"] + assert isinstance(start_time, float) and abs(start_time - time.perf_counter()) < 60 + + +# --- a model that is no string -------------------------------------------------------------------- + +Endpoint = Callable[[RouterCore, object], Awaitable[object]] + + +def _call(name: str, **request: object) -> Endpoint: # kwargs-ok: the endpoint's request + def call(core: RouterCore, model: object) -> Awaitable[object]: + answer: Final[object] = getattr(core, name)(model=model, **request) # pyright: ignore[reportAny] # an endpoint + + async def awaited() -> object: + return await answer if inspect.isawaitable(answer) else answer + + return awaited() + + return call + + +ENDPOINTS: Final = { + "acompletion": _call("acompletion", messages=MESSAGES), + "atext_completion": _call("atext_completion", prompt="hi"), + "aembedding": _call("aembedding", input=["hi"]), + "aimage_generation": _call("aimage_generation", prompt="hi"), + "aresponses": _call("aresponses", input="hi"), + "aanthropic_messages": _call("aanthropic_messages", messages=MESSAGES, max_tokens=5), + "completion": _call("completion", messages=MESSAGES), + "embedding": _call("embedding", input=["hi"]), + "image_generation": _call("image_generation", prompt="hi"), +} + + +def search_error(model: object) -> str: + """What ``"/" in model`` raises (``declared_provider``).""" + with pytest.raises(TypeError) as raised: + _ = "/" in model # pyright: ignore[reportOperatorIssue] # raising is the point + return str(raised.value) + + +def hash_error(lookup: Callable[[], object]) -> str: + with pytest.raises(TypeError) as raised: + _ = lookup() + return str(raised.value) + + +SEARCH_FAILS: Final = (b"bytes", 1, 1.5, True) +ROUTES_NOWHERE: Final = (0, 0.0, False, b"", (), ("a",)) + + +@pytest.mark.parametrize("value", SEARCH_FAILS, ids=repr) +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +@served +async def test_a_model_no_search_can_read_raises_that_search(make_core: MakeCore, endpoint: str, value: object) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(TypeError, match=f"^{re.escape(search_error(value))}$"): + await ENDPOINTS[endpoint](core, value) + + +@pytest.mark.parametrize( + "mock", + [ + {"mock_testing_fallbacks": True}, + {"mock_testing_rate_limit_error": True}, + {"mock_response": "litellm.RateLimitError"}, + ], + ids=["fallbacks", "rate_limit", "response"], +) +@served +async def test_bytes_raise_before_any_mock(make_core: MakeCore, mock: Mapping[str, object]) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + await core.aimage_generation(model=b"bytes", prompt="hi", **mock) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + +@pytest.mark.parametrize("fallbacks", [None, [{"*": ["other"]}]], ids=["plain", "default_fallback"]) +@served +async def test_bytes_raise_whatever_the_fallbacks(make_core: MakeCore, fallbacks: object) -> None: + core: Final = make_core(**lone(fallbacks=fallbacks)) + + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + await core.acompletion(model=b"bytes", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + +@pytest.mark.parametrize("value", [["g"], {"a": 1}], ids=["list", "dict"]) +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +@served +async def test_an_unhashable_model_raises_as_the_alias_lookup_raises( + make_core: MakeCore, endpoint: str, value: object +) -> None: + core: Final = make_core(**lone()) + expected: Final = hash_error(lambda: {}.get(value)) # pyright: ignore[reportUnknownLambdaType, reportArgumentType, reportUnknownMemberType, reportCallIssue] # raising is the point + + with pytest.raises(TypeError, match=f"^{re.escape(expected)}$"): + await ENDPOINTS[endpoint](core, value) + + +@served +def test_a_sync_pick_reads_the_names_first(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + expected: Final = hash_error(lambda: ["g"] in set[object]()) # pyright: ignore[reportUnhashable] # raising is the point + + with pytest.raises(TypeError, match=f"^{re.escape(expected)}$"): + core.text_completion(model=["g"], prompt="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + +@pytest.mark.parametrize("value", ROUTES_NOWHERE, ids=repr) +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +@served +async def test_a_model_the_lookups_accept_is_a_name_no_deployment_serves( + make_core: MakeCore, endpoint: str, value: object +) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(litellm.BadRequestError, match=re.escape(f"You passed in model={value}. There are no healthy")): + await ENDPOINTS[endpoint](core, value) + + +@pytest.mark.parametrize("value", ROUTES_NOWHERE, ids=repr) +@served +async def test_a_mock_failure_names_the_model_as_given(make_core: MakeCore, value: object) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(litellm.InternalServerError, match=re.escape(f"This is a mock exception for model={value},")): + await core.aimage_generation(model=value, prompt="hi", mock_testing_fallbacks=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + with pytest.raises(litellm.RateLimitError, match=re.escape(f"This is a mock exception for model={value},")): + await core.acompletion(model=value, messages=MESSAGES, mock_testing_rate_limit_error=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + +@pytest.mark.parametrize("value", ROUTES_NOWHERE, ids=repr) +@served +async def test_a_default_fallback_serves_such_a_model(make_core: MakeCore, host: ScriptedHost, value: object) -> None: + core: Final = make_core(**lone(fallbacks=[{"*": ["g"]}])) + + _ = await core.acompletion(model=value, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.parametrize("value", [0, b"", ()], ids=repr) +@served +async def test_a_catch_all_wildcard_serves_such_a_model(make_core: MakeCore, host: ScriptedHost, value: object) -> None: + core: Final = make_core(model_list=[deployment("*", "w")], num_retries=0) + + _ = await core.acompletion(model=value, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + assert host.deployments_called("acompletion") == ["w"] + + +@pytest.mark.parametrize("kind", ["pick", "healthy", "pass_through"]) +@served +async def test_a_selection_raises_as_a_call_does(make_core: MakeCore, kind: str) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + await core.select(b"bytes", kind=kind) # pyright: ignore[reportArgumentType] # raising is the point + with pytest.raises(litellm.BadRequestError, match=r"You passed in model=0\. There are no healthy"): + await core.select(0, kind=kind) # pyright: ignore[reportArgumentType] # see above + + +@served +def test_the_sync_pick_and_text_completion_raise_as_a_call_does(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + core.get_available_deployment(model=b"bytes") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the legacy member + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + core.text_completion(model=b"bytes", prompt="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + with pytest.raises(litellm.BadRequestError, match=r"You passed in model=0\. There are no healthy"): + core.text_completion(model=0, prompt="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + +@served +async def test_moderation_raises_or_goes_straight_to_the_host(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(TypeError, match=r"^a bytes-like object is required, not 'str'$"): + await core.amoderation(model=b"bytes", input="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + _ = await core.amoderation(model=("a",), input="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method + + assert len(host.calls_to("amoderation")) == 1 + + +@served +async def test_a_prioritized_call_fails_building_its_queue_item(make_core: MakeCore) -> None: + core: Final = make_core(**lone()) + + with pytest.raises(pydantic.ValidationError, match="model_name"): + await core.acompletion(model=0, messages=MESSAGES, priority=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # endpoint method diff --git a/tests/unit/test_router/core/test_parity_decisions_routing.py b/tests/unit/test_router/core/test_parity_decisions_routing.py new file mode 100644 index 00000000000..7a939baa833 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_decisions_routing.py @@ -0,0 +1,235 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on routing a Decisions call: a group's +deployments whose provider has no Decisions config are dropped before any later stage, unless that +leaves none (``_without_deployments_that_cannot_serve_route``), and a Decisions or SystemOne call +carrying audio or videos skips the deployments that refuse them, unless that leaves none +(``_without_decisions_deployments_refusing_media``). + +The weights put the deployment that must be skipped first, so a core that does not narrow the +group picks it every time. +""" + +import asyncio +from collections.abc import Callable, Coroutine, Mapping +from typing import Final, cast + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.router import RouterRateLimitError +from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints +from tests.unit.test_router.core.scripted_host import auth_error, deployment + +MakeCore = Callable[..., RouterCore] + +QUESTION: Final[Mapping[str, object]] = { + "input": "Is it?", + "questions": [{"type": "predicate", "name": "is_it", "instructions": "Is it true?"}], +} +CLIP: Final = [{"url": "https://media.example/clip"}] + + +@pytest.fixture +def host(monkeypatch: pytest.MonkeyPatch) -> EndpointHost: + """``litellm.decisions`` and ``litellm.systemone`` are bound directly, so they answer from this host.""" + endpoint_host: Final = EndpointHost() + patch_direct_endpoints(monkeypatch, endpoint_host) + return endpoint_host + + +def _sync(core: object, name: str) -> Callable[..., object]: + """An endpoint both cores serve outside the ``RouterCore`` protocol.""" + return cast("Callable[..., object]", getattr(core, name)) # cast-ok: the member is a plain function + + +def _async(core: object, name: str) -> Callable[..., Coroutine[object, object, object]]: + """``_sync`` for a coroutine endpoint.""" + return cast( + "Callable[..., Coroutine[object, object, object]]", getattr(core, name) + ) # cast-ok: the member is a coroutine function + + +def _call(core: RouterCore, op: str, **request: object) -> None: # kwargs-ok: the endpoint's request + """One call on ``op``: an ``a``-prefixed op is awaited, the other one called.""" + if op.startswith("a"): + _ = asyncio.run(_async(core, op)(**request)) + else: + _ = _sync(core, op)(**request) + + +# --- deployments whose provider has no Decisions config ------------------------------------------- + +_OPENAI: Final = deployment("e", "p7-openai", model="openai/gpt-4o", weight=0) +_ANTHROPIC: Final = deployment("e", "p7-anthropic", model="anthropic/claude-sonnet-4-5-20250929", weight=1) + + +def test_adecisions_skips_a_heavier_deployment_whose_provider_has_no_decisions_config( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[_OPENAI, _ANTHROPIC], num_retries=0) + + for _ in range(5): + _call(core, "adecisions", model="e", **QUESTION) + + assert host.deployments_called("adecisions") == ["p7-openai"] * 5 + + +def test_sync_decisions_skips_a_heavier_deployment_whose_provider_has_no_decisions_config( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[_OPENAI, _ANTHROPIC], num_retries=0) + + for _ in range(5): + _call(core, "decisions", model="e", **QUESTION) + + assert host.deployments_called("decisions") == ["p7-openai"] * 5 + + +@pytest.mark.parametrize("op", ["adecisions", "decisions"]) +def test_a_group_no_provider_of_which_serves_decisions_still_picks( + make_core: MakeCore, host: EndpointHost, op: str +) -> None: + other: Final = deployment("e", "p7-anthropic-2", model="anthropic/claude-opus-4-1", weight=0) + core: Final = make_core(model_list=[_ANTHROPIC, other], num_retries=0) + + _call(core, op, model="e", **QUESTION) + + assert host.deployments_called(op) == ["p7-anthropic"] + + +@pytest.mark.parametrize("op", ["adecisions", "decisions"]) +def test_a_decisions_pick_by_id_is_not_narrowed(make_core: MakeCore, host: EndpointHost, op: str) -> None: + core: Final = make_core(model_list=[_OPENAI, _ANTHROPIC], num_retries=0) + + _call(core, op, model="p7-anthropic", **QUESTION) + + assert host.deployments_called(op) == ["p7-anthropic"] + + +@pytest.mark.parametrize("op", ["adecisions", "decisions"]) +def test_the_route_filter_runs_before_order(make_core: MakeCore, host: EndpointHost, op: str) -> None: + first: Final = deployment("e", "p7-anthropic", model="anthropic/claude-sonnet-4-5-20250929", order=1) + second: Final = deployment("e", "p7-openai", model="openai/gpt-4o", order=2) + core: Final = make_core(model_list=[first, second], num_retries=0) + + _call(core, op, model="e", **QUESTION) + + assert host.deployments_called(op) == ["p7-openai"] + + +def test_the_route_filter_runs_before_cooldown(make_core: MakeCore, host: EndpointHost) -> None: + """With the only Decisions deployment cooling down, the group has nothing to serve: the + deployment the route filter dropped is not brought back.""" + host.answer("openai/gpt-4o", auth_error()) + core: Final = make_core(model_list=[_OPENAI, _ANTHROPIC], num_retries=0) + + with pytest.raises(litellm.AuthenticationError): + _call(core, "adecisions", model="e", **QUESTION) + with pytest.raises(RouterRateLimitError, match=r"(?i)no deployments available"): + _call(core, "adecisions", model="e", **QUESTION) + + assert host.deployments_called("adecisions") == ["p7-openai"] + + +def test_the_retry_count_reads_the_group_before_the_route_filter(make_core: MakeCore, host: EndpointHost) -> None: + """An authentication error is retried only when the group holds another deployment, counted + without the route filter: both deployments count, so the Decisions deployment is retried.""" + host.answer("openai/gpt-4o", auth_error()) + core: Final = make_core(model_list=[_OPENAI, _ANTHROPIC], num_retries=2, retry_after=0, disable_cooldowns=True) + + with pytest.raises(litellm.AuthenticationError): + _call(core, "adecisions", model="e", **QUESTION) + + assert host.deployments_called("adecisions") == ["p7-openai"] * 3 + + +# --- deployments refusing a call's audio or videos ------------------------------------------------ + + +def _media_group(**changes: object) -> list[dict[str, object]]: # mutable-ok: a fresh model_list + """``refuse`` declares no audio or video input and weighs more; ``take`` declares both.""" + flags: Final = ("supports_audio_input", "supports_video_input") + return [ + deployment( + "m", + "refuse", + model="hosted_vllm/refuse-m", + api_base="https://vllm.example", + weight=1, + model_info=dict.fromkeys(flags, False), + **changes, + ), + deployment( + "m", + "take", + model="hosted_vllm/take-m", + api_base="https://vllm.example", + weight=0, + model_info=dict.fromkeys(flags, True), + **changes, + ), + ] + + +@pytest.mark.parametrize("media", ["audio", "videos"]) +@pytest.mark.parametrize("op", ["adecisions", "decisions", "asystemone", "systemone"]) +def test_a_call_carrying_media_skips_the_deployment_refusing_it( + make_core: MakeCore, host: EndpointHost, op: str, media: str +) -> None: + core: Final = make_core(model_list=_media_group(), num_retries=0) + + for _ in range(3): + _call(core, op, model="m", **{media: CLIP}, **QUESTION) + + assert host.deployments_called(op) == ["take"] * 3 + + +@pytest.mark.parametrize("op", ["adecisions", "decisions"]) +def test_a_call_without_media_keeps_the_deployment_refusing_media( + make_core: MakeCore, host: EndpointHost, op: str +) -> None: + core: Final = make_core(model_list=_media_group(), num_retries=0) + + _call(core, op, model="m", **QUESTION) + + assert host.deployments_called(op) == ["refuse"] + + +@pytest.mark.parametrize( + ("router_arguments", "request_extra"), + [({"default_litellm_params": {"drop_params": True}}, {}), ({}, {"drop_params": True})], + ids=["router-default", "request"], +) +def test_drop_params_keeps_the_deployment_refusing_media( + make_core: MakeCore, + host: EndpointHost, + router_arguments: dict[str, object], + request_extra: dict[str, object], +) -> None: + """A deployment that drops the params it cannot send takes the call, media dropped.""" + core: Final = make_core(model_list=_media_group(), num_retries=0, **router_arguments) + + _call(core, "adecisions", model="m", audio=CLIP, **request_extra, **QUESTION) + + assert host.deployments_called("adecisions") == ["refuse"] + + +def test_the_media_filter_runs_before_order(make_core: MakeCore, host: EndpointHost) -> None: + refuse, take = _media_group() + cast("dict[str, object]", refuse["litellm_params"])["order"] = 1 # cast-ok: a model_list entry + cast("dict[str, object]", take["litellm_params"])["order"] = 2 # cast-ok: a model_list entry + core: Final = make_core(model_list=[refuse, take], num_retries=0) + + _call(core, "adecisions", model="m", audio=CLIP, **QUESTION) + + assert host.deployments_called("adecisions") == ["take"] + + +def test_every_deployment_refusing_media_still_picks(make_core: MakeCore, host: EndpointHost) -> None: + refuse, _ = _media_group() + core: Final = make_core(model_list=[refuse, deployment("m", "refuse-2", model="openai/gpt-4o", weight=0)]) + + _call(core, "adecisions", model="m", videos=CLIP, **QUESTION) + + assert host.deployments_called("adecisions") == ["refuse"] diff --git a/tests/unit/test_router/core/test_parity_failure_logging.py b/tests/unit/test_router/core/test_parity_failure_logging.py new file mode 100644 index 00000000000..59173b6ad69 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_failure_logging.py @@ -0,0 +1,647 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on what a failed request leaves behind: the +failure dispatched to the request's logging object (``_async_get_available_deployment``'s ``except``) +and the walk stamped on the caller's own metadata dict, which a proxy's failure SpendLogs row reads +(``async_function_with_fallbacks``, ``async_function_with_retries``). Also the decision headers of a +routed response, typed ``model_info`` read back, ``routing_strategy_args`` as a plain dict and the +``cooldown_list`` of a no-deployments error. +""" + +import asyncio +from collections.abc import Mapping +from datetime import datetime, time, timezone +from typing import Final + +import pytest +from pydantic_core import to_jsonable_python + +import litellm +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.proxy.spend_tracking.spend_tracking_utils import ( + _get_spend_logs_metadata, # pyright: ignore[reportPrivateUsage] # what a failure row keeps +) +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import AllowedFailsPolicy, RouterRateLimitError +from tests.unit.test_router.core.fallbacks_helpers import ( + MESSAGES, + MakeCore, + complete, + complete_sync, + headers, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit, server_error + +GROUP_OF_TWO: Final = [deployment("a", "a1"), deployment("a", "a2")] +UTC_2030: Final = datetime(2030, 1, 1, tzinfo=timezone.utc) + + +class _RecordingLoggingObject: + """Stands in for a request's ``litellm_logging_obj``: records the failure dispatches.""" + + def __init__(self) -> None: + self.dispatched: list[dict[str, object]] = [] # mutable-ok: the dispatches, in order + self.messages: list[object] = [] # mutable-ok: each error's message when the dispatch ran + + async def dispatch_failure_handlers(self, **kwargs: object) -> None: # kwargs-ok: the dispatch arguments + self.dispatched.append(kwargs) + self.messages.append(getattr(kwargs.get("exception"), "message", None)) + + +class _RedisAtARoundTrip: + """The shared tier of the cooldown store, down to the one thing these tests need of it: a read + gives the event loop a turn, as a real Redis round trip does.""" + + def __init__(self) -> None: + self.batches: list[list[str]] = [] # mutable-ok: the batch reads, in order + + async def async_batch_get_cache( + self, key_list: list[str], **_: object + ) -> dict[str, object]: # mutable-ok: as RedisCache + self.batches.append(key_list) + await asyncio.sleep(0) + return dict.fromkeys(key_list) + + +class _RecordingLogging(Logging): + """A real logging object whose failure handlers only record that they ran.""" + + def __init__(self, call_type: str) -> None: + super().__init__( + model="e", + messages=[], + stream=False, + call_type=call_type, + start_time=datetime.now(tz=timezone.utc), + litellm_call_id="c", + function_id="f", + ) + self.update_environment_variables(model="e", user=None, optional_params={}, litellm_params={}) + self.failures: list[tuple[str, object]] = [] # mutable-ok: the handlers that ran, in order + + def failure_handler(self, exception: Exception, traceback_exception: str, *_: object, **__: object) -> None: + self.failures.append(("sync", exception)) + + async def async_failure_handler( + self, exception: Exception, traceback_exception: str, *_: object, **__: object + ) -> None: + self.failures.append(("async", exception)) + + +async def _let_tasks_run() -> None: + for _ in range(10): + await asyncio.sleep(0) + + +async def _failed(core: RouterCore, model: str, **request: object) -> BaseException: # kwargs-ok: request kwargs + with pytest.raises(Exception) as raised: # noqa: PT011 # any failure: each caller asserts on what it gets back + await complete(core, model, **request) + await _let_tasks_run() + return raised.value + + +# --- router-raised errors ------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_unknown_model_is_dispatched_once_with_the_error_raised(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + logging_obj: Final = _RecordingLoggingObject() + + error: Final = await _failed(core, "nope", litellm_logging_obj=logging_obj, metadata={}) + + assert isinstance(error, litellm.BadRequestError) + (dispatched,) = logging_obj.dispatched + assert dispatched["exception"] is error + assert dispatched["prefer_async_handlers"] is True + assert isinstance(dispatched["traceback_exception"], str) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("behind_redis", [True, False], ids=["redis", "memory"]) +async def test_a_refused_pick_is_logged_before_the_note_when_the_retry_read_yields( + make_core: MakeCore, host: ScriptedHost, behind_redis: bool +) -> None: + """The retry handler reads the cooldowns even for a name with no group: over Redis that hands + the loop to the failure log, which so reads the error before the no-fallback note lands on it.""" + show_router_debug(host) + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0, fallbacks=[{"a": ["a"]}]) + redis: Final = _RedisAtARoundTrip() + if behind_redis: + core.health().store.redis_cache = redis # pyright: ignore[reportAttributeAccessIssue] # a double + logging_obj: Final = _RecordingLoggingObject() + + error: Final = await _failed(core, "nope", litellm_logging_obj=logging_obj) + + note: Final = "no fallback model group was found" + assert note in str(error) + (logged,) = logging_obj.messages + assert (note in str(logged)) is not behind_redis + assert redis.batches == ([["deployment:a1:cooldown", "deployment:a2:cooldown"]] if behind_redis else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("endpoint", "handlers"), + [("aembedding", ["sync", "async"]), ("embedding", ["sync"])], + ids=["async", "sync"], +) +async def test_an_oversized_embedding_is_stamped_logged_and_counted_as_its_deployments_failure( + make_core: MakeCore, host: ScriptedHost, endpoint: str, handlers: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("e", "e1", model_info={"max_embedding_batch_size": 1})], num_retries=0 + ) + logging_obj: Final = _RecordingLogging(endpoint) + + with pytest.raises(litellm.BadRequestError) as raised: + await core.call(endpoint, {"model": "e", "input": ["a", "b"], "litellm_logging_obj": logging_obj}) + + error: Final = raised.value + assert getattr(error, "failed_deployment_id", None) == "e1" + assert getattr(error, "retry_skip_deployment_id", None) == "e1" + assert [(kind, exception is error) for kind, exception in logging_obj.failures] == [(k, True) for k in handlers] + assert logging_obj.model_call_details["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True + assert host.calls == [] + assert dict(core.fail_calls) == {"openai/e1": 1} + assert dict(core.total_calls) == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["aembedding", "embedding"]) +async def test_an_admitted_embedding_drops_the_rejection_an_earlier_attempt_left( + make_core: MakeCore, endpoint: str +) -> None: + core: Final = make_core( + model_list=[deployment("e", "e1", model_info={"max_embedding_batch_size": 2})], num_retries=0 + ) + logging_obj: Final = _RecordingLogging(endpoint) + logging_obj.model_call_details["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] = True + + await core.call(endpoint, {"model": "e", "input": ["a", "b"], "litellm_logging_obj": logging_obj}) + + assert PROXY_REJECTED_BEFORE_ROUTING_KEY not in logging_obj.model_call_details["litellm_params"] + + +@pytest.mark.asyncio +async def test_an_unknown_model_stamps_the_walk_on_the_callers_metadata(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + metadata: Final[dict[str, object]] = {"user_api_key_alias": "k"} # mutable-ok: the caller's own dict + + _ = await _failed(core, "nope", metadata=metadata) + + assert metadata["attempted_retries"] == 0 + assert metadata["max_retries"] == 0 + assert metadata["attempted_fallbacks"] == 0 + assert metadata["original_model_group"] == "nope" + assert metadata["model_group"] == "nope" + assert metadata["model_group_size"] == 0 + assert metadata["user_api_key_alias"] == "k" + + +@pytest.mark.asyncio +async def test_a_refused_pick_leaves_every_field_of_the_failure_row(make_core: MakeCore) -> None: + """A real logging object, read the way ``get_logging_payload`` reads it: the walk from the + request's metadata, the rest from the payload the dispatched failure built.""" + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + logging_obj: Final = Logging( + model="nope", + messages=MESSAGES, + stream=False, + call_type="acompletion", + start_time=datetime.now(), # noqa: DTZ005 # SDK logging uses naive local timestamps + litellm_call_id="call", + function_id="fn", + applied_guardrails=["g1"], + ) + logging_obj.update_environment_variables( + model="nope", user="", optional_params={}, litellm_params={"metadata": metadata} + ) + + _ = await _failed(core, "nope", litellm_logging_obj=logging_obj, metadata=metadata) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["status"] == "failure" + row: Final = _get_spend_logs_metadata( + metadata, + applied_guardrails=payload["metadata"].get("applied_guardrails"), + usage_object=payload["metadata"].get("usage_object"), + model_map_information=payload["model_map_information"], + ) + assert row["attempted_retries"] == 0 + assert row["max_retries"] == 0 + assert row["attempted_fallbacks"] == 0 + assert row["original_model_group"] == "nope" + assert row["applied_guardrails"] == ["g1"] + assert row["usage_object"] == payload["metadata"]["usage_object"] + assert row["model_map_information"] == payload["model_map_information"] + + +@pytest.mark.asyncio +async def test_all_cooled_deployments_are_dispatched_with_their_cooldown_details(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("a", "a1", api_base="https://a1.example.com"), + deployment("a", "a2", api_base="https://a2.example.com"), + ], + num_retries=0, + cooldown_time=60, + ) + for deployment_id in ("a1", "a2"): + core.health().record_failure(deployment_id, rate_limit(), 429) + logging_obj: Final = _RecordingLoggingObject() + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + error: Final = await _failed(core, "a", litellm_logging_obj=logging_obj, metadata=metadata) + + assert isinstance(error, RouterRateLimitError) + assert error.cooldown_list == ["a1", "a2"] + assert error.cooldown_deployments == ( + {"model_id": "a1", "api_base": "https://a1.example.com"}, + {"model_id": "a2", "api_base": "https://a2.example.com"}, + ) + (dispatched,) = logging_obj.dispatched + assert dispatched["exception"] is error + assert metadata["model_group_size"] == 2 + + +@pytest.mark.asyncio +async def test_a_refused_pick_with_retries_settles_the_retry_budget(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=3) + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + error: Final = await _failed(core, "nope", metadata=metadata) + + assert isinstance(error, litellm.BadRequestError) + # A 400 is not retried: `should_retry_this_error` raises before the budget is written again. + assert metadata["attempted_retries"] == 0 + assert metadata["max_retries"] == 3 + + +@pytest.mark.asyncio +async def test_a_strategy_with_no_async_selector_does_not_dispatch(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0, routing_strategy="usage-based-routing") + logging_obj: Final = _RecordingLoggingObject() + + _ = await _failed(core, "nope", litellm_logging_obj=logging_obj, metadata={}) + + assert logging_obj.dispatched == [] + + +def test_the_sync_path_stamps_the_walk_and_does_not_dispatch(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + logging_obj: Final = _RecordingLoggingObject() + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + with pytest.raises(litellm.BadRequestError): + complete_sync(core, "nope", litellm_logging_obj=logging_obj, metadata=metadata) + + assert logging_obj.dispatched == [] + assert metadata["attempted_retries"] == 0 + assert metadata["attempted_fallbacks"] == 0 + assert metadata["original_model_group"] == "nope" + + +@pytest.mark.asyncio +async def test_a_refused_image_pick_with_no_logging_object_builds_one_to_dispatch( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + dispatched: Final[list[BaseException]] = [] # mutable-ok: the dispatches, in order + + async def record( + self: Logging, exception: Exception, **_arguments: object + ) -> None: # kwargs-ok: dispatch arguments + dispatched.append(exception) + + monkeypatch.setattr(Logging, "dispatch_failure_handlers", record) + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + + with pytest.raises(litellm.BadRequestError) as raised: + await core.aimage_generation(prompt="a cat", model="nope") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + await _let_tasks_run() + + assert dispatched == [raised.value] + + +@pytest.mark.asyncio +async def test_a_refused_pre_call_check_is_dispatched_on_chat_only(make_core: MakeCore, host: ScriptedHost) -> None: + class Gate(CustomLogger): + async def async_pre_call_check(self, deployment: dict, parent_otel_span: object) -> dict | None: # pyright: ignore[reportIncompatibleMethodOverride] + raise rate_limit(message="gate says no") + + host.lists["callbacks"].append(Gate()) + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + logging_obj: Final = _RecordingLoggingObject() + + error: Final = await _failed(core, "a", litellm_logging_obj=logging_obj, metadata={}) + + assert isinstance(error, litellm.RateLimitError) + (dispatched,) = logging_obj.dispatched + assert dispatched["exception"] is error + + embedding_logging: Final = _RecordingLoggingObject() + with pytest.raises(litellm.RateLimitError): + await core.aembedding(model="a", input="hi", litellm_logging_obj=embedding_logging) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + await _let_tasks_run() + assert embedding_logging.dispatched == [] + + +# --- provider failures ---------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_provider_failure_stamps_each_field_and_the_router_dispatches_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", server_error()) + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=1) + logging_obj: Final = _RecordingLoggingObject() + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + error: Final = await _failed(core, "a", litellm_logging_obj=logging_obj, metadata=metadata) + + assert isinstance(error, litellm.InternalServerError) + assert metadata["attempted_retries"] == 1 + assert metadata["max_retries"] == 1 + assert metadata["attempted_fallbacks"] == 0 + assert metadata["original_model_group"] == "a" + assert metadata["model_group_size"] == 2 + assert metadata["request_retry_count"] == 2 + breadcrumbs: Final = metadata["previous_models"] + assert isinstance(breadcrumbs, tuple) + assert len(breadcrumbs) == 2 + # The SDK's own wrapper logs a provider failure, on the same logging object. + assert logging_obj.dispatched == [] + + +@pytest.mark.asyncio +async def test_a_deployments_retry_budget_replaces_the_routers_in_the_stamp( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", server_error()) + core: Final = make_core( + model_list=[deployment("a", "a1", num_retries=0), deployment("a", "a2", num_retries=0)], num_retries=2 + ) + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + _ = await _failed(core, "a", metadata=metadata) + + assert metadata["max_retries"] == 0 + assert metadata["attempted_retries"] == 0 + + +@pytest.mark.asyncio +async def test_a_fallback_hop_works_on_its_own_copy_of_the_metadata(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("*", server_error()) + core: Final = make_core( + model_list=[deployment("a", "a1"), deployment("b", "b1")], num_retries=0, fallbacks=[{"a": ["b"]}] + ) + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + _ = await _failed(core, "a", metadata=metadata) + + assert metadata["attempted_fallbacks"] == 0 + assert metadata["attempted_retries"] == 0 + assert metadata["max_retries"] == 0 + assert metadata["original_model_group"] == "a" + assert metadata["model_group"] == "a" + hop: Final = host.calls_to("acompletion")[1].kwargs["metadata"] + assert isinstance(hop, Mapping) + assert hop["attempted_fallbacks"] == 1 + assert hop["original_model_group"] == "a" + assert hop["model_group"] == "b" + + +@pytest.mark.asyncio +async def test_the_fallbacks_mock_raises_before_the_retry_loop_stamps_anything(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=2) + logging_obj: Final = _RecordingLoggingObject() + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + _ = await _failed(core, "a", mock_testing_fallbacks=True, litellm_logging_obj=logging_obj, metadata=metadata) + + assert metadata["attempted_fallbacks"] == 0 + assert metadata["original_model_group"] == "a" + assert "attempted_retries" not in metadata + assert "max_retries" not in metadata + assert logging_obj.dispatched == [] + + +@pytest.mark.asyncio +async def test_the_rate_limit_mock_raises_inside_the_retry_loop(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + logging_obj: Final = _RecordingLoggingObject() + metadata: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + _ = await _failed(core, "a", mock_testing_rate_limit_error=True, litellm_logging_obj=logging_obj, metadata=metadata) + + assert metadata["attempted_retries"] == 0 + assert metadata["max_retries"] == 0 + assert metadata["model_group_size"] == 2 + assert logging_obj.dispatched == [] + + +@pytest.mark.asyncio +async def test_a_generic_endpoint_stamps_its_bucket_and_clears_the_siblings_walk( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", server_error()) + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + sibling: Final[dict[str, object]] = { # mutable-ok: the caller's own dict + "attempted_fallbacks": 7, + "original_model_group": "stale", + "kept": True, + } + bucket: Final[dict[str, object]] = {} # mutable-ok: the caller's own dict + + with pytest.raises(litellm.InternalServerError): + await core.aresponses(model="a", input="hi", metadata=sibling, litellm_metadata=bucket) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + assert "attempted_fallbacks" not in sibling + assert "original_model_group" not in sibling + assert sibling["kept"] is True + assert bucket["attempted_fallbacks"] == 0 + assert bucket["original_model_group"] == "a" + assert bucket["attempted_retries"] == 0 + assert bucket["max_retries"] == 0 + + +# --- R7: the decision headers --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_stamped_quality_decision_reaches_the_response_headers(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), num_retries=0) + decision: Final = {"routed_model": "t1", "quality_tier": 2, "routed_via": "default", "matched_keyword": None} + + response: Final = await complete(core, "a", metadata={"quality_router_decision": decision}) + + found: Final = headers(response) + assert found["x-litellm-quality-router-model"] == "t1" + assert found["x-litellm-quality-router-tier"] == "2" + assert found["x-litellm-quality-router-via"] == "default" + assert "x-litellm-quality-router-keyword" not in found + + +@pytest.mark.asyncio +async def test_a_quality_router_stamps_its_decision_headers(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment( + "t1", + "t1-21", + model="anthropic/claude-3-5-sonnet-20240620", + model_info={"litellm_routing_preferences": {"quality_tier": 1}}, + ), + deployment("qr", "qr0", model="auto_router/quality_router", quality_router_default_model="t1"), + ], + num_retries=0, + ) + + response: Final = await complete(core, "qr") + + assert headers(response)["x-litellm-quality-router-model"] == "t1" + + +# --- P9, N9: typed model_info --------------------------------------------------------------------- + + +def _typed_model_list() -> list[dict[str, object]]: # mutable-ok: a model_list + return [ + deployment( + "p", + "p1", + model="azure/gpt-4o", + api_base="https://example.openai.azure.com", + model_info={ + "ptu_shares": {"a": 2}, + "ptu_count": 2, + "cost_per_ptu_per_hour": 2, + "ptu_effective_from": "2026-01-01T00:00:00Z", + }, + ), + deployment( + "h", + "w3", + model_info={ + "access_windows": [ + {"start": "03:00:00", "end": "03:00:01", "timezone": "Asia/Tokyo", "team_ids": ["t1"]} + ] + }, + ), + deployment("d", "db1", model_info={"created_at": UTC_2030, "updated_at": UTC_2030, "db_model": True}), + ] + + +def _model_info(core: RouterCore, deployment_id: str) -> Mapping[str, object]: + for entry in core.registry().all(): + info = entry["model_info"] + assert isinstance(info, Mapping) + if info["id"] == deployment_id: + return info + raise AssertionError(f"no deployment {deployment_id}") + + +def test_model_info_reads_back_a_datetime(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_typed_model_list()) + effective_from: Final = _model_info(core, "p1")["ptu_effective_from"] + assert isinstance(effective_from, datetime) + assert effective_from == datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def test_model_info_reads_back_access_windows_typed(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_typed_model_list()) + windows: Final = _model_info(core, "w3")["access_windows"] + assert isinstance(windows, tuple) + (window,) = windows + assert window["start"] == time(3) + assert window["end"] == time(3, 0, 1) + assert window["team_ids"] == ("t1",) + + +def test_db_timestamps_read_back_as_the_datetimes_stamped(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_typed_model_list()) + info: Final = _model_info(core, "db1") + assert info["created_at"] == UTC_2030 + assert info["updated_at"] == UTC_2030 + created_at: Final = info["created_at"] + assert isinstance(created_at, datetime) + assert created_at.isoformat() == "2030-01-01T00:00:00+00:00" + + +@pytest.mark.asyncio +async def test_a_picked_deployment_carries_its_typed_model_info(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_typed_model_list()) + picked: Final = await core.select("d") + assert isinstance(picked, Mapping) + model_info: Final = picked["model_info"] + assert model_info["created_at"] == UTC_2030 + + +# --- PV1: routing_strategy_args ------------------------------------------------------------------- + + +def test_routing_strategy_args_serialize_as_a_plain_dict(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO), routing_strategy_args={"ttl": 10}) + args: Final = core.routing_strategy_args # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # the deprecated attribute + assert isinstance(args, dict) + assert to_jsonable_python(args) == {"ttl": 10} + + +def test_unset_routing_strategy_args_serialize_too(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_OF_TWO)) + assert to_jsonable_python(core.routing_strategy_args) == {} # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the deprecated attribute + + +# --- U13: the cooldown list ----------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_routing_group_with_nothing_left_lists_the_cooling_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/rc1", rate_limit()) + core: Final = make_core( + model_list=[deployment("g1", "rc1"), deployment("g2", "rc2")], + num_retries=0, + routing_groups=[ + { + "group_name": "pg", + "models": ["g1", "g2"], + "routing_strategy": "priority", + "model_priorities": {"g1": 1, "g2": 2}, + } + ], + ) + _ = await complete(core, "pg") + core.apply(ChangeSet(delete=("rc2",))) + + error: Final = await _failed(core, "pg") + + assert isinstance(error, RouterRateLimitError) + assert error.cooldown_list == ["rc1"] + + +@pytest.mark.asyncio +async def test_the_cooldown_list_names_every_cooling_deployment_of_the_router(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("a", "a1"), deployment("b", "b1")], + num_retries=0, + cooldown_time=60, + allowed_fails_policy=AllowedFailsPolicy( + RateLimitErrorAllowedFails=0 + ), # a lone deployment stays warm without it + ) + for deployment_id in ("a1", "b1"): + core.health().record_failure(deployment_id, rate_limit(), 429) + + error: Final = await _failed(core, "a") + + assert isinstance(error, RouterRateLimitError) + assert error.cooldown_list == ["a1", "b1"] + assert error.all_deployments_in_cooldown is True + assert error.cooldown_deployments == ({"model_id": "a1", "api_base": None},) diff --git a/tests/unit/test_router/core/test_parity_malformed_after_rate_limit.py b/tests/unit/test_router/core/test_parity_malformed_after_rate_limit.py new file mode 100644 index 00000000000..7181beaa39a --- /dev/null +++ b/tests/unit/test_router/core/test_parity_malformed_after_rate_limit.py @@ -0,0 +1,86 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on a retry whose answer fails conversion +after a rate limit (fuzz case ff68399265ba9aee, ruled 2026-11-05: a 500 on both cores). + +The Python core runs a retry inside the ``except`` of the attempt before it, so the retry's +conversion error is raised while the first attempt's RateLimitError is being handled. The +"Invalid response object" message once printed that whole chain, and exception mapping read the +rate-limit phrase in it and answered 429. The retry here runs the real conversion and the real +mapping, as ``OpenAIChatCompletion.acompletion`` and ``atext_completion`` do, at the moment the core +calls the deployment, so the error being handled then is whatever the core leaves open. +""" + +from collections.abc import Awaitable, Callable +from typing import Final, cast + +import pytest + +import litellm +from litellm.litellm_core_utils.exception_mapping_utils import exception_type # pyright: ignore[reportUnknownVariableType] # its untyped parameters +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import convert_to_model_response_object # pyright: ignore[reportUnknownVariableType] # its untyped start_time, end_time and dict parameters +from litellm.llms.openai.common_utils import OpenAIError +from litellm.router_core import RouterCore +from litellm.types.utils import ModelResponse +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit + +MakeCore = Callable[..., RouterCore] + +# A text_completion body handed to the chat conversion (gpt-4o is a chat model): no ``message``. +_TEXT_COMPLETION_BODY: Final = { + "id": "cmpl-b6", + "object": "text_completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"text": "ok", "index": 0, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _malformed_answer(**_request: object) -> BaseException: # kwargs-ok: the scripted call's request + """The error the OpenAI handler and ``atext_completion`` raise for ``_TEXT_COMPLETION_BODY``: + the conversion fails, the handler wraps it in a status-500 ``OpenAIError``, and ``exception_type`` + maps that. Returned, so the host raises and logs it as any scripted error.""" + try: + _ = convert_to_model_response_object( + response_object=dict(_TEXT_COMPLETION_BODY), model_response_object=ModelResponse() + ) + except Exception as conversion_error: # noqa: BLE001 # the handler wraps every conversion failure + try: + raise exception_type( # exception_type raises the mapped error itself + model="gpt-4o", + original_exception=OpenAIError(status_code=500, message=str(conversion_error)), + custom_llm_provider="openai", + ) + except Exception as mapped: # noqa: BLE001 # the mapped error is the scripted answer + return mapped + raise AssertionError("the malformed body converted") + + +def _atext_completion(core: RouterCore) -> Callable[..., Awaitable[object]]: + """``atext_completion``, which both cores serve outside the ``RouterCore`` protocol.""" + return cast( + "Callable[..., Awaitable[object]]", getattr(core, "atext_completion") + ) # cast-ok: the member is a coroutine function + + +@pytest.mark.asyncio +async def test_a_retry_whose_answer_fails_conversion_after_a_rate_limit_is_a_500( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "b6-d1", weight=1), deployment("g", "b6-d2", weight=1)], + num_retries=1, + retry_after=0, + ) + host.answer("*", rate_limit(message="rate limit exceeded"), _malformed_answer) + + with pytest.raises(litellm.InternalServerError) as raised: + _ = await _atext_completion(core)(model="g", prompt="hi") + + error: Final = raised.value + assert error.status_code == 500 + assert "Invalid response object" in str(error) + assert "KeyError: 'message'" in str(error) + assert "rate limit exceeded" not in str(error) + assert len(host.calls_to("atext_completion")) == 2 + assert type(host.raised[0]) is litellm.RateLimitError diff --git a/tests/unit/test_router/core/test_parity_pick_count_and_fallback_log.py b/tests/unit/test_router/core/test_parity_pick_count_and_fallback_log.py new file mode 100644 index 00000000000..01b1c1fe0e1 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_pick_count_and_fallback_log.py @@ -0,0 +1,316 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on two traces a call leaves on the proxy: +the per-deployment pick count in the router cache (``_track_deployment_metrics``, ``_update_usage``) +and the error log of a level whose fallbacks served nothing (the ``except`` around the regular +fallbacks of ``async_function_with_fallbacks_common_utils``). +""" + +import logging +from collections.abc import Generator +from typing import Final + +import openai +import pytest + +import litellm +from litellm._logging import verbose_router_logger +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from tests.unit.test_router.core.fallbacks_helpers import ( + TIERED_A, + MakeCore, + build, + complete, + complete_sync, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + content_policy, + deployment, + embedding_response, + rate_limit, + server_error, +) + +LOG_HEAD: Final = "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - " +NO_COOLDOWNS: Final = "\nDebug Information:\nCooldown Deployments=[]" + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(logging.ERROR) + self.messages: Final[list[str]] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def router_errors() -> Generator[list[str], None, None]: + """The router logger's error messages, in order.""" + records: Final = _Records() + verbose_router_logger.addHandler(records) + yield records.messages + verbose_router_logger.removeHandler(records) + + +def _picks(core: RouterCore, deployment_id: object) -> object: + return core.health().store.in_memory_cache.get_cache(deployment_id) + + +def _fallback_logs(messages: list[str]) -> list[str]: # mutable-ok: a fresh list + return [message for message in messages if message.startswith(LOG_HEAD)] + + +async def _fails(core: RouterCore, model: str = "a", **request: object) -> BaseException: # kwargs-ok: request kwargs + with pytest.raises(openai.APIStatusError) as raised: + await complete(core, model, **request) + return raised.value + + +# --- the pick count ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_each_async_chat_pick_counts_in_the_router_cache(make_core: MakeCore) -> None: + core: Final = build(make_core) + + _ = await complete(core) + _ = await complete(core) + + assert _picks(core, "a") == 2 + + +@pytest.mark.asyncio +async def test_a_pick_counts_before_its_attempt_fails_and_a_retry_picks_again( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, num_retries=2, retry_after=0) + + _ = await _fails(core) + + assert _picks(core, "a") == 3 + + +@pytest.mark.asyncio +async def test_every_hop_of_a_fallback_chain_counts_its_pick(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + + _ = await complete(core) + + assert _picks(core, "a") == 1 + assert _picks(core, "b") == 1 + assert _picks(core, "c") is None + + +def test_a_sync_chat_pick_does_not_count(make_core: MakeCore) -> None: + core: Final = build(make_core) + + _ = complete_sync(core) + + assert _picks(core, "a") is None + + +@pytest.mark.asyncio +async def test_an_embedding_pick_does_not_count(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", embedding_response()) + core: Final = build(make_core) + + _ = await core.aembedding(model="a", input="hi") # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + assert _picks(core, "a") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("assigned", "counted"), + [("fallback-id", "fallback-id"), (7, 7), (None, None)], + ids=["string-id", "other-id", "no-id"], +) +async def test_a_default_deployment_pick_counts_under_the_id_it_holds( + make_core: MakeCore, assigned: object, counted: object +) -> None: + core: Final = build(make_core) + served: Final = deployment("*", "x", model="openai/x") + _ = core.apply(ChangeSet(assign={"default_deployment": {**served, "model_info": {"id": assigned}}})) + + _ = await complete(core, "not-a-group") + + if counted is not None: + assert _picks(core, counted) == 1 + assert _picks(core, "x") is None + + +# --- the fallback failure log --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_chain_that_fails_logs_every_level_innermost_first( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + for name in ("a", "b", "c"): + host.answer(f"openai/{name}", rate_limit(message=f"{name} down")) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}], disable_cooldowns=True) + + _ = await _fails(core) + + # Each level logs what its `try` raised: `c` has no chain, so its own error, and `b`'s hop to it + # raised that; `a`'s hop to `b` raised `b`'s own error. + logged: Final = _fallback_logs(router_errors) + assert ["c down" in logged[0], "c down" in logged[1], "b down" in logged[2], len(logged)] == [True, True, True, 3] + assert all(entry.endswith(NO_COOLDOWNS) for entry in logged) + + +@pytest.mark.asyncio +async def test_a_failed_hop_is_logged_with_the_hops_error_not_the_levels( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a", server_error()) + host.answer("openai/b", auth_error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], disable_cooldowns=True) + + error: Final = await _fails(core) + + assert type(error).__name__ == "InternalServerError" + logged: Final = _fallback_logs(router_errors) + assert len(logged) == 2 + assert all("AuthenticationError" in entry for entry in logged) + + +@pytest.mark.asyncio +async def test_no_fallback_list_logs_nothing(make_core: MakeCore, host: ScriptedHost, router_errors: list[str]) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core) + + _ = await _fails(core) + + assert _fallback_logs(router_errors) == [] + + +@pytest.mark.asyncio +async def test_a_fallback_list_that_does_not_name_the_group_logs_the_levels_error( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"b": ["c"]}], disable_cooldowns=True) + + error: Final = await _fails(core) + + assert _fallback_logs(router_errors) == [f"{LOG_HEAD}{error}{NO_COOLDOWNS}"] + + +@pytest.mark.asyncio +async def test_the_missing_group_explanation_is_in_the_log_and_on_the_error_once( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + show_router_debug(host) + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"b": ["c"]}], disable_cooldowns=True) + + error: Final = await _fails(core) + + assert str(error).count("no fallback model group was found") == 1 + assert _fallback_logs(router_errors) == [f"{LOG_HEAD}{error}{NO_COOLDOWNS}"] + + +@pytest.mark.asyncio +async def test_the_outcome_message_of_a_failed_chain_stays_out_of_the_log( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + show_router_debug(host) + host.answer("openai/a", server_error()) + host.answer("openai/b", server_error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], disable_cooldowns=True) + + error: Final = await _fails(core) + + logged: Final = _fallback_logs(router_errors) + assert len(logged) == 2 + assert "Fallback to b also failed" in str(error) + assert not any("also failed" in entry for entry in logged) + + +@pytest.mark.asyncio +async def test_a_content_policy_list_without_the_group_logs_before_its_outcome_message( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + show_router_debug(host) + host.answer("openai/a", content_policy()) + core: Final = build(make_core, content_policy_fallbacks=[{"b": ["c"]}], disable_cooldowns=True) + + error: Final = await _fails(core) + + (logged,) = _fallback_logs(router_errors) + assert logged.endswith(NO_COOLDOWNS) + assert len(logged) < len(f"{LOG_HEAD}{error}{NO_COOLDOWNS}") + + +@pytest.mark.asyncio +async def test_max_fallbacks_of_zero_logs_the_levels_error( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], max_fallbacks=0, disable_cooldowns=True) + + error: Final = await _fails(core) + + assert _fallback_logs(router_errors) == [f"{LOG_HEAD}{error}{NO_COOLDOWNS}"] + + +@pytest.mark.asyncio +async def test_the_order_path_logs_only_where_its_last_tier_reaches_the_regular_fallbacks( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a1", server_error()) + host.answer("openai/a2", server_error()) + core: Final = make_core(model_list=list(TIERED_A), num_retries=0, fallbacks=[{"b": ["c"]}], disable_cooldowns=True) + + _ = await _fails(core) + + assert len(_fallback_logs(router_errors)) == 1 + + +@pytest.mark.asyncio +async def test_disabled_fallbacks_log_nothing( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + + _ = await _fails(core, disable_fallbacks=True) + + assert _fallback_logs(router_errors) == [] + + +def test_a_sync_call_logs_through_the_same_handling( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a", server_error()) + core: Final = build(make_core, fallbacks=[{"b": ["c"]}], disable_cooldowns=True) + + with pytest.raises(litellm.InternalServerError) as raised: + _ = complete_sync(core) + + assert _fallback_logs(router_errors) == [f"{LOG_HEAD}{raised.value}{NO_COOLDOWNS}"] + + +@pytest.mark.asyncio +async def test_the_log_lists_the_deployments_cooling_down( + make_core: MakeCore, host: ScriptedHost, router_errors: list[str] +) -> None: + host.answer("openai/a1", auth_error()) + host.answer("openai/a2", auth_error()) + core: Final = make_core( + model_list=[deployment("a", "a1"), deployment("a", "a2"), deployment("b", "b")], + num_retries=0, + fallbacks=[{"b": ["c"]}], + ) + + _ = await _fails(core) + + (logged,) = _fallback_logs(router_errors) + assert "Cooldown Deployments=[('a" in logged diff --git a/tests/unit/test_router/core/test_parity_provider_scoped_fallbacks.py b/tests/unit/test_router/core/test_parity_provider_scoped_fallbacks.py new file mode 100644 index 00000000000..5bd48bfa4ea --- /dev/null +++ b/tests/unit/test_router/core/test_parity_provider_scoped_fallbacks.py @@ -0,0 +1,182 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on fallbacks for a request that creates or +names a resource only one provider holds (``run_async_fallback``'s ``same_model_group_only``): a file +upload, a batch on an uploaded file, a batch cancel and the generic file and fine-tuning ops fall +back within their own model group only, while the same op naming no such resource still falls back +to another group. +""" + +import io +from collections.abc import Awaitable, Callable +from typing import Final, cast + +import pytest + +from litellm.router_core import RouterCore +from litellm.router_python_host import Endpoint +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.scripted_host import deployment, server_error + +MakeCore = Callable[..., RouterCore] + +_BATCH: Final = {"endpoint": "/v1/chat/completions", "completion_window": "24h"} +_TWO_GROUPS: Final = [deployment("a", "a1"), deployment("b", "b1")] +_TO_B: Final = [{"a": ["b"]}] + + +class _NamedHost(EndpointHost): + """``EndpointHost`` whose endpoints carry the op's name, as litellm's own functions do: the Python + core reads a generic op's handler name to decide it names a provider-scoped resource.""" + + __slots__ = () + + def invoke(self, op: str) -> Endpoint: + endpoint: Final = super().invoke(op) + endpoint.__name__ = op.rsplit(".", 1)[-1] + return endpoint + + +@pytest.fixture +def host() -> _NamedHost: + return _NamedHost() + + +def _async(core: object, name: str) -> Callable[..., Awaitable[object]]: + """An endpoint both cores serve outside the ``RouterCore`` protocol.""" + return cast("Callable[..., Awaitable[object]]", getattr(core, name)) # cast-ok: the member is a coroutine function + + +def _upload() -> dict[str, object]: # mutable-ok: a fresh request + return {"file": io.BytesIO(b'{"custom_id": "1", "body": {"model": "a", "messages": []}}\n'), "purpose": "batch"} + + +def _failing_a( + make_core: MakeCore, host: EndpointHost, model_list: list[dict[str, object]] | None = None +) -> RouterCore: + host.answer("openai/a1", server_error()) + return make_core(model_list=list(model_list or _TWO_GROUPS), num_retries=0) + + +# --- no fallback to another group ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_file_upload_does_not_fall_back_to_another_group(make_core: MakeCore, host: _NamedHost) -> None: + core: Final = _failing_a(make_core, host) + + with pytest.raises(type(server_error())): + await _async(core, "acreate_file")(model="a", fallbacks=_TO_B, **_upload()) + + assert host.deployments_called("acreate_file") == ["a1"] + + +@pytest.mark.asyncio +async def test_a_batch_on_an_uploaded_file_does_not_fall_back_to_another_group( + make_core: MakeCore, host: _NamedHost +) -> None: + core: Final = _failing_a(make_core, host) + + with pytest.raises(type(server_error())): + await _async(core, "acreate_batch")(model="a", input_file_id="file-1", fallbacks=_TO_B, **_BATCH) + + assert host.deployments_called("acreate_batch") == ["a1"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("op", "request_fields"), + [ + ("acancel_batch", {"batch_id": "batch-1"}), + ("afile_content", {"file_id": "file-1"}), + ("afile_delete", {"file_id": "file-1"}), + ("acreate_fine_tuning_job", {"training_file": "file-1"}), + ("acancel_fine_tuning_job", {"fine_tuning_job_id": "ftjob-1"}), + ], + ids=["cancel-batch", "file-content", "file-delete", "create-fine-tuning-job", "cancel-fine-tuning-job"], +) +async def test_an_op_naming_a_provider_resource_does_not_fall_back_to_another_group( + make_core: MakeCore, host: _NamedHost, op: str, request_fields: dict[str, str] +) -> None: + core: Final = _failing_a(make_core, host) + + with pytest.raises(type(server_error())): + await _async(core, op)(model="a", fallbacks=_TO_B, **request_fields) + + assert host.deployments_called(op) == ["a1"] + + +@pytest.mark.asyncio +async def test_a_dict_fallback_without_a_model_is_skipped_for_a_scoped_batch( + make_core: MakeCore, host: _NamedHost +) -> None: + core: Final = _failing_a(make_core, host) + + with pytest.raises(type(server_error())): + await _async(core, "acreate_batch")( + model="a", input_file_id="file-1", fallbacks=[{"a": [{"user": "u"}]}], **_BATCH + ) + + assert host.deployments_called("acreate_batch") == ["a1"] + + +# --- what still falls back ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_batch_without_an_input_file_still_falls_back_to_another_group( + make_core: MakeCore, host: _NamedHost +) -> None: + core: Final = _failing_a(make_core, host) + + await _async(core, "acreate_batch")(model="a", fallbacks=_TO_B, **_BATCH) + + assert host.deployments_called("acreate_batch") == ["a1", "b1"] + + +@pytest.mark.asyncio +async def test_a_file_op_naming_no_file_still_falls_back_to_another_group( + make_core: MakeCore, host: _NamedHost +) -> None: + core: Final = _failing_a(make_core, host) + + await _async(core, "afile_content")(model="a", fallbacks=_TO_B) + + assert host.deployments_called("afile_content") == ["a1", "b1"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("op", ["acreate_file", "acreate_batch"], ids=["file-upload", "batch-on-file"]) +async def test_a_scoped_request_still_falls_back_within_its_own_group( + make_core: MakeCore, host: _NamedHost, op: str +) -> None: + core: Final = _failing_a(make_core, host) + request: Final = _upload() if op == "acreate_file" else {"input_file_id": "file-1", **_BATCH} + # A dict target naming the failed group with a param of its own is a distinct target, not the failed one. + same_group: Final = [{"a": [{"model": "a", "user": "u"}, "b"]}] + + with pytest.raises(type(server_error())): + await _async(core, op)(model="a", fallbacks=same_group, **request) + + assert host.deployments_called(op) == ["a1", "a1"] + + +@pytest.mark.asyncio +async def test_a_batch_on_an_uploaded_file_still_falls_back_to_the_next_order_level( + make_core: MakeCore, host: _NamedHost +) -> None: + tiered: Final = [deployment("a", "a1", order=1), deployment("a", "a2", order=2), deployment("b", "b1")] + core: Final = _failing_a(make_core, host, tiered) + + await _async(core, "acreate_batch")(model="a", input_file_id="file-1", fallbacks=_TO_B, **_BATCH) + + assert host.deployments_called("acreate_batch") == ["a1", "a2"] + + +@pytest.mark.asyncio +async def test_a_file_upload_moves_to_the_next_order_level(make_core: MakeCore, host: _NamedHost) -> None: + tiered: Final = [deployment("a", "a1", order=1), deployment("a", "a2", order=2)] + core: Final = _failing_a(make_core, host, tiered) + + await _async(core, "acreate_file")(model="a", **_upload()) + + assert host.deployments_called("acreate_file") == ["a1", "a2"] diff --git a/tests/unit/test_router/core/test_parity_regress5.py b/tests/unit/test_router/core/test_parity_regress5.py new file mode 100644 index 00000000000..3337b7abdf6 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_regress5.py @@ -0,0 +1,482 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the selection causes of the regress5 pack: +LAR-1 set after build, endpoint eligibility, web search, ``order`` in ``model_info``, ``priority`` +through an alias, old-style custom strategies, wildcard filling and cache flags, ``specific_deployment`` +read as a strict True, pass-through picks, affinity narrowing and a health scope given as a dict. +""" + +import time +from collections.abc import Mapping, Sequence +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import EndpointNotSupportedByModelError +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import CustomRoutingStrategyBase, RouterRateLimitError +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, ids_of, picked_ids + +SPREAD_PICKS: Final = 40 +WEB_SEARCH_TOOLS: Final = [{"type": "web_search"}] +EMBEDDINGS_ONLY: Final = {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]} + + +def _embedding_group() -> Sequence[Mapping[str, object]]: + return ( + deployment("g", "embonly", weight=1, model_info=dict(EMBEDDINGS_ONLY)), + deployment("g", "chat", weight=0), + ) + + +def _web_group() -> Sequence[Mapping[str, object]]: + return ( + deployment("g", "noweb", weight=1, model_info={"supports_web_search": False}), + deployment("g", "web", weight=0), + ) + + +# --- V1: lar1 set after build --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lar1_set_after_build_serves_the_next_async_call(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("a", "a1")], num_retries=0) + core.apply(ChangeSet(settings={"routing_strategy": "lar1"})) + await core.acompletion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["a1"] + assert core.settings().routing_strategy == "lar1" + + +@pytest.mark.asyncio +async def test_lar1_set_after_build_picks_without_calling(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("a", "a1"), deployment("a", "a2")], num_retries=0) + core.apply(ChangeSet(settings={"routing_strategy": "lar1"})) + assert await picked_ids(core, "a", times=SPREAD_PICKS, request_kwargs={}) <= {"a1", "a2"} + + +def test_lar1_set_after_build_refuses_the_sync_path(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("a", "a1")], num_retries=0) + core.apply(ChangeSet(settings={"routing_strategy": "lar1"})) + with pytest.raises(NotImplementedError, match="LAR-1 routing only supports async routing"): + core.completion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +@pytest.mark.asyncio +async def test_leaving_lar1_goes_back_to_the_router_strategy(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("a", "a1")], num_retries=0) + core.apply(ChangeSet(settings={"routing_strategy": "lar1"})) + core.apply(ChangeSet(settings={"routing_strategy": "simple-shuffle"})) + core.completion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("completion") == ["a1"] + + +# --- R4: endpoint eligibility --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_acompletion_skips_a_deployment_that_enforces_other_endpoints( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["chat"] + + +@pytest.mark.asyncio +async def test_atext_completion_skips_a_deployment_that_enforces_other_endpoints( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + await core.atext_completion(model="g", prompt="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("atext_completion") == ["chat"] + + +def test_sync_text_completion_skips_a_deployment_that_enforces_other_endpoints( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + core.text_completion(model="g", prompt="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + # The sync text path sends the group as ``model`` and no ``model_info``: the params name the deployment. + assert [call.kwargs["weight"] for call in host.calls_to("text_completion")] == [0] + + +def test_sync_completion_skips_a_deployment_that_enforces_other_endpoints( + make_core: MakeCore, host: ScriptedHost +) -> None: + """``function_with_fallbacks`` is the async walk, whose ``endpoint_selection`` covers the sync attempt.""" + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("completion") == ["chat"] + + +@pytest.mark.asyncio +async def test_an_endpoint_outside_the_chat_table_keeps_every_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + await core.aembedding(model="g", input="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("aembedding") == ["embonly"] + + +@pytest.mark.asyncio +async def test_a_group_that_serves_no_chat_endpoint_names_what_it_serves(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "e1", model_info=dict(EMBEDDINGS_ONLY))], + num_retries=0, + ) + with pytest.raises(EndpointNotSupportedByModelError) as refused: + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert ( + refused.value.message + == "/v1/chat/completions is not supported by model=g. Supported endpoints: /v1/embeddings" + ) + + +@pytest.mark.asyncio +async def test_the_refusal_falls_back_like_any_bad_request_naming_no_provider( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "e1", model_info=dict(EMBEDDINGS_ONLY)), deployment("h", "h1")], + num_retries=0, + fallbacks=[{"g": ["h"]}], + ) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["h1"] + + +@pytest.mark.asyncio +async def test_a_direct_pick_runs_no_endpoint_selection(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "e1", model_info=dict(EMBEDDINGS_ONLY))], num_retries=0) + assert await picked_ids(core, "g", times=1) == {"e1"} + + +def test_routable_deployments_read_the_call_type(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(_embedding_group()), num_retries=0) + routable: Final = core.registry().deployments("g", source="routable", call_type="completion") + assert ids_of(list(routable)) == ["chat"] + + +# --- R5: web search ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_web_search_request_skips_deployments_that_declare_no_support( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(_web_group()), num_retries=0) + await core.acompletion(model="g", messages=MESSAGES, tools=WEB_SEARCH_TOOLS) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["web"] + + +def test_the_sync_path_runs_no_web_search_filter(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=list(_web_group()), num_retries=0) + core.completion(model="g", messages=MESSAGES, tools=WEB_SEARCH_TOOLS) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("completion") == ["noweb"] + + +@pytest.mark.asyncio +async def test_no_deployment_with_web_search_is_a_no_deployments_error(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "noweb", model_info={"supports_web_search": False})], + num_retries=0, + ) + with pytest.raises(RouterRateLimitError): + await core.acompletion(model="g", messages=MESSAGES, tools=WEB_SEARCH_TOOLS) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +@pytest.mark.asyncio +async def test_the_healthy_listing_runs_the_web_search_filter(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(_web_group()), num_retries=0) + listed: Final = await core.select("g", {"tools": WEB_SEARCH_TOOLS}, kind="healthy") # pyright: ignore[reportCallIssue, reportArgumentType] # PythonCore's select takes keyword options + assert ids_of(listed) == ["web"] + + +# --- R6: order ------------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first", "second"), + [ + ({"model_info": {"order": 1}}, {"model_info": {"order": 2}}), + ({"order": "1"}, {"order": "2"}), + ({"model_info": {"order": "1"}}, {"model_info": {"order": "2"}}), + ({"order": None, "model_info": {"order": 1}}, {"model_info": {"order": 2}}), + ], + ids=["model-info", "numeric-string", "model-info-numeric-string", "null-param-reads-model-info"], +) +async def test_order_is_read_where_get_deployment_order_reads_it( + make_core: MakeCore, first: dict[str, object], second: dict[str, object] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "first", weight=0, **first), deployment("g", "second", weight=1, **second)], + num_retries=0, + ) + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"first"} + + +# --- R10: priority through an alias --------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_priority_through_an_alias_reads_the_target_group(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", drop_params=True)], + num_retries=0, + model_group_alias={"al": "g"}, + ) + await core.acompletion(model="al", messages=MESSAGES, priority="high") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["d1"] + + +# --- R11: old-style custom strategies ------------------------------------------------------------- + + +class _OldStyleStrategy(CustomRoutingStrategyBase): + """Written before picks took ``request_kwargs``.""" + + def __init__(self, picked: Mapping[str, object]) -> None: + self.picked: Final = picked + + async def async_get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] # the old signature is the point + self, model: str, messages: object = None, input: object = None, specific_deployment: bool | None = False + ) -> Mapping[str, object]: + return self.picked + + def get_available_deployment( # pyright: ignore[reportIncompatibleMethodOverride] # the old signature is the point + self, model: str, messages: object = None, input: object = None, specific_deployment: bool | None = False + ) -> Mapping[str, object]: + return self.picked + + +def _with_old_style_strategy(make_core: MakeCore) -> RouterCore: + picked: Final = deployment("b", "db") + core: Final = make_core(model_list=[deployment("a", "da"), picked], num_retries=0) + core.register_plugin(_OldStyleStrategy(picked)) + return core + + +@pytest.mark.asyncio +async def test_an_old_style_strategy_answers_a_pick_that_names_only_the_model(make_core: MakeCore) -> None: + core: Final = _with_old_style_strategy(make_core) + picked: Final = await core._core_async_get_available_deployment("b") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportPrivateUsage] # the member the deprecated pick returns + assert ids_of(picked) == ["db"] + + +def test_an_old_style_strategy_answers_a_sync_pick_that_names_only_the_model(make_core: MakeCore) -> None: + core: Final = _with_old_style_strategy(make_core) + assert ids_of(core.get_available_deployment("b")) == ["db"] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the sync pick + + +@pytest.mark.asyncio +async def test_an_endpoint_always_hands_the_strategy_its_request(make_core: MakeCore) -> None: + core: Final = _with_old_style_strategy(make_core) + with pytest.raises(TypeError, match="request_kwargs"): + await core.acompletion(model="b", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +# --- N1, N15: wildcards --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_alias_to_a_wildcard_fills_the_model_from_the_alias_target( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("anthropic/*", "w", model="anthropic/*")], + num_retries=0, + model_group_alias={"al": "anthropic/claude-3-haiku"}, + ) + await core.acompletion(model="al", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.calls_to("acompletion")[0].model == "anthropic/claude-3-haiku" + + +@pytest.mark.asyncio +async def test_a_name_no_pattern_matches_is_tried_under_its_provider(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("openai/*", "w0", model="openai/*")], num_retries=0) + listed: Final = await core.select("gpt-4o", {"user": "u"}, kind="healthy") # pyright: ignore[reportCallIssue, reportArgumentType] # PythonCore's select takes keyword options + assert ids_of(listed) == ["w0"] + assert isinstance(listed, list) + assert listed[0]["litellm_params"]["model"] == "openai/gpt-4o" + + +@pytest.mark.asyncio +async def test_a_call_through_the_provider_spelling_is_served(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("openai/*", "w0", model="openai/*")], num_retries=0) + await core.acompletion(model="gpt-4o", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.calls_to("acompletion")[0].model == "openai/gpt-4o" + + +# --- #49685: wildcard cache flags ----------------------------------------------------------------- + +CACHE_FLAGS: Final = {"supports_prompt_cache_breakpoint": True, "supports_prompt_caching": True} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pattern", "request_model", "kept"), + [ + ("openai/*", "openai/gpt-4o", {}), + ("azure_ai/*", "azure_ai/gpt-4o", {"supports_prompt_cache_breakpoint": True}), + ], + ids=["any-wildcard", "azure-ai-keeps-the-breakpoint-flag"], +) +async def test_a_wildcard_leaves_the_cache_flags_to_the_catalog( + make_core: MakeCore, host: ScriptedHost, pattern: str, request_model: str, kept: Mapping[str, object] +) -> None: + core: Final = make_core( + model_list=[deployment(pattern, "w", model=pattern, model_info=dict(CACHE_FLAGS))], + num_retries=0, + ) + await core.acompletion(model=request_model, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + model_info: Final = host.calls_to("acompletion")[0].kwargs["model_info"] + assert isinstance(model_info, Mapping) + assert {flag: model_info[flag] for flag in CACHE_FLAGS if flag in model_info} == kept + + +@pytest.mark.asyncio +async def test_a_wildcard_group_with_a_fixed_model_keeps_its_cache_flags( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("openai/*", "w", model="openai/gpt-4o", model_info=dict(CACHE_FLAGS))], + num_retries=0, + ) + await core.acompletion(model="openai/whatever", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + model_info: Final = host.calls_to("acompletion")[0].kwargs["model_info"] + assert isinstance(model_info, Mapping) + assert model_info["supports_prompt_caching"] is True + + +def test_the_wildcard_route_lookup_leaves_the_cache_flags_to_the_catalog(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("openai/*", "w", model="openai/*", model_info=dict(CACHE_FLAGS))], + num_retries=0, + ) + routed: Final = core.registry().deployments("openai/gpt-4o", source="wildcard") + assert not set(CACHE_FLAGS) & set(routed[0]["model_info"]) + + +# --- N12: specific_deployment --------------------------------------------------------------------- + + +def test_only_true_names_a_litellm_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("gpt-4o", "d1")], num_retries=0) + with pytest.raises(litellm.BadRequestError): + core.get_available_deployment( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the sync pick + model="gpt-4o ", specific_deployment="yes", request_kwargs={"input": "hi"} + ) + + +# --- N14: pass-through picks ---------------------------------------------------------------------- + +COMPLEXITY_GROUP: Final = [ + deployment("g1", "b-1", model="openai/gpt-4o", use_in_pass_through=True), + deployment("g2", "b-2", model="openai/gpt-4o-mini", use_in_pass_through=True), + { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "g2", "REASONING": "g1"}}, + "complexity_router_default_model": "g2", + }, + }, +] + + +@pytest.mark.asyncio +async def test_a_listed_pass_through_pick_names_the_group_the_alias_resolved_to(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("team-gpt", "d1")], + num_retries=0, + model_group_alias={"alias-b": "team-gpt"}, + ) + with pytest.raises(litellm.BadRequestError, match="Model team-gpt has no deployment"): + await core.select("alias-b", {}, kind="pass_through") # pyright: ignore[reportCallIssue] # PythonCore's select takes keyword options + + +@pytest.mark.asyncio +async def test_an_async_pass_through_pick_of_a_strategy_marker_group_takes_the_tier(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=list(COMPLEXITY_GROUP), num_retries=0, routing_strategy="usage-based-routing-v2" + ) + picked: Final = await core.async_get_available_deployment_for_pass_through( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the async pass-through pick + "smart", {}, messages=[{"role": "user", "content": "what is up"}] + ) + assert ids_of(picked) == ["b-2"] + + +# --- N16: affinity -------------------------------------------------------------------------------- + +ENCRYPTED_INPUT: Final = [ + {"role": "user", "content": "hi"}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "thought"}], + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6bTktMg==;gAAAAABenc", + }, + {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6bTktMQ==;gAAAAABenc", "summary": []}, +] + + +@pytest.mark.asyncio +async def test_encrypted_affinity_reads_the_ids_the_pick_excludes(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("h", "m9-0", order=1), + deployment("h", "m9-1", order=1), + deployment("g", "m9-2"), + ], + num_retries=0, + model_group_affinity_config={"h": ["encrypted_content_affinity"]}, + ) + request: Final = { + "input": list(ENCRYPTED_INPUT), + "_excluded_deployment_ids": ["m9-1"], + "_retry_skipped_deployment_ids": ["m9-2", "m9-0"], + } + assert ids_of(await core.select("h", request, input=list(ENCRYPTED_INPUT))) == ["m9-0"] # pyright: ignore[reportCallIssue, reportArgumentType] # PythonCore's select takes keyword options + + +@pytest.mark.asyncio +async def test_a_target_order_on_a_call_narrows_its_pick(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "id-a"), deployment("gpt", "id-b")], + num_retries=0, + optional_pre_call_checks=["session_affinity", "deployment_affinity"], + ) + with pytest.raises(RouterRateLimitError): + await core.acompletion(model="gpt", messages=MESSAGES, _target_order=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +@pytest.mark.asyncio +async def test_a_target_order_on_a_call_serves_that_level(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("gpt", "low", order=1), deployment("gpt", "high", order=2)], + num_retries=0, + ) + await core.acompletion(model="gpt", messages=MESSAGES, _target_order=2) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["high"] + assert "_target_order" not in host.calls_to("acompletion")[0].kwargs + + +# --- S7: a health scope as a dict ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_health_scope_given_as_a_dict_reads_its_keys(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0), deployment("h", "h1")], + num_retries=0, + enable_health_check_routing=True, + background_health_check_model_groups={"h": True}, + ) + assert core.settings().background_health_check_model_groups == frozenset({"h"}) + core.health().health_checks.set_deployment_health_states( # pyright: ignore[reportArgumentType] # health states + {"d1": {"is_healthy": False, "timestamp": time.time(), "reason": "probe failed"}} + ) + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"d1"} diff --git a/tests/unit/test_router/core/test_parity_regress5_errors.py b/tests/unit/test_router/core/test_parity_regress5_errors.py new file mode 100644 index 00000000000..a0122f8eda6 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_regress5_errors.py @@ -0,0 +1,501 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the error causes of the regress5 pack: when +the routing note is added and what it says, the error an order chain raises, what a blocked response +and a failed attempt stamp and record, the names the errors carry, and non-string names at the lookup +members. +""" + +import asyncio +import datetime +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import GuardrailRaisedException, UnsupportedEndpointError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router import Router +from litellm.router_core import RouterCore +from tests.unit.test_router.core.endpoints_helpers import EndpointHost, patch_direct_endpoints +from tests.unit.test_router.core.fallbacks_helpers import ( + MESSAGES, + MakeCore, + build, + complete, + complete_sync, + show_router_debug, +) +from tests.unit.test_router.core.scripted_host import ( + auth_error, + bad_request, + completion_response, + deployment, + rate_limit, + server_error, + timeout, +) +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts, install_orjson_stub_if_missing + +install_orjson_stub_if_missing() + +NOTE: Final = "LiteLLM: model group" +NO_FALLBACK: Final = "\n\nLiteLLM: model group 'a' failed with the error above. No fallback was attempted." +LONG_MESSAGE: Final = "x" * 900 + + +@pytest.fixture +def host(monkeypatch: pytest.MonkeyPatch) -> EndpointHost: + endpoint_host: Final = EndpointHost() + patch_direct_endpoints(monkeypatch, endpoint_host) + return endpoint_host + + +def as_router(core: RouterCore) -> Router: + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach + return router + + +def content_filtered() -> object: + response: Final = completion_response() + response.choices[0].finish_reason = "content_filter" # pyright: ignore[reportAttributeAccessIssue] # Choices + return response + + +def cooled(core: RouterCore, *ids: str) -> Mapping[str, Mapping[str, object]]: + return dict(core.health().cooldowns.get_active_cooldowns(list(ids), None)) # pyright: ignore[reportReturnType] # a mapping of entries + + +async def settle() -> None: + """Let the tasks a failed stream scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +async def failed_with(core: RouterCore, **request: object) -> BaseException: # kwargs-ok: request kwargs + with pytest.raises(Exception) as raised: # noqa: PT011 # any failure: each caller asserts on what it gets back + await complete(core, **request) + return raised.value + + +# --- N2: when the note is added ------------------------------------------------------------------- + + +async def test_disable_fallbacks_raises_with_no_note(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", server_error()) + error: Final = await failed_with(core, disable_fallbacks=True) + assert NOTE not in str(error) + assert host.deployments_called() == ["a"] + + +async def test_a_guardrail_block_raises_with_no_note(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", GuardrailRaisedException(guardrail_name="g", message="blocked")) + error: Final = await failed_with(core) + assert isinstance(error, GuardrailRaisedException) + assert NOTE not in str(error) + + +async def test_a_providers_bad_request_falls_back_and_notes_the_failed_fallback( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", bad_request()) + host.answer("openai/b", bad_request()) + error: Final = await failed_with(core) + assert f"{NOTE} 'a' failed with the error above. Fallback to b also failed" in str(error) + assert host.deployments_called() == ["a", "b"] + + +def test_a_single_attempt_endpoint_raises_with_no_note(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", server_error()) + with pytest.raises(litellm.InternalServerError) as raised: + core.responses(model="a", input="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert NOTE not in str(raised.value) + + +async def test_an_empty_request_list_says_no_fallbacks_are_configured(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + message: Final = str(await failed_with(core, fallbacks=[])) + assert "and no fallback model group was found for it" in message + assert "No fallbacks are configured." in message + + +def test_a_strategy_marker_group_with_an_empty_list_says_no_fallbacks_are_configured( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core( + model_list=[ + { + "model_name": "qr", + "litellm_params": { + "model": "auto_router/quality_router", + "quality_router_default_model": "t1", + "quality_router_config": { + "available_models": ["t1", "t3"], + "complexity_to_quality": {"SIMPLE": 2, "REASONING": 3}, + }, + }, + "model_info": {"id": "qr"}, + }, + deployment("t1", "t1", model_info={"litellm_routing_preferences": {"quality_tier": 1}}), + deployment("t3", "t3", model_info={"litellm_routing_preferences": {"quality_tier": 3}}), + ], + num_retries=0, + ) + show_router_debug(host) + with pytest.raises( + Exception, match="model group 'qr' failed with the error above and no fallback model group was found" + ) as raised: + complete_sync(core, "qr", fallbacks=[]) + assert "No fallbacks are configured." in str(raised.value) + + +async def test_a_chain_that_only_names_the_failed_group_reads_the_primary_error( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["a"]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + assert str(await failed_with(core)).endswith( + "LiteLLM: model group 'a' failed with the error above. " + "Fallback to a also failed: litellm.RateLimitError: slow down" + ) + + +async def test_a_depth_spent_before_any_hop_reads_the_primary_error(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], max_fallbacks=0) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + assert str(await failed_with(core)).endswith("Fallback to b also failed: litellm.RateLimitError: slow down") + assert host.deployments_called() == ["a"] + + +# --- U11: an order chain --------------------------------------------------------------------------- + + +async def test_every_order_level_failing_raises_the_last_levels_error_with_no_note( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core( + model_list=[deployment("bb", "bb1", order=1), deployment("bb", "bb2", order=2)], num_retries=0 + ) + show_router_debug(host) + host.answer("openai/bb1", rate_limit(message="first")) + host.answer("openai/bb2", rate_limit(message="second")) + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core, "bb") + assert str(raised.value) == "litellm.RateLimitError: second" + assert getattr(raised.value, "failed_deployment_id", None) == "bb2" + + +# --- R13: a blocked response ------------------------------------------------------------------------ + + +async def test_a_blocked_response_stamps_the_deployment(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, content_policy_fallbacks=[{"a": ["a"]}]) + host.answer("openai/a", content_filtered()) + error: Final = await failed_with(core) + assert isinstance(error, litellm.ContentPolicyViolationError) + assert getattr(error, "failed_deployment_id", None) == "a" + + +def test_a_blocked_sync_response_stamps_the_deployment(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, content_policy_fallbacks=[{"a": ["a"]}]) + host.answer("openai/a", content_filtered()) + with pytest.raises(litellm.ContentPolicyViolationError) as raised: + complete_sync(core) + assert getattr(raised.value, "failed_deployment_id", None) == "a" + + +async def test_a_blocked_response_counts_as_a_failure(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core, content_policy_fallbacks=[{"a": ["a"]}]) + host.answer("openai/a", content_filtered()) + await failed_with(core) + assert core.fail_calls["openai/a"] == 1 # pyright: ignore[reportAttributeAccessIssue] # the router's live counters + assert core.success_calls["openai/a"] == 0 # pyright: ignore[reportAttributeAccessIssue] # see above + + +# --- R12, S11, U6: what a cooldown keeps and what a timeout says -------------------------------------- + + +async def test_a_cooldown_keeps_the_timeout_as_it_was_raised(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core( + model_list=[deployment("a", "a1", weight=1), deployment("a", "a2", weight=0)], num_retries=0, allowed_fails=0 + ) + show_router_debug(host) + host.answer("openai/a1", timeout()) + error: Final = await failed_with(core) + assert "Deployment Info" in str(error) + assert len(str(cooled(core, "a1")["a1"]["exception_received"])) == len(str(timeout())) + + +async def test_a_cooldown_keeps_the_whole_error(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core( + model_list=[deployment("a", "a1", weight=1), deployment("a", "a2", weight=0)], num_retries=0, allowed_fails=0 + ) + host.answer("openai/a1", rate_limit(message=LONG_MESSAGE)) + await failed_with(core) + expected: Final = str(rate_limit(message=LONG_MESSAGE)) + assert len(str(cooled(core, "a1")["a1"]["exception_received"])) == len(expected) + + +async def test_a_stream_failing_on_its_first_chunk_cools_with_the_routing_note( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=2, retry_after=0) + show_router_debug(host) + host.answer("openai/a", chat_stream(error=auth_error(), fail_after=0, logged=True)) + error: Final = await failed_with(core, stream=True) + await settle() + assert NOTE in str(error) + assert len(str(cooled(core, "a")["a"]["exception_received"])) == len(str(error)) + + +def test_a_sync_timeout_says_nothing_of_the_deployment(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core) + show_router_debug(host) + host.answer("openai/a", timeout()) + with pytest.raises(litellm.Timeout) as raised: + complete_sync(core) + assert "Deployment Info" not in str(raised.value) + + +async def test_an_async_timeout_counts_no_failure(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = build(make_core) + host.answer("openai/a", timeout()) + await failed_with(core) + assert core.fail_calls["openai/a"] == 0 # pyright: ignore[reportAttributeAccessIssue] # the router's live counters + + +# --- R14, N19: a name nothing serves ------------------------------------------------------------------ + + +async def test_a_default_fallback_with_no_deployment_is_the_name_the_error_gives( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0, default_fallbacks=["missing"]) + with pytest.raises(litellm.BadRequestError) as raised: + await complete(core, "nope") + assert raised.value.model == "missing" + assert "You passed in model=missing." in str(raised.value) + + +def test_an_empty_router_names_the_first_default_fallback(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[], num_retries=1, retry_after=0, default_fallbacks=["team-gpt", "gpt-4o-mini"]) + with pytest.raises(litellm.BadRequestError) as raised: + complete_sync(core, "gpt-4o") + assert raised.value.model == "team-gpt" + + +async def test_an_unknown_alias_target_is_the_name_the_error_gives(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0, model_group_alias={"al": "nowhere"}) + with pytest.raises(litellm.BadRequestError) as raised: + await complete(core, "al") + assert raised.value.model == "nowhere" + + +async def test_a_missing_message_list_through_a_wildcard_alias_reaches_the_call( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core( + model_list=[deployment("openai/*", "w", model="openai/*"), deployment("g", "d1")], + num_retries=0, + model_group_alias={"al": {"hidden": False, "model": "gpt-4o-mini"}}, + ) + await core.acompletion(model="al", messages=None) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("acompletion") == ["w"] + + +async def test_moderation_of_a_name_only_a_wildcard_serves_uses_the_wildcard( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[deployment("openai/*", "w", model="openai/*")], num_retries=0) + await core.amoderation(model="openai/omni-moderation-latest", input="hello") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert host.deployments_called("amoderation") == ["w"] + + +# --- R15, T7: pass-through ------------------------------------------------------------------------ + + +async def test_a_listed_pick_of_an_id_names_its_litellm_model(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "p1", model="openai/b"), deployment("g", "p4", use_in_pass_through=True)], + num_retries=0, + ) + with pytest.raises(litellm.BadRequestError, match="Deployment p1 does not support pass-through") as raised: + await core.select("p1", {}, kind="pass_through") # pyright: ignore[reportCallIssue] # PythonCore's select takes keyword options + assert raised.value.model == "openai/b" + + +async def test_an_async_pick_of_an_id_names_the_id(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "p1", model="openai/b"), deployment("g", "p4", use_in_pass_through=True)], + num_retries=0, + ) + with pytest.raises(litellm.BadRequestError, match="Deployment p1 does not support pass-through") as raised: + await core.async_get_available_deployment_for_pass_through("p1", {}, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the async pass-through pick + assert raised.value.model == "p1" + + +def unknown_provider(model: str = "nope") -> litellm.BadRequestError: + return litellm.BadRequestError(message="LLM Provider NOT provided", model=model, llm_provider="") + + +async def test_a_pass_through_to_an_unknown_model_carries_the_note(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + show_router_debug(host) + host.answer("nope", unknown_provider()) + with pytest.raises(litellm.BadRequestError) as raised: + await core.allm_passthrough_route(model="nope", method="GET", endpoint="/models") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert str(raised.value).endswith( + "\n\nLiteLLM: model group 'nope' failed with the error above. No fallback was attempted." + ) + + +async def test_a_pass_through_with_no_deployment_calls_the_handler_before_any_fallback( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0, fallbacks=[{"nope": ["g"]}]) + host.answer("nope", unknown_provider()) + await core.allm_passthrough_route(model="nope", method="GET", endpoint="/models") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert [call.model for call in host.calls_to("allm_passthrough_route")] == ["nope", "openai/d1"] + + +# --- S9: the deployment an exception names -------------------------------------------------------- + + +def unsupported() -> UnsupportedEndpointError: + return UnsupportedEndpointError(message="not here", llm_provider="openai", model="gpt-4o") + + +async def test_an_exception_naming_a_retried_deployment_exhausts_the_retries( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0)], num_retries=1, retry_after=0 + ) + reused: Final = unsupported() + reused.failed_deployment_id = "d1" # pyright: ignore[reportAttributeAccessIssue] # what the first failure stamped + host.answer("openai/d1", unsupported()) + host.answer("openai/d2", reused) + with pytest.raises(UnsupportedEndpointError) as raised: + await complete(core, "g") + assert (raised.value.num_retries, raised.value.max_retries) == (1, 1) + assert "LiteLLM Retried: 1 times" in str(raised.value) + assert host.deployments_called("acompletion") == ["d1", "d2"] + + +async def test_every_deployment_refusing_raises_before_the_retries_are_exhausted( + make_core: MakeCore, host: EndpointHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0)], num_retries=1, retry_after=0 + ) + host.answer("*", unsupported()) + with pytest.raises(UnsupportedEndpointError) as raised: + await complete(core, "g") + assert raised.value.max_retries is None + assert host.deployments_called("acompletion") == ["d1", "d2"] + + +# --- N5: model=None ----------------------------------------------------------------------------------- + + +async def test_model_none_is_a_bad_request(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + show_router_debug(host) + with pytest.raises(litellm.BadRequestError, match="You passed in model=None") as raised: + await core.acompletion(model=None, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert raised.value.status_code == 400 + assert NOTE not in str(raised.value) + + +def test_sync_model_none_is_a_bad_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(litellm.BadRequestError, match="You passed in model=None"): + core.completion(model=None, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +# --- N4: lookup members given a non-string name --------------------------------------------------- + + +def test_a_non_string_name_is_not_recognized(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + assert core.registry().resolve(True, kind="recognized") is False + + +def test_an_unhashable_name_raises_as_the_index_lookup_raises(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + with pytest.raises(TypeError, match="unhashable type: 'list'"): + core.registry().resolve(["g"], kind="recognized") # pyright: ignore[reportArgumentType] # raising is the point + + +@pytest.mark.parametrize( + ("member", "expected"), + [ + ("get_configured_display_name", None), + ("get_configured_mode", None), + ("get_deployment", None), + ("get_model_ids", []), + ], +) +def test_a_lookup_given_an_int_finds_nothing(make_core: MakeCore, member: str, expected: object) -> None: + router: Final = as_router(make_core(model_list=[deployment("g", "d1")], num_retries=0)) + with pytest.warns(DeprecationWarning, match="is deprecated"): + assert getattr(router, member)(1) == expected # pyright: ignore[reportAny] # each old member's own type + + +def test_a_model_list_lookup_given_an_int_raises_as_the_wildcard_step_raises(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("g", "d1")], num_retries=0)) + with ( + pytest.warns(DeprecationWarning, match="is deprecated"), + pytest.raises(TypeError, match="argument of type 'int' is not"), + ): + router.get_model_list(1) # pyright: ignore[reportArgumentType] # raising is the point + + +# --- V6: a base_model that is no string ----------------------------------------------------------- + + +def test_an_int_base_model_fails_as_startswith_fails(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "az1", model="azure/my-dep", api_base="https://x", base_model=7)], num_retries=0 + ) + with pytest.raises(AttributeError, match=r"^'int' object has no attribute 'startswith'$"): + core.registry().router_model_info(None, "g", id="az1") + + +# --- a shared logging object ---------------------------------------------------------------------- + + +async def test_a_failure_already_logged_blocks_a_streams_count(make_core: MakeCore, host: EndpointHost) -> None: + core: Final = make_core( + model_list=[deployment("a", "a1", weight=1), deployment("a", "a2", weight=0)], num_retries=0, allowed_fails=0 + ) + logging_obj: Final = LiteLLMLogging( + model="a", + messages=MESSAGES, + stream=True, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="c", + function_id="f", + ) + logging_obj.update_environment_variables(model="a", user=None, optional_params={}, litellm_params={}) + logging_obj.model_call_details["has_logged_async_failure"] = True + host.answer("openai/a1", chat_stream("hi", error=rate_limit(), fail_after=1)) + stream: Final = await complete(core, stream=True, litellm_logging_obj=logging_obj) + with pytest.raises(litellm.RateLimitError): + await collect_texts(stream) + await settle() + assert cooled(core, "a1", "a2") == {} diff --git a/tests/unit/test_router/core/test_parity_regress5_settings.py b/tests/unit/test_router/core/test_parity_regress5_settings.py new file mode 100644 index 00000000000..6d3c8dea7ce --- /dev/null +++ b/tests/unit/test_router/core/test_parity_regress5_settings.py @@ -0,0 +1,473 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the settings causes of the regress5 pack: +attributes written after build, ``update_settings`` applied key by key, a request ``num_retries`` +that is no count, calls after ``retire``, deleted attributes, and a default deployment that fails +every call. The refusals at the end are ruled to differ, so they run on the Rust core only. +""" + +import inspect +from typing import Final + +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import Deployment, LiteLLM_Params, RetryPolicy +from tests.unit.test_router.core.gaps_l11100_helpers import TIERS, complexity_marker +from tests.unit.test_router.core.retries_health_helpers import MakeCore, attempt +from tests.unit.test_router.core.scripted_host import ( + CORE_BUILDERS, + ScriptedHost, + deployment, + rate_limit, + server_error, +) + +HEURISTIC_V2: Final = {"classifier_type": "heuristic_v2", "tiers": dict(TIERS)} + + +def _one_group(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(model_list=[deployment("g", "d1")], **{"num_retries": 0, **arguments}) + + +# --- U7: retry policies written after build --------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_retry_policy_assigned_after_build_governs_the_next_failure( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one_group(make_core, num_retries=1, retry_after=0, disable_cooldowns=True) + core.retry_policy = RetryPolicy(RateLimitErrorRetries=0) # pyright: ignore[reportAttributeAccessIssue] # a plain attribute of both cores + host.answer("*", lambda **_: rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert host.deployments_called("acompletion") == ["d1"] + + +def test_model_group_retry_policy_assigned_after_build_answers_the_lookup(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + policies: Final = {"g": RetryPolicy(RateLimitErrorRetries=1)} + core.model_group_retry_policy = policies # pyright: ignore[reportAttributeAccessIssue] # see above + + assert core.model_group_retry_policy == policies # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.get_num_retries_from_retry_policy(rate_limit(), "g") == 1 # pyright: ignore[reportAttributeAccessIssue] # see above + + +@pytest.mark.asyncio +async def test_a_retry_policy_written_as_a_dict_stays_a_dict_and_applies( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one_group(make_core, num_retries=1, retry_after=0, disable_cooldowns=True) + core.retry_policy = {"RateLimitErrorRetries": 0} # pyright: ignore[reportAttributeAccessIssue] # stored as given + host.answer("*", lambda **_: rate_limit()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert core.retry_policy == {"RateLimitErrorRetries": 0} # pyright: ignore[reportAttributeAccessIssue] # see above + assert host.deployments_called("acompletion") == ["d1"] + + +@pytest.mark.asyncio +async def test_a_retry_policy_written_as_no_policy_raises_on_the_first_failure_only( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = _one_group(make_core, num_retries=1, retry_after=0, disable_cooldowns=True) + core.retry_policy = 5 # pyright: ignore[reportAttributeAccessIssue] # stored as given + + served: Final = await attempt(core) + host.answer("*", lambda **_: rate_limit()) + error: Final = await attempt(core) + + assert not isinstance(served, Exception) + assert isinstance(error, AttributeError) + assert "DefaultRetries" in str(error) + + +@pytest.mark.parametrize( + ("name", "value"), [("enable_weighted_failover", True), ("fallback_on_unsupported_params", False)] +) +def test_other_plain_settings_read_back_what_was_written(make_core: MakeCore, name: str, value: object) -> None: + core: Final = _one_group(make_core) + setattr(core, name, value) + assert getattr(core, name) is value + + +# --- N8: update_settings applies its keys in order, with no rollback -------------------------------- + + +def test_update_settings_keeps_the_keys_before_the_one_that_raises(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + with pytest.raises(TypeError): + core.update_settings( # pyright: ignore[reportAttributeAccessIssue] # the deprecated member + context_window_fallbacks=[{"g": ["h"]}], allowed_fails=["not a number"], timeout=3 + ) + assert core.settings().context_window_fallbacks == [{"g": ["h"]}] + assert core.timeout != 3 # pyright: ignore[reportAttributeAccessIssue] # a plain attribute of both cores + + +@pytest.mark.parametrize("strategy", [1, "bogus"]) +def test_update_settings_refuses_an_unknown_strategy(make_core: MakeCore, strategy: object) -> None: + core: Final = _one_group(make_core) + with pytest.raises(ValueError, match="Invalid routing_strategy"): + core.update_settings(fallbacks=[{"g": ["h"]}], routing_strategy=strategy, timeout=3) # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.settings().routing_strategy == "simple-shuffle" + assert core.settings().fallbacks == [{"g": ["h"]}] + + +def test_update_settings_keeps_no_strategy_as_given(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + core.update_settings(routing_strategy=None) # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.settings().routing_strategy is None + + +def test_a_failed_setting_leaves_the_groups_passed_before_it_unread(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + groups: Final = [{"group_name": "z", "models": ["g"], "routing_strategy": "least-busy"}] + with pytest.raises(ValueError, match="invalid literal"): + core.update_settings(routing_groups=groups, timeout="soon") # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.settings().as_dict()["routing_groups"] == [] + + +def test_groups_that_do_not_parse_raise_after_every_other_key(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + with pytest.raises(Exception, match="group_name"): + core.update_settings(timeout=6, routing_groups=[{"bad": 1}], retry_after=5) # pyright: ignore[reportAttributeAccessIssue] # see above + assert (core.timeout, core.retry_after) == (6, 5) # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_optional_pre_call_checks_that_cannot_be_read_stop_the_keys_after_them(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + with pytest.raises(TypeError, match="not iterable"): + core.update_settings(timeout=5, optional_pre_call_checks=1, retry_after=4) # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.timeout == 5 # pyright: ignore[reportAttributeAccessIssue] # see above + assert core.retry_after != 4 # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_a_changeset_stops_at_the_setting_that_raises(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("h", "d2")], num_retries=0) + with pytest.raises(TypeError): + core.apply(ChangeSet(settings={"fallbacks": [{"g": ["h"]}], "allowed_fails": [1]}, delete=["d1"])) + assert core.settings().fallbacks == [{"g": ["h"]}] + assert sorted(core.registry().names(kind="ids")) == ["d1", "d2"] + + +# --- N7: a request num_retries is compared, never cast ---------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("value", ["2", 1.5, [1], -1, float("nan"), True, False, 0.0]) +async def test_a_request_that_succeeds_never_reads_its_num_retries(make_core: MakeCore, value: object) -> None: + served: Final = await attempt(_one_group(make_core), num_retries=value) + assert not isinstance(served, Exception) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("value", "message"), + [ + ("2", "'>' not supported between instances of 'str' and 'int'"), + ([1], "'>' not supported between instances of 'list' and 'int'"), + (1.5, "'float' object cannot be interpreted as an integer"), + ], +) +async def test_a_failure_reads_a_num_retries_that_is_no_count( + make_core: MakeCore, host: ScriptedHost, value: object, message: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0) + host.answer("*", lambda **_: server_error()) + + error: Final = await attempt(core, num_retries=value) + + assert isinstance(error, TypeError) + assert str(error) == message + assert isinstance(error.__context__, litellm.InternalServerError) + assert len(host.deployments_called("acompletion")) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("value", [-1, float("nan"), False, 0.0]) +async def test_a_num_retries_below_one_leaves_the_failure_standing( + make_core: MakeCore, host: ScriptedHost, value: object +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=3) + host.answer("*", lambda **_: server_error()) + + error: Final = await attempt(core, num_retries=value) + + assert isinstance(error, litellm.InternalServerError) + assert len(host.deployments_called("acompletion")) == 1 + + +@pytest.mark.asyncio +async def test_a_retry_policy_outranks_a_num_retries_that_is_no_count( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + num_retries=0, + retry_policy=RetryPolicy(InternalServerErrorRetries=1), + retry_after=0, + ) + host.answer("*", server_error(), lambda **_: None) + + served: Final = await attempt(core, num_retries="2") + + assert not isinstance(served, Exception) + assert len(host.deployments_called("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_num_retries_true_counts_as_one_retry(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, retry_after=0, disable_cooldowns=True + ) + host.answer("*", lambda **_: server_error()) + + error: Final = await attempt(core, num_retries=True) + + assert isinstance(error, litellm.InternalServerError) + assert len(host.deployments_called("acompletion")) == 2 + + +# --- N17: retire releases registrations, not the router ---------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_retired_core_keeps_serving(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one_group(make_core) + core.retire() + + served: Final = await attempt(core) + + assert not isinstance(served, Exception) + assert host.deployments_called("acompletion") == ["d1"] + + +# --- N18: attributes the Python core keeps as plain attributes --------------------------------------- + + +@pytest.mark.asyncio +async def test_a_deleted_scheduler_is_gone_for_reads_and_for_prioritized_calls(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + del core.scheduler # pyright: ignore[reportAttributeAccessIssue] # a plain attribute of both cores + + with pytest.raises(AttributeError, match="scheduler"): + _ = core.scheduler # pyright: ignore[reportAttributeAccessIssue] # see above + served: Final = await attempt(core) + refused: Final = await attempt(core, priority=1) + + assert not isinstance(served, Exception) + assert isinstance(refused, AttributeError) + assert "scheduler" in str(refused) + + +@pytest.mark.parametrize("name", ["total_calls", "success_calls", "fail_calls"]) +def test_call_counters_are_replaced_and_deleted_like_attributes(make_core: MakeCore, name: str) -> None: + core: Final = _one_group(make_core) + replacement: Final[dict[str, int]] = {"openai/d1": 7} # mutable-ok: the value under test + + setattr(core, name, replacement) + assert getattr(core, name) is replacement + + delattr(core, name) + with pytest.raises(AttributeError, match=name): + getattr(core, name) + + +@pytest.mark.asyncio +async def test_a_success_counter_replaced_with_a_plain_dict_fails_the_attempt(make_core: MakeCore) -> None: + core: Final = _one_group(make_core) + core.success_calls = {} # pyright: ignore[reportAttributeAccessIssue] # see above + + error: Final = await attempt(core) + + assert isinstance(error, KeyError) + assert getattr(error, "failed_deployment_id", None) == "d1" + assert core.fail_calls == {"openai/d1": 1} # pyright: ignore[reportAttributeAccessIssue] # see above + + +def test_embedding_names_its_parameters(make_core: MakeCore) -> None: + signature: Final = inspect.signature(_one_group(make_core).embedding) # pyright: ignore[reportAttributeAccessIssue] # an endpoint + assert list(signature.parameters) == ["model", "input", "is_async", "kwargs"] + assert signature.parameters["is_async"].default is False + + +def test_embedding_takes_its_arguments_by_position(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = _one_group(make_core) + core.embedding("g", "hi") # pyright: ignore[reportAttributeAccessIssue] # an endpoint + assert host.deployments_called("embedding") == ["d1"] + + +# --- O2: deleted aliases and retry budget ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_deleted_alias_map_fails_every_read_until_it_is_assigned_again(make_core: MakeCore) -> None: + core: Final = _one_group(make_core, model_group_alias={"al": "g"}) + del core.model_group_alias # pyright: ignore[reportAttributeAccessIssue] # the deprecated attribute + + assert "model_group_alias" not in core.settings().as_dict() + with pytest.raises(AttributeError, match="model_group_alias"): + core.registry().aliases() + with pytest.raises(AttributeError, match="model_group_alias"): + core.registry().names() + for model in ("al", "g"): + refused = await attempt(core, model=model) + assert isinstance(refused, AttributeError) + assert "model_group_alias" in str(refused) + with pytest.raises(AttributeError, match="model_group_alias"): + del core.model_group_alias # pyright: ignore[reportAttributeAccessIssue] # see above + + core.apply(ChangeSet(assign={"model_group_alias": {"x": "g"}})) + + assert not isinstance(await attempt(core, model="x"), Exception) + + +@pytest.mark.asyncio +async def test_a_deleted_num_retries_fails_calls_that_name_none(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("h", "d2")], fallbacks=[{"g": ["h"]}]) + del core.num_retries # pyright: ignore[reportAttributeAccessIssue] # a plain attribute of both cores + + assert "num_retries" not in core.settings().as_dict() + with pytest.raises(AttributeError, match="num_retries"): + _ = core.num_retries # pyright: ignore[reportAttributeAccessIssue] # see above + refused: Final = await attempt(core) + assert isinstance(refused, AttributeError) + assert "num_retries" in str(refused) + assert host.deployments_called("acompletion") == [] + + assert not isinstance(await attempt(core, num_retries=0), Exception) + with pytest.raises(AttributeError, match="num_retries"): + del core.num_retries # pyright: ignore[reportAttributeAccessIssue] # see above + + core.num_retries = 1 # pyright: ignore[reportAttributeAccessIssue] # see above + + assert core.num_retries == 1 # pyright: ignore[reportAttributeAccessIssue] # see above + assert not isinstance(await attempt(core), Exception) + + +# --- T5: a default deployment every call to fails ------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "stamped"), [(None, None), ({"id": "dd"}, "dd")], ids=["no_model_info", "its_own_id"] +) +async def test_a_failing_default_deployment_stamps_only_its_own_id( + make_core: MakeCore, model_info: dict[str, object] | None, stamped: str | None +) -> None: + core: Final = _one_group(make_core) + assigned: Final = { + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}, + **({"model_info": model_info} if model_info is not None else {}), + } + core.apply(ChangeSet(assign={"default_deployment": assigned})) + + error: Final = await attempt(core, model="unknown") + + assert isinstance(error, KeyError) + assert error.args == ("model_name",) + assert getattr(error, "failed_deployment_id", None) == stamped + + +# --- S10: the capability limit is called as given -------------------------------------------------------- + + +def test_a_capability_limit_that_cannot_be_called_fails_the_build(make_core: MakeCore) -> None: + with pytest.raises(TypeError, match="not callable"): + make_core( + model_list=[ + deployment("big", "big"), + deployment("small", "small"), + complexity_marker("smart", "m1", config=HEURISTIC_V2), + ], + auto_router_capability_limit=0, + ) + + +# --- ruled to differ: the Rust core only --------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("params", "named"), + [ + ({"model_info": {"input_cost_per_token": "inf"}}, "model_info.input_cost_per_token"), + ({"model_info": {"output_cost_per_token": "nan"}}, "model_info.output_cost_per_token"), + ({"model_info": {"input_cost_per_token": "1e400"}}, "model_info.input_cost_per_token"), + ({"model_info": {"input_cost_per_token": float("inf")}}, "model_info.input_cost_per_token"), + ({"input_cost_per_token": "inf"}, "litellm_params.input_cost_per_token"), + ], + ids=["inf", "nan", "overflow", "float", "litellm_params"], +) +def test_the_rust_core_refuses_a_cost_that_is_not_finite( + host: ScriptedHost, params: dict[str, object], named: str +) -> None: + with pytest.raises(ValueError, match=f"{named} is .*Set a finite cost"): + CORE_BUILDERS["rust"](host, model_list=[deployment("g", "m", model="openai/gpt-4o", **params)]) + + +def test_the_rust_core_refuses_a_cost_that_is_not_finite_on_add(host: ScriptedHost) -> None: + core: Final = CORE_BUILDERS["rust"](host, model_list=[deployment("g", "m")]) + added: Final = Deployment( + model_name="g", + litellm_params=LiteLLM_Params(model="openai/gpt-4o", api_key="k"), + model_info={"id": "late", "input_cost_per_token": float("inf")}, + ) + with pytest.raises(ValueError, match="Set a finite cost"): + core.add_deployment(added) # pyright: ignore[reportAttributeAccessIssue] # the deprecated member + assert core.registry().names(kind="ids") == ["m"] + + +def test_the_rust_core_leaves_out_a_bad_cost_under_ignore_invalid_deployments(host: ScriptedHost) -> None: + core: Final = CORE_BUILDERS["rust"]( + host, + model_list=[deployment("g", "m", model_info={"input_cost_per_token": "inf"}), deployment("g", "n")], + ignore_invalid_deployments=True, + ) + assert core.registry().names(kind="ids") == ["n"] + + +@pytest.mark.parametrize("share", [10**20, 1e20], ids=["int", "float"]) +def test_the_rust_core_refuses_a_share_wider_than_it_can_hold(host: ScriptedHost, share: object) -> None: + listed: Final = deployment("g", "sh", model_info={"db_model": True, "ptu_shares": {"T1": share}}) + with pytest.raises(ValueError, match=r"model_info\.ptu_shares\.T1 is 100000000000000000000, outside"): + CORE_BUILDERS["rust"](host, model_list=[listed]) + + +def test_the_rust_core_refuses_a_share_wider_than_it_can_hold_on_add(host: ScriptedHost) -> None: + core: Final = CORE_BUILDERS["rust"](host, model_list=[deployment("g", "free")]) + added: Final = Deployment( + model_name="g", + litellm_params=LiteLLM_Params(model="openai/gpt-4o", api_key="k"), + model_info={"id": "late", "db_model": True, "ptu_shares": {"T1": 10**20}}, + ) + with pytest.raises(ValueError, match=r"model_info\.ptu_shares\.T1 is 100000000000000000000, outside"): + core.add_deployment(added) # pyright: ignore[reportAttributeAccessIssue] # the deprecated member + assert core.registry().names(kind="ids") == ["free"] + + +# --- S5: in-place edits never reach the Rust walk, or its notes ------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_rust_error_note_reads_the_fallbacks_the_walk_used(host: ScriptedHost) -> None: + host.set_dynamic(expose_router_debug_in_errors=True) + core: Final = CORE_BUILDERS["rust"]( + host, + model_list=[deployment("g", "d1"), deployment("h", "h1")], + num_retries=0, + fallbacks=[{"x": ["h"]}], + ) + core.settings().fallbacks.append({"g": ["h"]}) # pyright: ignore[reportOptionalMemberAccess] # the live list + host.answer("*", lambda **_: server_error()) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.InternalServerError) + assert "configured: h" not in error.message + assert host.deployments_called("acompletion") == ["d1"] diff --git a/tests/unit/test_router/core/test_parity_regress5_streams.py b/tests/unit/test_router/core/test_parity_regress5_streams.py new file mode 100644 index 00000000000..173cb2c5db8 --- /dev/null +++ b/tests/unit/test_router/core/test_parity_regress5_streams.py @@ -0,0 +1,297 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the stream causes of the regress5 pack: +request-level fallback lists on a ``/v1/messages`` stream, what a stream's failure records, a +malformed group retry policy, the error a failed recovery raises, and the hint on a stream's failure +event. +""" + +import asyncio +from collections.abc import Callable, Mapping +from typing import Final + +import pytest +from pydantic import ValidationError + +import litellm +from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_core import RouterCore +from litellm.types.router import RetryPolicy +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + content_policy, + context_window, + deployment, + rate_limit, + server_error, +) +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGE_START, + MESSAGE_STOP, + MESSAGES, + anthropic_stream, + chat_stream, + collect_bytes, + collect_texts, + collect_texts_sync, + error_frame, + install_orjson_stub_if_missing, + refusal_delta, + text_delta, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] +PREFILL_MODEL: Final = "anthropic/claude-sonnet-4-5" +FALLBACK_FRAMES: Final = [MESSAGE_START, text_delta("B"), MESSAGE_STOP] +REFUSAL_DETAILS: Final = {"type": "refusal", "category": "cyber"} +PER_USER_COPILOT: Final = { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "ghc", + "github_copilot_auth_type": "per_user_oauth", +} + + +async def open_messages(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.aanthropic_messages(model="a", messages=MESSAGES, max_tokens=5, stream=True, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +async def settle() -> None: + """Let the logging and recording tasks a failed stream scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +def cooled(core: RouterCore, *ids: str) -> Mapping[str, Mapping[str, object]]: + return dict(core.health().cooldowns.get_active_cooldowns(list(ids), None)) # pyright: ignore[reportReturnType] # a mapping of entries + + +class BridgedStream(AnthropicSSEStream): + """The byte stream the completion bridge returns for ``/v1/messages``: a chat stream's failure is + logged by that stream and reaches the caller as an ``event: error`` frame.""" + + def __init__(self, stream: CustomStreamWrapper) -> None: + super().__init__( + AnthropicStreamWrapper(completion_stream=stream, model="d1", litellm_logging_obj=stream.logging_obj) + ) + self.logging_obj: Final = stream.logging_obj + + +def bridged(error: Exception) -> Callable[..., BridgedStream]: + chat: Final = chat_stream(error=error, fail_after=0, logged=True) + + def answer(**request: object) -> BridgedStream: # kwargs-ok: the host passes the request kwargs + return BridgedStream(chat(**request)) + + return answer + + +# --- T1: the request's fallback lists on a /v1/messages stream ------------------------------------- + + +@pytest.mark.parametrize("fallbacks", [["b"], [{"a": ["b"]}], [{"*": ["b"]}]], ids=["names", "keyed", "star"]) +async def test_request_fallbacks_rescue_a_messages_stream_before_content( + make_core: MakeCore, host: ScriptedHost, fallbacks: list[object] +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0) + assert await collect_bytes(await open_messages(core, fallbacks=fallbacks)) == FALLBACK_FRAMES + assert host.deployments_called() == ["a", "b"] + + +async def test_request_content_policy_fallbacks_rescue_a_refusal_before_content( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, refusal_delta(REFUSAL_DETAILS))) + host.answer("openai/b", anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0) + stream: Final = await open_messages(core, content_policy_fallbacks=[{"a": ["b"]}]) + assert await collect_bytes(stream) == FALLBACK_FRAMES + assert host.deployments_called() == ["a", "b"] + + +async def test_a_request_without_fallbacks_replaces_the_router_list_on_a_messages_stream( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + assert await collect_bytes(await open_messages(core, fallbacks=None)) == [MESSAGE_START, error_frame()] + assert host.deployments_called() == ["a"] + + +# --- T2: what a bridged stream's failure records -------------------------------------------------- + + +async def test_a_bridged_401_before_content_cools_the_deployment_down(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", bridged(auth_error())) + host.answer("openai/b", anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + assert await collect_bytes(await open_messages(core)) == FALLBACK_FRAMES + await settle() + assert sorted(cooled(core, "a", "b")) == ["a"] + assert str(cooled(core, "a")["a"]["status_code"]) == "401" + + +async def test_a_bridged_rate_limit_cools_with_the_provider_error_not_the_frame( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", bridged(rate_limit(message="provider text"))) + host.answer("openai/b", anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, allowed_fails=0, fallbacks=[{"a": ["b"]}]) + assert await collect_bytes(await open_messages(core)) == FALLBACK_FRAMES + await settle() + received: Final = str(cooled(core, "a")["a"]["exception_received"]) + assert "provider text" in received + assert "AnthropicException" not in received + + +async def test_a_caller_scoped_401_on_a_bridged_stream_never_cools_the_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "github_copilot/gpt-4o", + bridged( + litellm.CallerCredentialAuthenticationError( + message="not connected", llm_provider="github_copilot", model="gpt-4o" + ) + ), + ) + host.answer("openai/b", anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core( + model_list=[deployment("a", "a", **PER_USER_COPILOT), deployment("b", "b")], + num_retries=0, + allowed_fails=0, + fallbacks=[{"a": ["b"]}], + ) + assert await collect_bytes(await open_messages(core)) == FALLBACK_FRAMES + await settle() + assert cooled(core, "a", "b") == {} + + +# --- T3: a malformed group retry policy on a messages stream --------------------------------------- + + +async def test_a_malformed_group_policy_raises_on_a_messages_stream_that_never_opened( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", rate_limit()) + core: Final = make_core( + model_list=[deployment("a", "a1"), deployment("a", "a2")], + num_retries=1, + model_group_retry_policy={"a": {"RateLimitErrorRetries": "many"}}, + ) + with pytest.raises(ValidationError): + await open_messages(core) + assert len(host.deployments_called()) == 1 + + +@pytest.mark.parametrize("retry_policy", [None, RetryPolicy(InternalServerErrorRetries=0)], ids=["alone", "router"]) +async def test_a_malformed_group_policy_governs_nothing_on_a_stream_that_opened( + make_core: MakeCore, host: ScriptedHost, retry_policy: RetryPolicy | None +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=server_error()), anthropic_stream(*FALLBACK_FRAMES)) + core: Final = make_core( + model_list=[deployment("a", "a")], + num_retries=1, + retry_policy=retry_policy, + model_group_retry_policy={"a": {"RateLimitErrorRetries": "many"}}, + ) + assert await collect_bytes(await open_messages(core)) == FALLBACK_FRAMES + assert host.deployments_called() == ["a", "a"] + + +# --- S8: the error a failed recovery raises -------------------------------------------------------- + + +async def test_a_continuation_whose_fallback_stream_fails_raises_the_fallback_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", error=rate_limit(message="primary"))) + host.answer(PREFILL_MODEL, chat_stream(" wor", "ld", error=server_error(), fail_after=1)) + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b", model=PREFILL_MODEL)], + num_retries=0, + fallbacks=[{"a": ["b"]}], + enable_mid_stream_fallback_continuation=True, + ) + stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.InternalServerError, match="boom"): + await collect_texts(stream) + assert host.deployments_called() == ["a", "b"] + + +async def test_a_fallback_stream_that_fails_raises_its_own_error(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="primary"), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B", error=server_error())) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.InternalServerError, match="boom"): + await collect_texts(stream) + + +def test_a_sync_fallback_stream_that_fails_raises_its_own_error(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="primary"), leading_role_chunk=True, sync=True)) + host.answer("openai/b", chat_stream("B", error=server_error(), sync=True)) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.InternalServerError, match="boom"): + collect_texts_sync(stream) + + +async def test_a_messages_fallback_stream_that_fails_raises_its_own_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", anthropic_stream(MESSAGE_START, text_delta("B"), error=server_error())) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + stream: Final = await open_messages(core) + with pytest.raises(litellm.InternalServerError, match="boom"): + await collect_bytes(stream) + + +async def test_an_order_chain_that_serves_nothing_raises_its_last_hop_error_from_a_stream( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The order path of the fallback walk raises its last hop's error, not the failure it was handed.""" + host.answer("openai/a1", chat_stream(error=rate_limit(message="primary"), leading_role_chunk=True)) + host.answer("openai/a2", server_error()) + core: Final = make_core(model_list=[deployment("a", "a1", order=1), deployment("a", "a2", order=2)], num_retries=0) + stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.InternalServerError, match="boom"): + await collect_texts(stream) + assert host.deployments_called() == ["a1", "a2"] + + +# --- V3: the hint on a stream's failure event ------------------------------------------------------ + + +@pytest.mark.parametrize( + ("error", "raised", "hint"), + [ + (context_window, litellm.ContextWindowExceededError, "Set 'context_window_fallback'"), + (content_policy, litellm.ContentPolicyViolationError, "content_policy_fallback"), + ], + ids=["context_window", "content_policy"], +) +async def test_a_failure_event_of_a_stream_failing_before_its_first_chunk_carries_the_hint( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], raised: type[Exception], hint: str +) -> None: + host.set_dynamic(expose_router_debug_in_errors=True) + host.answer("openai/a", chat_stream(error=error(), fail_after=0, logged=True)) + host.answer("openai/b", server_error()) + core: Final = make_core(model_list=GROUPS_AB, num_retries=0, fallbacks=[{"a": ["b"]}]) + logged: Final[list[str]] = [] + + def record(kwargs: Mapping[str, object], *_rest: object) -> None: + logged.append(str(getattr(kwargs.get("exception"), "message", ""))) + + host.callback_list("failure_callback").append(record) + with pytest.raises(raised): + await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await settle() + # The fallback's own failure may be logged first. + assert any(hint in message for message in logged) diff --git a/tests/unit/test_router/core/test_parity_response_headers.py b/tests/unit/test_router/core/test_parity_response_headers.py new file mode 100644 index 00000000000..22794eccbef --- /dev/null +++ b/tests/unit/test_router/core/test_parity_response_headers.py @@ -0,0 +1,205 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the router's response headers, what a sync +pick filters, and the names the no-deployments errors carry. +""" + +import asyncio +import warnings +from collections.abc import Callable, Mapping +from typing import Final + +import pytest +from pydantic import ValidationError + +import litellm +from litellm.exceptions import BadRequestError, EndpointNotSupportedByModelError +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.router import RouterRateLimitError +from tests.unit.test_router.core.fallbacks_helpers import MESSAGES, MakeCore, complete, complete_sync, headers +from tests.unit.test_router.core.scripted_host import CORE_BUILDERS, ScriptedHost, completion_response, deployment, server_error + +GROUP_LIMITS: Final = {"rpm": 5, "tpm": 1000} +TEAM_OWNED: Final = {"team_id": "T1"} +UNREADABLE_LIMIT: Final = {"rpm": "abc"} +RETRY_COUNT: Final = "x-litellm-attempted-retries" +FALLBACK_COUNT: Final = "x-litellm-attempted-fallbacks" +GUARDRAILS: Final = [{"guardrail_name": "gr", "litellm_params": {"guardrail": "aporia", "mode": "pre_call"}}] +BLOCKED_RESPONSE: Final = { + "id": "chatcmpl-blocked", + "object": "chat.completion", + "model": "gpt-4o", + "created": 1, + "choices": [{"index": 0, "finish_reason": "content_filter", "message": {"role": "assistant", "content": "no"}}], +} +# The dumps of a deployment `_create_deployment` makes, each of which warns about a mocked ModelResponse. +PYTHON_DUMPS_OF_A_DEPLOYMENT: Final = 4 + + +# --- x-ratelimit-* and the counts ------------------------------------------------------------------ + + +def test_a_sync_call_reads_the_remaining_usage_of_its_group(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", **GROUP_LIMITS)], num_retries=0) + stamped: Final = headers(complete_sync(core, "g")) + assert stamped["x-ratelimit-limit-requests"] == 5 + assert stamped["x-ratelimit-remaining-requests"] == 4 + assert stamped["x-ratelimit-limit-tokens"] == 1000 + assert stamped[RETRY_COUNT] == 0 + assert stamped[FALLBACK_COUNT] == 0 + + +async def test_an_unreadable_group_limit_leaves_the_counts_off_the_response( + make_core: MakeCore, host: ScriptedHost +) -> None: + answered: Final = completion_response() + host.answer("openai/d1", answered) + core: Final = make_core(model_list=[deployment("g", "d1", model_info=UNREADABLE_LIMIT)], num_retries=0) + with pytest.raises(ValidationError): + await complete(core, "g") + stamped: Final = headers(answered) + assert stamped["x-litellm-model-group"] == "g" + assert RETRY_COUNT not in stamped + assert FALLBACK_COUNT not in stamped + + +def test_a_sync_call_reads_the_group_limit_too(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", model_info=UNREADABLE_LIMIT)], num_retries=0) + with pytest.raises(ValidationError): + complete_sync(core, "g") + + +def test_a_sync_call_opens_at_most_one_event_loop(make_core: MakeCore, monkeypatch: pytest.MonkeyPatch) -> None: + """Each loop of its own hands the GIL to the logging thread before the headers are on the response.""" + opened: Final[list[object]] = [] # mutable-ok: filled by the patched factory + real: Final[Callable[[], asyncio.AbstractEventLoop]] = asyncio.new_event_loop + + def counted() -> asyncio.AbstractEventLoop: + loop: Final = real() + opened.append(loop) + return loop + + core: Final = make_core(model_list=[deployment("g", "d1", **GROUP_LIMITS)], num_retries=0) + monkeypatch.setattr(asyncio, "new_event_loop", counted) + _ = complete_sync(core, "g") + assert len(opened) <= 1 + + +class _FallbackCountAtLog(CustomLogger): + def __init__(self, response: object) -> None: + super().__init__() + self._response: Final = response + self.seen: Final[list[object]] = [] # mutable-ok: filled as the events log + + async def log_success_fallback_event( + self, original_model_group: str, kwargs: dict, original_exception: Exception + ) -> None: + self.seen.append(headers(self._response).get(FALLBACK_COUNT)) + + +async def test_a_fallback_counts_itself_before_its_success_is_logged( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + """``run_async_fallback`` stamps the response, then awaits ``log_success_fallback_event``.""" + answered: Final = completion_response() + logger: Final = _FallbackCountAtLog(answered) + monkeypatch.setattr(litellm, "callbacks", [logger]) + host.answer("openai/a1", server_error()) + host.answer("openai/b1", answered) + core: Final = make_core( + model_list=[deployment("a", "a1"), deployment("b", "b1")], num_retries=0, fallbacks=[{"a": ["b"]}] + ) + _ = await complete(core, "a") + assert logger.seen == [1] + + +# --- guardrails ------------------------------------------------------------------------------------ + + +async def test_a_guardrail_result_carries_the_router_headers(make_core: MakeCore) -> None: + answered: Final = completion_response() + + async def guard(**_kwargs: object) -> object: # kwargs-ok: the guardrail's keyword payload + return answered + + core: Final = make_core(model_list=[deployment("g", "d1")], guardrail_list=GUARDRAILS) + result: Final = await core.aguardrail(guardrail_name="gr", original_function=guard) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + assert headers(result) == {"x-litellm-model-group": "gr", RETRY_COUNT: 0, FALLBACK_COUNT: 0} + + +# --- the team filter ------------------------------------------------------------------------------- + + +# S2 and Y4 are ruled to differ: the Rust core filters every pick, so these run on it only. +def test_the_rust_core_filters_a_team_owned_deployment_on_a_sync_pick(host: ScriptedHost) -> None: + core: Final = CORE_BUILDERS["rust"](host, model_list=[deployment("own", "o1", model_info=TEAM_OWNED)], num_retries=0) + with pytest.raises(RouterRateLimitError): + _ = complete_sync(core, "own") + assert host.deployments_called("completion") == [] + + +async def test_an_async_pick_runs_the_team_filter(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("own", "o1", model_info=TEAM_OWNED)], num_retries=0) + with pytest.raises(RouterRateLimitError): + await complete(core, "own") + + +@pytest.mark.parametrize( + "request_fields", + [ + pytest.param({}, id="no-team"), + pytest.param({"metadata": {"user_api_key_team_id": "T2"}}, id="another-team"), + ], +) +async def test_the_rust_core_refuses_a_team_owned_deployment_named_by_its_id( + host: ScriptedHost, request_fields: Mapping[str, object] +) -> None: + core: Final = CORE_BUILDERS["rust"](host, model_list=[deployment("own", "o1", model_info=TEAM_OWNED)], num_retries=0) + with pytest.raises(RouterRateLimitError): + _ = await complete(core, "o1", **request_fields) + assert host.deployments_called("acompletion") == [] + + +# --- what the errors name -------------------------------------------------------------------------- + + +async def test_a_wildcard_refused_for_a_team_names_the_requested_model(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("anthropic/*", "w1", "anthropic/*", model_info=TEAM_OWNED)], num_retries=0 + ) + with pytest.raises(RouterRateLimitError) as refused: + await complete(core, "anthropic/claude-x", metadata={"user_api_key_team_id": "T2"}) + assert refused.value.model == "anthropic/claude-x" + + +def test_a_pass_through_refusal_names_the_group_an_alias_resolved_to(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("target", "t1")], num_retries=0, model_group_alias={"alias": "target"} + ) + with pytest.raises(BadRequestError) as refused: + core.get_available_deployment_for_pass_through("alias", []) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # the legacy member + assert refused.value.model == "target" + + +def test_a_sync_completion_is_refused_by_a_deployment_that_serves_no_chat(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment( + "g", "e1", model_info={"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]} + ) + ], + num_retries=0, + ) + with pytest.raises(EndpointNotSupportedByModelError): + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + + +# --- construction ---------------------------------------------------------------------------------- + + +def test_a_mocked_model_response_warns_once_per_dump_python_makes(make_core: MakeCore) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = make_core(model_list=[deployment("g", "d1", mock_response=BLOCKED_RESPONSE)], num_retries=0) + serializer: Final = [w for w in caught if "Pydantic serializer warnings" in str(w.message)] + assert len(serializer) == PYTHON_DUMPS_OF_A_DEPLOYMENT + diff --git a/tests/unit/test_router/core/test_parity_rulings_2026_11.py b/tests/unit/test_router/core/test_parity_rulings_2026_11.py new file mode 100644 index 00000000000..dd78224774c --- /dev/null +++ b/tests/unit/test_router/core/test_parity_rulings_2026_11.py @@ -0,0 +1,286 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on the router differences ruled on +2026-11-05, where both cores now take the side that was ruled correct: team ownership on the sync +and by-id paths, settings lists that change only through ``apply()``, one unreadable health record, +a failed ``set_model_list`` that keeps the old list, odd ``cooldown_time`` and status values, +numbers a deployment cannot be served with, an empty fallback entry, and a ``/v1/messages`` +stream that raises before its first frame. +""" + +import asyncio +import datetime +import time +from collections.abc import Awaitable, Callable, Mapping +from typing import Final, cast + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.utils import get_deployment_order +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGE_START, + MESSAGE_STOP, + MESSAGES, + anthropic_stream, + collect_bytes, + install_orjson_stub_if_missing, + text_delta, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] + + +def _cooled(core: RouterCore, *ids: str) -> dict[str, Mapping[str, object]]: # mutable-ok: a fresh dict + return dict(core.health().cooldowns.get_active_cooldowns(list(ids), None)) + + +def _sync(core: object, name: str) -> Callable[..., object]: + """A member both cores serve outside the ``RouterCore`` protocol (an endpoint or a deprecated name).""" + return cast("Callable[..., object]", getattr(core, name)) # cast-ok: the member is a plain function + + +def _async(core: object, name: str) -> Callable[..., Awaitable[object]]: + """``_sync`` for a coroutine member.""" + return cast("Callable[..., Awaitable[object]]", getattr(core, name)) # cast-ok: the member is a coroutine function + + +async def _on_failure(core: RouterCore, logged: Mapping[str, object]) -> None: + """``deployment_callback_on_failure``, awaited when the core answers a coroutine.""" + now: Final = datetime.datetime(2026, 1, 1) + outcome: Final = _sync(core, "deployment_callback_on_failure")(logged, None, now, now) + if asyncio.iscoroutine(outcome): + _ = cast("object", await outcome) # cast-ok: the callback's verdict, unused + await _settle() + + +def _ids(core: RouterCore) -> list[str]: # mutable-ok: a fresh list + return [str(entry["model_info"]["id"]) for entry in core.registry().all()] # pyright: ignore[reportAny] # model_list rows + + +async def _settle() -> None: + for _ in range(20): + await asyncio.sleep(0) + + +# --- team ownership -------------------------------------------------------------------------------- + +_TEAM_OWNED: Final = [deployment("own", "o1", model_info={"team_id": "T1"})] + + +@pytest.mark.parametrize("metadata", [{}, {"user_api_key_team_id": "T2"}], ids=["no-team", "another-team"]) +def test_a_sync_call_is_refused_a_deployment_another_team_owns( + make_core: MakeCore, host: ScriptedHost, metadata: dict[str, str] +) -> None: + core: Final = make_core(model_list=list(_TEAM_OWNED), num_retries=0) + + with pytest.raises(Exception, match=r"(?i)no (healthy )?deployments"): + _sync(core, "completion")(model="own", messages=MESSAGES, metadata=dict(metadata)) + + assert host.deployments_called("completion") == [] + + +def test_a_sync_pick_skips_a_deployment_another_team_owns(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(_TEAM_OWNED), num_retries=0) + + with pytest.raises(Exception, match=r"(?i)no (healthy )?deployments"): + _sync(core, "get_available_deployment")(model="own", messages=MESSAGES, request_kwargs={}) + + +@pytest.mark.asyncio +async def test_a_pick_by_id_is_refused_another_teams_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "t1-dep", model_info={"team_id": "T1"})], num_retries=0) + + with pytest.raises(Exception, match="g"): + await _async(core, "acompletion")(model="t1-dep", messages=MESSAGES, metadata={"user_api_key_team_id": "T2"}) + + assert host.deployments_called("acompletion") == [] + + +# --- settings change only through apply() --------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_entry_appended_to_the_fallbacks_list_is_ignored(make_core: MakeCore, host: ScriptedHost) -> None: + listed: Final[list[object]] = [] # mutable-ok: the caller's list, edited after construction + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("fb", "f1")], num_retries=0, fallbacks=listed) + host.answer("openai/d1", rate_limit()) + listed.append({"g": ["fb"]}) + + with pytest.raises(litellm.RateLimitError): + await _async(core, "acompletion")(model="g", messages=MESSAGES) + + assert host.deployments_called("acompletion") == ["d1"] + + +# --- one unreadable health record ------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timestamp", ["soon", True, None], ids=["string", "bool", "none"]) +async def test_an_unreadable_health_timestamp_drops_only_that_record( + make_core: MakeCore, host: ScriptedHost, timestamp: object +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0), deployment("x", "x1")], + num_retries=0, + enable_health_check_routing=True, + ) + states: Final = { + "d1": {"is_healthy": False, "timestamp": time.time()}, + "x1": {"is_healthy": False, "timestamp": timestamp}, + } + health_checks: Final = core.health().health_checks + # As another writer left it in the shared cache, then merged by this router's own writer. + _sync(health_checks.cache, "set_cache")(key=health_checks.CACHE_KEY, value=states) + await _async(core, "acompletion")(model="g", messages=MESSAGES) + health_checks.set_deployment_health_states(states) # pyright: ignore[reportArgumentType] # a record with a bad timestamp, on purpose + await _async(core, "acompletion")(model="g", messages=MESSAGES) + + assert host.deployments_called("acompletion") == ["d2", "d2"] + + +# --- a failed set_model_list ------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_failed_set_model_list_keeps_the_old_list(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("a", "d-a"), deployment("b", "d-b")], num_retries=0) + unservable: Final = deployment("e", "d-e", input_cost_per_token=float("inf")) + + with pytest.raises(ValueError, match="cannot be served"): + _sync(core, "set_model_list")([deployment("c", "d-c"), unservable]) + + assert sorted(_ids(core)) == ["d-a", "d-b"] + await _async(core, "acompletion")(model="a", messages=MESSAGES) + assert host.deployments_called("acompletion") == ["d-a"] + + +# --- odd cooldown_time and status values ------------------------------------------------------------ + + +@pytest.mark.parametrize("value", [True, False], ids=["true", "false"]) +def test_a_bool_cooldown_time_is_refused_at_construction(make_core: MakeCore, value: bool) -> None: + with pytest.raises(TypeError, match="cooldown_time"): + make_core(model_list=[deployment("g", "d1")], cooldown_time=value) + + +# A group of one is never cooled down, so these groups hold two. +_TWO: Final = [deployment("g", "d1"), deployment("g", "d2")] + + +def _failure( + deployment_id: str, exception: Exception, **litellm_params: object +) -> dict[str, object]: # mutable-ok: logging kwargs + return { + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": deployment_id}, **litellm_params}, + "call_type": "acompletion", + "exception": exception, + } + + +@pytest.mark.asyncio +async def test_a_bool_cooldown_time_on_the_call_cools_for_the_router_default(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(_TWO), num_retries=0, allowed_fails=0, cooldown_time=60) + + await _on_failure(core, _failure("d1", rate_limit(), cooldown_time=True)) + + assert _cooled(core, "d1")["d1"]["cooldown_time"] == 60 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [70000, True, -1], ids=["too-wide", "bool", "negative"]) +async def test_a_status_outside_http_is_recorded_as_500(make_core: MakeCore, status: object) -> None: + core: Final = make_core(model_list=list(_TWO), num_retries=0, allowed_fails=0) + error: Final = rate_limit() + error.status_code = status # pyright: ignore[reportAttributeAccessIssue] # a provider error with an odd status + + await _on_failure(core, _failure("d1", error)) + + assert str(_cooled(core, "d1")["d1"]["status_code"]) == "500" + + +# --- numbers a deployment cannot be served with ----------------------------------------------------- + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("input_cost_per_token", float("inf")), + ("output_cost_per_token", float("nan")), + ("input_cost_per_token", "1e400"), + ], + ids=["inf", "nan", "overflowing-string"], +) +def test_a_non_finite_cost_is_refused_by_name(make_core: MakeCore, field: str, value: object) -> None: + with pytest.raises(ValueError, match=f"Deployment 'g' cannot be served: .*{field}.*finite costs"): + make_core(model_list=[deployment("g", "d1", model_info={field: value})]) + + +def test_a_ptu_share_wider_than_64_bits_is_refused_at_build_and_on_add(make_core: MakeCore) -> None: + wide: Final = deployment("g", "sh", model_info={"db_model": True, "ptu_shares": {"T1": 10**20}}) + with pytest.raises(ValueError, match=r"model_info\.ptu_shares\.T1 is 100000000000000000000, outside"): + make_core(model_list=[wide]) + + core: Final = make_core(model_list=[deployment("g", "free")]) + with pytest.raises(ValueError, match=r"ptu_shares\.T1 .* outside"): + _sync(core, "add_deployment")( + litellm.Deployment.model_validate( + deployment("g", "late", model_info={"db_model": True, "ptu_shares": {"T1": 10**20}}) + ) + ) + assert _ids(core) == ["free"] + + +# --- an empty fallback entry -------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_an_empty_fallback_entry_is_skipped(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1"), deployment("fb", "f1")], num_retries=0) + host.answer("openai/d1", rate_limit()) + + await _async(core, "acompletion")(model="g", messages=MESSAGES, fallbacks=[{}, {"g": ["fb"]}]) + + assert host.deployments_called("acompletion") == ["d1", "f1"] + + +# --- a deployment's order -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("order", "expected"), + [(1, 1), ("2", 2), (" 3", None), (2.0, 2), (2.5, None), (True, 1), ("x", None), (None, None)], + ids=["int", "digits", "padded", "whole-float", "fraction", "bool", "word", "unset"], +) +def test_a_deployment_order_reads_as_a_whole_number(order: object, expected: int | None) -> None: + assert get_deployment_order({"litellm_params": {"order": order}}) == expected + + +# --- a /v1/messages stream that raises before its first frame ---------------------------------------- + + +@pytest.mark.asyncio +async def test_a_messages_stream_raising_before_its_first_frame_cools_and_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + frames: Final = [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + host.answer("openai/d1", anthropic_stream(error=rate_limit(message="provider text"))) + host.answer("openai/d2", anthropic_stream(*frames)) + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0)], + num_retries=1, + retry_after=0, + allowed_fails=0, + ) + + stream: Final = await _async(core, "aanthropic_messages")(model="g", messages=MESSAGES, max_tokens=5, stream=True) + assert await collect_bytes(stream) == frames + await _settle() + + assert host.deployments_called("anthropic_messages") == ["d1", "d2"] + assert sorted(_cooled(core, "d1", "d2")) == ["d1"] + assert "provider text" in str(_cooled(core, "d1")["d1"]["exception_received"]) diff --git a/tests/unit/test_router/core/test_registry_changesets.py b/tests/unit/test_router/core/test_registry_changesets.py index d4aa4d14a75..c889255a5a6 100644 --- a/tests/unit/test_router/core/test_registry_changesets.py +++ b/tests/unit/test_router/core/test_registry_changesets.py @@ -636,7 +636,8 @@ def _registered(host: ScriptedHost) -> Mapping[str, int]: def test_retire_releases_every_callback_the_core_registered_on_its_host( make_core: MakeCore, host: ScriptedHost ) -> None: - core = make_core(model_list=_model_list()) + # least-busy registers its selector on the host on both cores; the Python core adds its outcome callbacks. + core = make_core(model_list=_model_list(), routing_strategy="least-busy") assert _registered(host) != {} core.retire() @@ -645,8 +646,8 @@ def test_retire_releases_every_callback_the_core_registered_on_its_host( def test_retire_leaves_another_cores_callbacks_registered(make_core: MakeCore, host: ScriptedHost) -> None: - kept = make_core(model_list=_model_list()) - retired = make_core(model_list=[deployment("other", "o")]) + kept = make_core(model_list=_model_list(), routing_strategy="least-busy") + retired = make_core(model_list=[deployment("other", "o")], routing_strategy="least-busy") registered_by_both = _registered(host) retired.retire() diff --git a/tests/unit/test_router/core/test_registry_characterization_construction.py b/tests/unit/test_router/core/test_registry_characterization_construction.py new file mode 100644 index 00000000000..7b80f6c3b71 --- /dev/null +++ b/tests/unit/test_router/core/test_registry_characterization_construction.py @@ -0,0 +1,379 @@ +""" +How the core builds its registry from ``model_list``: deployment ids, provider checks, environment +gating, ``os.environ/`` references, per-organization fan-out, wildcard and team registration, and +the pricing each deployment writes into ``litellm.model_cost``. +""" + +import pytest + +import litellm +from litellm.router_api import deployment_id +from tests.unit.test_router.core.registry_helpers import MakeCore, ids, team_deployment, unique_id +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment + + +def _bare( + model_name: str, model: str, **params: object +) -> dict[str, object]: # mutable-ok: a model_list entry # kwargs-ok: litellm_params + """A model_list entry with no ``model_info`` at all.""" + return {"model_name": model_name, "litellm_params": {"model": model, "api_key": "sk-test", **params}} + + +# -- deployment ids ----------------------------------------------------------------------------- + + +def test_a_deployment_without_an_id_gets_the_stable_hash_of_its_group_and_params(make_core: MakeCore) -> None: + core = make_core(model_list=[_bare("gpt", "openai/gpt-4o")]) + + expected = deployment_id("gpt", {"model": "openai/gpt-4o", "api_key": "sk-test"}) + assert core.registry().names(kind="ids") == [expected] + assert core.registry().has_deployment(expected) + + +def test_generated_ids_are_the_same_across_cores_built_from_the_same_list(make_core: MakeCore) -> None: + first = make_core(model_list=[_bare("gpt", "openai/gpt-4o")]) + second = make_core(model_list=[_bare("gpt", "openai/gpt-4o")]) + + assert first.registry().names(kind="ids") == second.registry().names(kind="ids") + + +def test_generated_ids_differ_when_the_group_or_params_differ(make_core: MakeCore) -> None: + core = make_core( + model_list=[_bare("gpt", "openai/gpt-4o"), _bare("other", "openai/gpt-4o"), _bare("gpt", "openai/gpt-4o-mini")] + ) + + assert len(set(core.registry().names(kind="ids"))) == 3 + + +def test_a_declared_non_string_id_is_kept_as_a_string(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + {"model_name": "gpt", "litellm_params": {"model": "openai/x", "api_key": "k"}, "model_info": {"id": 1}} + ] + ) + + assert core.registry().names(kind="ids") == ["1"] + found = core.registry().deployment("1") + assert found is not None + assert found.model_info.id == "1" + + +def test_two_deployments_declaring_one_id_are_both_kept_and_the_id_finds_the_last(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "d"), deployment("emb", "d")]) + + assert [entry["model_name"] for entry in core.registry().all()] == ["gpt", "emb"] + assert core.registry().index("model_name") == {"gpt": [0], "emb": [1]} + found = core.registry().deployment("d") + assert found is not None + assert found.model_name == "emb" + + +# -- the stored entries ------------------------------------------------------------------------- + + +def test_the_stored_entry_carries_litellm_param_defaults_and_model_info_flags(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + (entry,) = core.registry().all() + assert entry["model_name"] == "gpt" + assert entry["litellm_params"]["model"] == "openai/a" + assert entry["litellm_params"]["api_key"] == "sk-test" + assert entry["model_info"] == {"id": "a", "db_model": False, "member_auto_router": False} + + +def test_top_level_rpm_and_tpm_are_copied_into_litellm_params(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + { + "model_name": "gpt", + "litellm_params": {"model": "openai/x", "api_key": "k"}, + "model_info": {"id": "r"}, + "rpm": 7, + "tpm": 9, + } + ] + ) + + (entry,) = core.registry().all() + assert (entry["litellm_params"]["rpm"], entry["litellm_params"]["tpm"]) == (7, 9) + assert (entry["rpm"], entry["tpm"]) == (7, 9) + + +def test_os_environ_references_in_litellm_params_are_resolved( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("REGISTRY_CHAR_BASE", "https://env.example") + monkeypatch.setenv("REGISTRY_CHAR_KEY", "sk-from-env") + core = make_core( + model_list=[ + deployment("gpt", "a", api_key="os.environ/REGISTRY_CHAR_KEY", api_base="os.environ/REGISTRY_CHAR_BASE") + ] + ) + + params = core.registry().all()[0]["litellm_params"] + assert (params["api_key"], params["api_base"]) == ("sk-from-env", "https://env.example") + + +@pytest.mark.asyncio +async def test_resolved_environment_credentials_are_sent_to_the_provider( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("REGISTRY_CHAR_BASE", "https://env.example") + core = make_core(model_list=[deployment("gpt", "a", api_base="os.environ/REGISTRY_CHAR_BASE")]) + + await core.acompletion(model="gpt", messages=[{"role": "user", "content": "hi"}]) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert [call.kwargs.get("api_base") for call in host.calls_to("acompletion")] == ["https://env.example"] + + +def test_a_list_of_organizations_fans_out_into_one_deployment_per_organization(make_core: MakeCore) -> None: + core = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])]) + + entries = core.registry().all() + assert [entry["litellm_params"]["organization"] for entry in entries] == ["org-1", "org-2"] + assert len(set(ids(entries))) == 1 + assert core.registry().index("model_name") == {"gpt": [0, 1]} + + +def test_deployment_names_list_each_deployments_litellm_model(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b"), deployment("emb", "e")]) + + assert core.registry().names(kind="deployment_names") == ["openai/a", "openai/b", "openai/e"] + + +def test_every_configured_litellm_model_starts_with_zero_latency(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]) + + assert core.health().latency == {"openai/a": 0, "openai/b": 0} + + +@pytest.mark.parametrize("model_list", [None, []], ids=["none", "empty"]) +def test_a_core_without_deployments_has_an_empty_registry(make_core: MakeCore, model_list: object) -> None: + core = make_core(model_list=model_list) + + assert core.registry().all() == [] + assert core.registry().names(kind="ids") == [] + assert core.registry().names(kind="group_set") == set() + assert core.registry().deployments() == [] + + +# -- invalid deployments ------------------------------------------------------------------------ + + +def test_an_unknown_provider_raises_bad_request_at_construction(make_core: MakeCore) -> None: + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): + make_core(model_list=[deployment("gpt", "a", model="notaprovider/model")]) + + +def test_ignore_invalid_deployments_drops_the_invalid_one_and_keeps_the_rest(make_core: MakeCore) -> None: + core = make_core( + model_list=[deployment("gpt", "a", model="notaprovider/model"), deployment("gpt", "b")], + ignore_invalid_deployments=True, + ) + + assert core.registry().names(kind="ids") == ["b"] + + +def test_an_invalid_tag_regex_raises_value_error_naming_the_model(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match=r"Invalid regex in tag_regex for model 'gpt': '\('"): + make_core(model_list=[deployment("gpt", "a", tag_regex=["("])]) + + +def test_ignore_invalid_deployments_drops_a_deployment_with_an_invalid_tag_regex(make_core: MakeCore) -> None: + core = make_core( + model_list=[deployment("gpt", "a", tag_regex=["("]), deployment("gpt", "b", tag_regex=["^ok$"])], + ignore_invalid_deployments=True, + ) + + assert core.registry().names(kind="ids") == ["b"] + + +# -- environment-gated deployments -------------------------------------------------------------- + + +def test_a_deployment_for_another_environment_is_left_out(make_core: MakeCore, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_ENVIRONMENT", "staging") + core = make_core( + model_list=[ + deployment("prod-only", "a", model_info={"supported_environments": ["production"]}), + deployment("both", "b", model_info={"supported_environments": ["staging", "production"]}), + ] + ) + + assert core.registry().names(kind="ids") == ["b"] + assert core.registry().deployments("prod-only") == [] + + +def test_a_left_out_environment_deployment_still_names_its_group( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + """Odd but current: the group set is read from the given model_list, not from what was added.""" + monkeypatch.setenv("LITELLM_ENVIRONMENT", "staging") + core = make_core( + model_list=[ + deployment("prod-only", "a", model_info={"supported_environments": ["production"]}), + deployment("h", "b"), + ] + ) + + assert core.registry().names(kind="group_set") == {"prod-only", "h"} + assert core.registry().index("model_name") == {"h": [0]} + + +def test_supported_environments_without_litellm_environment_raises( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("LITELLM_ENVIRONMENT", raising=False) + + with pytest.raises(ValueError, match="Set 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set"): + make_core(model_list=[deployment("gpt", "a", model_info={"supported_environments": ["production"]})]) + + +@pytest.mark.parametrize( + ("environment", "supported", "message"), + [ + ("bogus", ["production"], "LITELLM_ENVIRONMENT must be one of"), + ("staging", ["prod"], "supported_environments must be one of"), + ("staging", "production", "supported_environments must be a list of"), + ], + ids=["unknown_environment", "unknown_supported_value", "not_a_list"], +) +def test_invalid_environment_configuration_raises( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, environment: str, supported: object, message: str +) -> None: + monkeypatch.setenv("LITELLM_ENVIRONMENT", environment) + + with pytest.raises(ValueError, match=message): + make_core(model_list=[deployment("gpt", "a", model_info={"supported_environments": supported})]) + + +def test_ignore_invalid_deployments_drops_an_invalid_environment_deployment( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("LITELLM_ENVIRONMENT", raising=False) + core = make_core( + model_list=[ + deployment("gpt", "a", model_info={"supported_environments": ["production"]}), + deployment("gpt", "b"), + ], + ignore_invalid_deployments=True, + ) + + assert core.registry().names(kind="ids") == ["b"] + + +@pytest.mark.parametrize( + ("model_info", "active"), + [ + ({}, True), + ({"supported_environments": ["development"]}, True), + ({"supported_environments": ["production"]}, False), + ], + ids=["unset", "matching", "other"], +) +def test_deployment_is_active_for_environment_follows_litellm_environment( + make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, model_info: dict[str, object], active: bool +) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + monkeypatch.setenv("LITELLM_ENVIRONMENT", "development") + candidate = litellm.types.router.Deployment(**deployment("gpt", "z", model_info=model_info)) # pyright: ignore[reportArgumentType] # a model_list entry + + assert core.deployment_is_active_for_environment(candidate) is active # pyright: ignore[reportAttributeAccessIssue] # public on the core + + +# -- wildcard and team registration ------------------------------------------------------------- + + +def test_a_wildcard_group_registers_a_pattern_and_a_provider_default_deployment(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment("openai/*", "w", model="openai/*"), + deployment("anthropic/*", "x", model="anthropic/*"), + deployment("gpt", "a"), + ] + ) + + assert core.settings().provider_default_deployment_ids == ["w", "x"] + assert ids(core.registry().deployments("openai/gpt-4o")) == ["w"] + assert ids(core.registry().deployments("anthropic/claude-x")) == ["x"] + + +def test_team_deployments_are_indexed_by_team_and_public_name(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + team_deployment("internal-1", "t1", "team-a", "pub"), + team_deployment("internal-2", "t2", "team-a", "pub"), + team_deployment("internal-3", "t3", "team-b", "other"), + deployment("gpt", "g"), + ] + ) + + assert core.registry().index("team_model") == {("team-a", "pub"): [0, 1], ("team-b", "other"): [2]} + assert core.registry().names(kind="team_public") == frozenset({"pub", "other"}) + assert core.registry().names(kind="team_public", team_id="team-a") == frozenset({"pub"}) + assert core.registry().names(kind="team_public", team_id="nobody") == frozenset() + + +def test_a_team_deployment_needs_both_team_id_and_public_name_to_be_team_indexed(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment("gpt", "a", model_info={"team_id": "team-a"}), + deployment("gpt", "b", model_info={"team_public_model_name": "pub"}), + ] + ) + + assert core.registry().index("team_model") == {} + assert core.registry().names(kind="team_public") == frozenset() + + +# -- pricing written into litellm.model_cost ----------------------------------------------------- + + +def test_custom_pricing_is_registered_under_the_deployment_id(make_core: MakeCore) -> None: + model_id = unique_id() + make_core( + model_list=[ + deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3, output_cost_per_token=2e-3) + ] + ) + + entry = litellm.model_cost[model_id] + assert (entry["input_cost_per_token"], entry["output_cost_per_token"]) == (1e-3, 2e-3) + + +def test_custom_input_pricing_inherits_the_backend_models_cache_read_price(make_core: MakeCore) -> None: + model_id = unique_id() + make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3)]) + + builtin = litellm.get_model_info("openai/gpt-4o") + assert litellm.model_cost[model_id]["cache_read_input_token_cost"] == builtin["cache_read_input_token_cost"] + + +def test_a_user_set_cache_read_price_is_not_overwritten_by_inheritance(make_core: MakeCore) -> None: + model_id = unique_id() + make_core( + model_list=[ + deployment( + "gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3, cache_read_input_token_cost=7e-9 + ) + ] + ) + + assert litellm.model_cost[model_id]["cache_read_input_token_cost"] == 7e-9 + + +def test_a_deployment_without_custom_pricing_registers_no_prices_under_its_id(make_core: MakeCore) -> None: + model_id = unique_id() + make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o")]) + + entry = litellm.model_cost[model_id] + assert entry.get("input_cost_per_token") is None + assert entry.get("output_cost_per_token") is None + + +def test_model_info_fields_are_registered_under_the_deployment_id(make_core: MakeCore) -> None: + model_id = unique_id() + make_core(model_list=[deployment("gpt", model_id, model_info={"max_input_tokens": 1000, "mode": "embedding"})]) + + entry = litellm.model_cost[model_id] + assert (entry["max_input_tokens"], entry["mode"]) == (1000, "embedding") diff --git a/tests/unit/test_router/core/test_registry_characterization_lookups.py b/tests/unit/test_router/core/test_registry_characterization_lookups.py new file mode 100644 index 00000000000..19d58ff10d4 --- /dev/null +++ b/tests/unit/test_router/core/test_registry_characterization_lookups.py @@ -0,0 +1,682 @@ +""" +Registry reads: deployments by group, alias, wildcard and team; names and ids; name resolution; +access groups; blocked and unhealthy groups; credentials lookups. +""" + +import time + +import pytest + +from litellm.router_core import RouterCore +from tests.unit.test_router.core.registry_helpers import MakeCore, ids, names_and_ids, team_deployment, typed +from tests.unit.test_router.core.scripted_host import deployment + + +def _core(make_core: MakeCore) -> RouterCore: + """ + gpt (a, b), emb (e), the openai/* wildcard (w), team TA's ``pub`` (t1) and ``openai/*`` (tw), + team TB's ``pub`` (t2). Aliases: fast -> gpt, hid -> gpt (hidden), gpt -> emb. + """ + return make_core( + model_list=[ + deployment("gpt", "a", model_info={"access_groups": ["ag1", "ag2"]}), + deployment("gpt", "b", model_info={"access_groups": ["ag1"]}), + deployment("emb", "e", model="openai/text-embedding-3-small", model_info={"access_groups": ["ag2"]}), + deployment("openai/*", "w", model="openai/*", model_info={"access_groups": ["wild"]}), + team_deployment("int-a", "t1", "TA", "pub", model="openai/team-model"), + team_deployment("int-b", "t2", "TB", "pub", model="anthropic/claude-x"), + team_deployment("team-wild", "tw", "TA", "openai/*", model="openai/*"), + ], + model_group_alias={"fast": "gpt", "hid": {"model": "gpt", "hidden": True}, "gpt": "emb"}, + ) + + +# -- registry().deployments(): the group listing ------------------------------------------------ + + +def test_listing_every_deployment_puts_visible_alias_rows_before_the_model_list(make_core: MakeCore) -> None: + listed = _core(make_core).registry().deployments() + + assert names_and_ids(listed) == [ + ("fast", "a"), + ("fast", "b"), + ("gpt", "e"), + ("gpt", "a"), + ("gpt", "b"), + ("emb", "e"), + ("openai/*", "w"), + ("int-a", "t1"), + ("int-b", "t2"), + ("team-wild", "tw"), + ] + + +def test_an_alias_lists_its_targets_deployments_under_the_alias_name(make_core: MakeCore) -> None: + assert names_and_ids(_core(make_core).registry().deployments("fast")) == [("fast", "a"), ("fast", "b")] + + +def test_a_hidden_alias_is_listed_when_named(make_core: MakeCore) -> None: + assert names_and_ids(_core(make_core).registry().deployments("hid")) == [("hid", "a"), ("hid", "b")] + + +def test_a_group_that_is_also_an_alias_lists_its_own_deployments_then_the_alias_targets(make_core: MakeCore) -> None: + assert names_and_ids(_core(make_core).registry().deployments("gpt")) == [("gpt", "a"), ("gpt", "b"), ("gpt", "e")] + + +def test_an_alias_row_is_a_copy_and_leaves_the_configured_entry_named_as_configured(make_core: MakeCore) -> None: + core = _core(make_core) + + core.registry().deployments("fast") + + assert [entry["model_name"] for entry in core.registry().all()[:2]] == ["gpt", "gpt"] + + +def test_an_unknown_name_lists_nothing(make_core: MakeCore) -> None: + assert _core(make_core).registry().deployments("nope") == [] + + +def test_a_name_matching_only_a_wildcard_lists_the_wildcard_deployment_under_the_requested_name( + make_core: MakeCore, +) -> None: + assert names_and_ids(_core(make_core).registry().deployments("openai/gpt-x")) == [("openai/gpt-x", "w")] + + +def test_a_team_caller_gets_its_own_wildcard_deployment_after_the_global_one(make_core: MakeCore) -> None: + listed = _core(make_core).registry().deployments("openai/gpt-x", team_id="TA") + + assert names_and_ids(listed) == [("openai/gpt-x", "w"), ("openai/gpt-x", "tw")] + + +@pytest.mark.parametrize(("team_id", "expected"), [("TA", [("int-a", "t1")]), ("TB", [("int-b", "t2")]), (None, [])]) +def test_a_team_public_name_lists_only_that_teams_deployment( + make_core: MakeCore, team_id: str | None, expected: list[tuple[str, str]] +) -> None: + assert names_and_ids(_core(make_core).registry().deployments("pub", team_id=team_id)) == expected + + +@pytest.mark.parametrize(("team_id", "expected"), [(None, ["t1"]), ("TA", ["t1"]), ("TB", [])]) +def test_a_team_deployments_internal_name_lists_it_for_no_team_or_its_own_team_only( + make_core: MakeCore, team_id: str | None, expected: list[str] +) -> None: + assert ids(_core(make_core).registry().deployments("int-a", team_id=team_id)) == expected + + +def test_all_deployments_renames_rows_to_the_given_model_alias(make_core: MakeCore) -> None: + listed = _core(make_core).registry().deployments("gpt", source="all", model_alias="renamed") + + assert names_and_ids(listed) == [("renamed", "a"), ("renamed", "b")] + + +def test_all_deployments_follow_no_alias_and_no_wildcard(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.deployments("fast", source="all") == [] + assert registry.deployments("openai/gpt-x", source="all") == [] + + +def test_all_deployments_for_a_team_public_name_reads_the_team_index(make_core: MakeCore) -> None: + assert ids(_core(make_core).registry().deployments("pub", source="all", team_id="TA")) == ["t1"] + + +def test_named_deployments_are_the_group_index_entry_or_none(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert ids(registry.deployments("gpt", source="named")) == ["a", "b"] + assert registry.deployments("fast", source="named") is None + + +def test_first_in_group_is_the_first_configured_deployment_of_the_group(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + first = registry.deployments("gpt", source="first_in_group") + assert first is not None + assert first.model_info.id == "a" + assert registry.deployments("fast", source="first_in_group") is None + + +@pytest.mark.parametrize("source", ["all", "named", "wildcard", "routable", "first_in_group"]) +def test_a_source_that_needs_a_model_name_raises_without_one(make_core: MakeCore, source: str) -> None: + with pytest.raises(TypeError, match="needs a model_name"): + _core(make_core).registry().deployments(source=source) # pyright: ignore[reportCallIssue, reportArgumentType] # source is one of the literals + + +def test_an_unknown_deployment_source_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown deployment source 'bogus'"): + _core(make_core).registry().deployments("gpt", source="bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid source on purpose + + +# -- wildcards ------------------------------------------------------------------------------------ + + +@pytest.mark.parametrize(("team_id", "expected"), [("TA", ["tw"]), ("TB", ["w"]), (None, ["w"])]) +def test_wildcard_route_deployments_prefer_the_teams_own_pattern_over_the_global_one( + make_core: MakeCore, team_id: str | None, expected: list[str] +) -> None: + assert ids(_core(make_core).registry().deployments("openai/gpt-x", source="wildcard", team_id=team_id)) == expected + + +def test_wildcard_routed_models_name_the_requested_model(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.names("openai/gpt-x", kind="wildcard") == frozenset({"openai/gpt-x"}) + assert registry.names("anthropic/x", kind="wildcard") == frozenset() + + +# -- names ------------------------------------------------------------------------------------------ + + +def test_group_names_repeat_per_deployment_and_include_visible_aliases(make_core: MakeCore) -> None: + assert _core(make_core).registry().names() == ["fast", "fast", "gpt", "gpt", "gpt", "emb", "openai/*"] + + +def test_group_names_for_a_team_add_its_public_names(make_core: MakeCore) -> None: + assert _core(make_core).registry().names(team_id="TA") == [ + "fast", + "fast", + "gpt", + "gpt", + "gpt", + "emb", + "openai/*", + "pub", + "openai/*", + ] + + +def test_ids_can_leave_out_team_deployments(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.names(kind="ids") == ["a", "b", "e", "w", "t1", "t2", "tw"] + assert registry.names(kind="ids", exclude_team_models=True) == ["a", "b", "e", "w"] + + +def test_ids_of_a_group_read_only_the_group_index(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.names("fast", kind="ids") == [] + assert registry.names("pub", kind="ids") == [] + assert registry.names("int-a", kind="ids") == ["t1"] + assert registry.names("int-a", kind="ids", exclude_team_models=True) == [] + + +def test_litellm_models_follow_aliases_and_wildcards(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.names("fast", kind="litellm_models") == ("openai/a", "openai/b") + assert registry.names("openai/gpt-x", kind="litellm_models") == ("openai/gpt-x",) + assert registry.names("nope", kind="litellm_models") == () + + +def test_litellm_base_models_use_a_deployments_base_model(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment( + "gpt", "a", model="azure/my-deployment", api_base="https://x", model_info={"base_model": "azure/gpt-4o"} + ), + deployment("gpt", "b", model="openai/gpt-4o-mini"), + ] + ) + + assert core.registry().names("gpt", kind="litellm_base_models") == ("azure/gpt-4o", "openai/gpt-4o-mini") + assert core.registry().names("gpt", kind="litellm_models") == ("azure/my-deployment", "openai/gpt-4o-mini") + + +@pytest.mark.parametrize( + ("name", "team_id", "expected"), + [("fast", None, {"a", "b"}), ("openai/gpt-x", None, {"w"}), ("pub", "TA", {"t1"}), ("nope", None, set())], +) +def test_route_candidates_are_the_ids_a_request_for_the_name_may_reach( + make_core: MakeCore, name: str, team_id: str | None, expected: set[str] +) -> None: + assert _core(make_core).registry().names(name, kind="route_candidates", team_id=team_id) == expected + + +@pytest.mark.parametrize("kind", ["litellm_models", "wildcard", "route_candidates"]) +def test_a_name_kind_that_needs_a_model_name_raises_without_one(make_core: MakeCore, kind: str) -> None: + with pytest.raises(TypeError, match="needs a model_name"): + _core(make_core).registry().names(kind=kind) # pyright: ignore[reportCallIssue, reportArgumentType] # kind is one of the literals + + +# -- resolve ---------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "expected"), + [("fast", "gpt"), ("hid", "gpt"), ("gpt", "emb"), ("emb", "emb"), ("nope", "nope")], +) +def test_resolving_a_group_follows_one_alias_hop(make_core: MakeCore, name: str, expected: str) -> None: + assert _core(make_core).registry().resolve(name) == expected + + +@pytest.mark.parametrize(("name", "expected"), [("fast", "gpt"), ("hid", "gpt"), ("emb", None), ("pub", None)]) +def test_resolving_an_alias_reads_plain_and_hidden_aliases( + make_core: MakeCore, name: str, expected: str | None +) -> None: + assert _core(make_core).registry().resolve(name, kind="alias") == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("fast", True), + ("hid", True), + ("gpt", True), + ("a", True), + ("pub", False), + ("openai/gpt-x", False), + ("nope", False), + ], +) +def test_recognized_names_are_groups_aliases_and_deployment_ids(make_core: MakeCore, name: str, expected: bool) -> None: + assert _core(make_core).registry().resolve(name, kind="recognized") is expected + + +@pytest.mark.parametrize( + ("name", "team_id", "expected"), + [ + ("fast", None, "openai/a"), + ("gpt", None, "openai/text-embedding-3-small"), + ("pub", "TA", "openai/team-model"), + ("pub", None, None), + ("openai/gpt-x", None, None), + ("a", None, None), + ], +) +def test_upstream_is_the_first_routable_deployments_litellm_model( + make_core: MakeCore, name: str, team_id: str | None, expected: str | None +) -> None: + assert _core(make_core).registry().resolve(name, kind="upstream", team_id=team_id) == expected + + +def test_upstream_skips_a_blocked_deployment(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("half", "x2", model_info={"blocked": True}), deployment("half", "x3")]) + + assert core.registry().resolve("half", kind="upstream") == "openai/x3" + + +@pytest.mark.parametrize( + ("served", "expected"), [("gpt", "gpt"), ("emb", "fast"), (None, "fast")], ids=["target", "other", "none"] +) +def test_the_served_name_is_the_served_model_only_when_the_name_routes_to_it( + make_core: MakeCore, served: str | None, expected: str +) -> None: + assert _core(make_core).registry().resolve("fast", kind="served", served_model=served) == expected + + +def test_group_of_id_lists_the_deployments_group_including_alias_rows(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert names_and_ids(registry.resolve("a", kind="group_of_id")) == [("gpt", "a"), ("gpt", "b"), ("gpt", "e")] + assert registry.resolve("gpt", kind="group_of_id") is None + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("gpt", "gpt"), + ("a", "gpt"), + ("fast", "fast"), + ("nope", None), + (None, None), + ("", None), + ], +) +def test_from_id_resolves_a_group_name_or_deployment_id( + make_core: MakeCore, name: str | None, expected: str | None +) -> None: + assert _core(make_core).registry().resolve(name, kind="from_id") == expected + + +def test_from_id_matches_a_litellm_model_suffix_and_the_first_matching_row_wins(make_core: MakeCore) -> None: + """Odd but current: the alias row ``gpt -> emb`` is listed first, so ``emb``'s model resolves to ``gpt``.""" + registry = _core(make_core).registry() + + assert registry.resolve("text-embedding-3-small", kind="from_id") == "gpt" + assert registry.resolve("openai/text-embedding-3-small", kind="from_id") == "gpt" + + +def test_from_id_suffix_match_on_a_plain_core(make_core: MakeCore) -> None: + core = make_core( + model_list=[deployment("veo", "v", model="vertex_ai/veo-2.0", vertex_project="p", vertex_location="l")] + ) + + assert core.registry().resolve("veo-2.0", kind="from_id") == "veo" + + +def test_from_id_with_a_provider_returns_the_provider_prefixed_name_a_wildcard_serves(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.resolve("zzz", kind="from_id", custom_llm_provider="openai") == "openai/zzz" + assert registry.resolve("zzz", kind="from_id", custom_llm_provider="anthropic") is None + + +@pytest.mark.parametrize( + ("name", "team_id", "provider", "expected"), + [ + ("pub", "TA", None, "pub"), + ("gpt", "TA", None, None), + (None, "TA", None, "pub"), + (None, "TA", "openai", "pub"), + (None, "TB", "openai", None), + (None, "TB", "anthropic", "pub"), + (None, "nobody", None, None), + ], +) +def test_team_model_maps_a_teams_public_name_or_picks_its_first_deployment( + make_core: MakeCore, name: str | None, team_id: str, provider: str | None, expected: str | None +) -> None: + registry = _core(make_core).registry() + + assert registry.resolve(name, kind="team_model", team_id=team_id, custom_llm_provider=provider) == expected + + +def test_team_model_without_a_team_id_raises(make_core: MakeCore) -> None: + with pytest.raises(TypeError, match="needs a team_id"): + _core(make_core).registry().resolve("pub", kind="team_model") + + +def test_an_unknown_resolve_kind_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown resolve kind 'bogus'"): + _core(make_core).registry().resolve("gpt", kind="bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid kind on purpose + + +# -- includes ------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("team_id", "expected"), [(None, False), ("TA", True), ("TB", False)]) +def test_a_team_deployment_is_usable_only_by_its_own_team( + make_core: MakeCore, team_id: str | None, expected: bool +) -> None: + registry = _core(make_core).registry() + team_entry = registry.all()[4] + + assert registry.includes(team_entry, team_id=team_id) is expected + + +def test_a_shared_deployment_is_usable_by_any_team(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert all(registry.includes(registry.all()[0], team_id=team) for team in (None, "TA", "TB")) + + +@pytest.mark.parametrize( + ("model_name", "team_id", "expected"), + [("pub", "TA", True), ("pub", "TB", False), ("int-a", None, True), ("int-a", "TB", False), ("gpt", "TA", False)], +) +def test_includes_with_a_model_name_matches_the_public_or_internal_name( + make_core: MakeCore, model_name: str, team_id: str | None, expected: bool +) -> None: + registry = _core(make_core).registry() + + assert registry.includes(registry.all()[4], model_name=model_name, team_id=team_id) is expected + + +# -- access groups -------------------------------------------------------------------------------- + + +def test_access_groups_map_each_group_to_the_model_names_that_carry_it(make_core: MakeCore) -> None: + assert _core(make_core).registry().access_groups() == { + "ag1": ["fast", "fast", "gpt", "gpt"], + "ag2": ["fast", "gpt", "gpt", "emb"], + "wild": ["openai/*"], + } + + +def test_access_groups_of_one_model_read_its_listing(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert dict(registry.access_groups(model_name="gpt")) == {"ag1": ["gpt", "gpt"], "ag2": ["gpt", "gpt"]} + assert dict(registry.access_groups(model_name="openai/gpt-x")) == {"wild": ["openai/gpt-x"]} + assert dict(registry.access_groups(model_name="nope")) == {} + + +def test_access_groups_filtered_to_one_group(make_core: MakeCore) -> None: + assert dict(_core(make_core).registry().access_groups(model_access_group="ag2")) == { + "ag2": ["fast", "gpt", "gpt", "emb"] + } + + +def test_access_groups_follow_a_deployment_added_later(make_core: MakeCore) -> None: + core = _core(make_core) + assert "late" not in core.registry().access_groups() + + core.add_deployment( # pyright: ignore[reportAttributeAccessIssue] # public on the core + typed("new", "n", model_info={"access_groups": ["late"]}) + ) + + assert core.registry().access_groups()["late"] == ["new"] + + +@pytest.mark.parametrize(("group", "expected"), [("wild", True), ("ag1", False), ("nope", False)]) +def test_an_access_group_is_a_wildcard_route_when_one_of_its_models_is_a_pattern( + make_core: MakeCore, group: str, expected: bool +) -> None: + assert _core(make_core).registry().resolve(group, kind="wildcard_access_group") is expected + + +# -- blocked and unhealthy groups ----------------------------------------------------------------- + + +def _blocked_core(make_core: MakeCore) -> RouterCore: + return make_core( + model_list=[ + deployment("blk", "x1", model_info={"blocked": True}), + deployment("half", "x2", model_info={"blocked": True}), + deployment("half", "x3"), + deployment("ok", "x4"), + ], + model_group_alias={"blk-alias": "blk"}, + ) + + +def test_fully_blocked_names_are_groups_whose_every_deployment_is_blocked(make_core: MakeCore) -> None: + assert _blocked_core(make_core).health().fully_blocked_model_names() == {"blk", "blk-alias"} + + +def test_is_blocked_and_all_blocked_read_model_info_blocked(make_core: MakeCore) -> None: + core = _blocked_core(make_core) + health = core.health() + entries = core.registry().all() + + blocked = core.registry().deployment("x1") + open_ = core.registry().deployment("x3") + assert blocked is not None + assert open_ is not None + assert (health.is_blocked(blocked), health.is_blocked(open_)) == (True, False) + assert health.all_blocked(entries[:1]) is True + assert health.all_blocked(entries[1:3]) is False + assert health.all_blocked([]) is False + + +@pytest.mark.asyncio +async def test_fully_unhealthy_names_are_groups_whose_every_deployment_failed_its_health_check( + make_core: MakeCore, +) -> None: + core = make_core( + model_list=[ + deployment("sick", "s1"), + deployment("half", "h1"), + deployment("half", "h2"), + team_deployment("int", "t1", "TA", "team-sick"), + ] + ) + now = time.time() + core.health().health_checks.set_deployment_health_states( + { + "s1": {"is_healthy": False, "timestamp": now, "reason": "down"}, + "h1": {"is_healthy": False, "timestamp": now, "reason": "down"}, + "h2": {"is_healthy": True, "timestamp": now, "reason": ""}, + "t1": {"is_healthy": False, "timestamp": now, "reason": "down"}, + } + ) + + assert await core.health().fully_unhealthy_model_names() == {"sick", "int", "team-sick"} + + +@pytest.mark.asyncio +async def test_no_health_state_hides_nothing(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + assert await core.health().fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_a_stale_unhealthy_state_hides_nothing(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + stale = time.time() - 10 * core.health().health_checks.staleness_threshold + core.health().health_checks.set_deployment_health_states( + {"a": {"is_healthy": False, "timestamp": stale, "reason": "x"}} + ) + + assert await core.health().fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_an_allowed_fails_policy_hides_no_unhealthy_group(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")], allowed_fails_policy={"RateLimitErrorAllowedFails": 2}) + core.health().health_checks.set_deployment_health_states( + {"a": {"is_healthy": False, "timestamp": time.time(), "reason": "x"}} + ) + + assert await core.health().fully_unhealthy_model_names() == set() + + +# -- credentials ------------------------------------------------------------------------------------ + + +def _credential_core(make_core: MakeCore) -> RouterCore: + return make_core( + model_list=[ + deployment("gpt", "a", api_base="https://a.example", temperature=0.2), + deployment("plain", "p", model="gpt-4o-mini"), + deployment("claude", "c", model="claude-3-haiku", custom_llm_provider="anthropic"), + deployment("blk", "x", model_info={"blocked": True}), + deployment("openai/*", "w", model="openai/*"), + team_deployment("shared", "ta", "TA", "team-name", api_key="sk-team-a"), + team_deployment("team-wild", "tw", "TA", "openai/*", model="openai/*", api_key="sk-team-wild"), + ] + ) + + +def test_credentials_are_the_credential_params_only(make_core: MakeCore) -> None: + assert _credential_core(make_core).registry().credentials("a") == { + "api_key": "sk-test", + "api_base": "https://a.example", + } + + +def test_credentials_of_an_unknown_or_blocked_deployment_are_none(make_core: MakeCore) -> None: + registry = _credential_core(make_core).registry() + + assert registry.credentials("missing") is None + assert registry.credentials("x") is None + assert registry.credentials("x", form="with_provider") is None + assert registry.credentials("x", form="deployment") is None + + +@pytest.mark.parametrize( + ("lookup", "provider"), + [("a", "openai"), ("p", "openai"), ("c", "anthropic")], + ids=["prefix", "default", "explicit"], +) +def test_credentials_with_provider_add_the_model_and_its_provider( + make_core: MakeCore, lookup: str, provider: str +) -> None: + registry = _credential_core(make_core).registry() + + found = registry.credentials(lookup, form="with_provider") + deployment_found = registry.deployment(lookup) + assert found is not None + assert deployment_found is not None + assert (found["model"], found["custom_llm_provider"]) == (deployment_found.litellm_params.model, provider) + + +@pytest.mark.parametrize( + ("lookup", "team_id", "expected"), + [ + ("a", None, "a"), + ("gpt", None, "a"), + ("team-name", "TA", "ta"), + ("team-name", None, None), + ("shared", "TB", None), + ("shared", "TA", "ta"), + ("openai/gpt-x", None, "w"), + ("openai/gpt-x", "TA", "tw"), + ("openai/gpt-x", "TB", "w"), + ("nope", None, None), + ], +) +def test_the_credential_deployment_is_found_by_id_group_team_name_then_wildcard( + make_core: MakeCore, lookup: str, team_id: str | None, expected: str | None +) -> None: + found = _credential_core(make_core).registry().credentials(lookup, form="deployment", team_id=team_id) + + assert (None if found is None else found.model_info.id) == expected + + +def test_credentials_with_provider_for_a_wildcard_name_carry_the_requested_model(make_core: MakeCore) -> None: + found = _credential_core(make_core).registry().credentials("openai/gpt-x", form="with_provider", team_id="TA") + + assert found is not None + assert (found["api_key"], found["model"], found["custom_llm_provider"]) == ( + "sk-team-wild", + "openai/gpt-x", + "openai", + ) + + +def test_an_unknown_credential_form_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown credential form 'bogus'"): + _credential_core(make_core).registry().credentials("a", form="bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid form on purpose + + +# -- single deployments and the raw list ------------------------------------------------------------ + + +def test_deployment_dict_is_the_live_model_list_entry(make_core: MakeCore) -> None: + registry = _core(make_core).registry() + + assert registry.deployment_dict("a") is registry.all()[0] + assert registry.deployment_dict("missing") is None + + +def test_a_looked_up_deployment_is_a_copy(make_core: MakeCore) -> None: + core = _core(make_core) + found = core.registry().deployment("a") + assert found is not None + + found.litellm_params.api_base = "https://mutated.example" + + assert core.registry().all()[0]["litellm_params"].get("api_base") is None + + +def test_a_visible_dict_alias_is_listed_like_a_plain_alias(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")], model_group_alias={"vis": {"model": "gpt", "hidden": False}}) + + assert names_and_ids(core.registry().deployments()) == [("vis", "a"), ("gpt", "a")] + assert core.registry().names() == ["vis", "gpt"] + + +def test_the_deprecated_generate_model_id_is_the_registry_id_hash(make_core: MakeCore) -> None: + from litellm.router_api import deployment_id + + core = _core(make_core) + + with pytest.warns(DeprecationWarning, match="generate_model_id is deprecated"): + generated = core.generate_model_id("g", {"model": "x"}) # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + assert generated == deployment_id("g", {"model": "x"}) + + +def test_assigning_model_list_swaps_the_list_without_reindexing(make_core: MakeCore) -> None: + """Odd but current: the id index still maps ``a`` to position 0, which now holds the new entry.""" + core = make_core(model_list=[deployment("gpt", "a")]) + version = core.registry().version + + core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable + + assert core.registry().version == version + 1 + assert core.registry().names(kind="ids") == ["a"] + found = core.registry().deployment("a") + assert found is not None + assert found.model_name == "n" diff --git a/tests/unit/test_router/core/test_registry_characterization_model_info.py b/tests/unit/test_router/core/test_registry_characterization_model_info.py new file mode 100644 index 00000000000..8d287ab46a2 --- /dev/null +++ b/tests/unit/test_router/core/test_registry_characterization_model_info.py @@ -0,0 +1,580 @@ +""" +What the registry knows about models: group info, deployment and router model info, configured +limits/mode/display name/service tiers/listing info, group usage this minute, remaining usage and +the rate limit headers ``set_response_headers`` attaches. +""" + +import pytest + +import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.router_core import RouterCore +from litellm.types.router import Deployment, RouterCacheEnum +from litellm.utils import get_utc_datetime +from tests.unit.test_router.core.registry_helpers import MakeCore, typed, unique_id +from tests.unit.test_router.core.scripted_host import completion_response, deployment + + +def _usage_core(make_core: MakeCore) -> RouterCore: + """gpt: a (tpm 100, rpm 10) and b (top-level tpm 1000, model_info tpm 50 / rpm 5); io: i (itpm 300, otpm 40).""" + return make_core( + model_list=[ + deployment("gpt", "a", model="openai/gpt-4o", tpm=100, rpm=10), + { + "model_name": "gpt", + "litellm_params": {"model": "anthropic/claude-3-haiku-20240307", "api_key": "k"}, + "model_info": {"id": "b", "tpm": 50, "rpm": 5, "description": "desc b"}, + "tpm": 1000, + }, + deployment("io", "i", model="openai/gpt-4o", itpm=300, otpm=40), + deployment("plain", "p", model="openai/gpt-4o"), + ], + model_group_alias={"hid": {"model": "gpt", "hidden": True}, "al": "gpt"}, + ) + + +def _minute() -> str: + return get_utc_datetime().strftime("%H-%M") + + +async def _count(core: RouterCore, kind: RouterCacheEnum, deployment_id: str, model: str, value: int) -> None: + key = kind.value.format(id=deployment_id, model=model, current_minute=_minute()) + await core.health().store.async_set_cache(key, value) + + +# -- model group info ----------------------------------------------------------------------------- + + +def test_group_info_merges_providers_limits_and_description_across_deployments(make_core: MakeCore) -> None: + info = _usage_core(make_core).registry().group("gpt") + + assert info is not None + assert info.model_group == "gpt" + assert info.providers == ["openai", "anthropic"] + assert info.description == "desc b" + assert info.mode == "chat" + builtin = litellm.get_model_info("openai/gpt-4o") + assert info.max_input_tokens == builtin["max_input_tokens"] + assert info.input_cost_per_token == builtin["input_cost_per_token"] + + +def test_group_info_sums_tpm_and_rpm_with_top_level_values_before_model_info(make_core: MakeCore) -> None: + info = _usage_core(make_core).registry().group("gpt") + + assert info is not None + assert (info.tpm, info.rpm) == (1100, 15) + + +def test_group_info_sums_itpm_and_otpm(make_core: MakeCore) -> None: + info = _usage_core(make_core).registry().group("io") + + assert info is not None + assert (info.itpm, info.otpm, info.tpm, info.rpm) == (300, 40, None, None) + + +def test_group_info_through_an_alias_is_reported_under_the_alias(make_core: MakeCore) -> None: + info = _usage_core(make_core).registry().group("al") + + assert info is not None + assert (info.model_group, info.tpm) == ("al", 1100) + + +def test_group_info_of_a_hidden_alias_needs_include_hidden(make_core: MakeCore) -> None: + registry = _usage_core(make_core).registry() + + assert registry.group("hid") is None + shown = registry.group("hid", include_hidden=True) + assert shown is not None + assert shown.model_group == "hid" + + +def test_group_info_of_an_unknown_name_is_none(make_core: MakeCore) -> None: + assert _usage_core(make_core).registry().group("nope") is None + + +def test_group_info_of_a_name_a_wildcard_serves_reads_the_requested_model(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("openai/*", "w", model="openai/*")]) + + info = core.registry().group("openai/gpt-4o-mini") + + assert info is not None + assert (info.model_group, info.providers) == ("openai/gpt-4o-mini", ["openai"]) + assert info.max_input_tokens == litellm.get_model_info("openai/gpt-4o-mini")["max_input_tokens"] + + +def test_group_info_of_an_unmapped_model_uses_its_configured_mode_and_costs(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment( + "unm", + unique_id(), + model="openai/totally-unmapped-registry-model", + model_info={"mode": "embedding", "input_cost_per_token": "0.5"}, + ) + ] + ) + + info = core.registry().group("unm") + + assert info is not None + assert (info.mode, info.input_cost_per_token, info.providers) == ("embedding", 0.5, ["openai"]) + assert info.max_input_tokens is None + + +def test_group_info_reports_configurable_clientside_auth_params(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a", configurable_clientside_auth_params=["api_base"])]) + + info = core.registry().group("gpt") + + assert info is not None + assert info.configurable_clientside_auth_params == ["api_base"] + + +def test_cached_group_info_equals_group_info(make_core: MakeCore) -> None: + registry = _usage_core(make_core).registry() + + assert registry.group("gpt", field="cached_info") == registry.group("gpt") + + +def test_cached_group_info_follows_a_deployment_added_later(make_core: MakeCore) -> None: + core = _usage_core(make_core) + before = core.registry().group("gpt", field="cached_info") + assert before is not None + + core.add_deployment(typed("gpt", "c", tpm=1)) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + after = core.registry().group("gpt", field="cached_info") + assert after is not None + assert (before.tpm, after.tpm) == (1100, 1101) + + +def test_an_unknown_group_field_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown group field 'bogus'"): + _usage_core(make_core).registry().group("gpt", field="bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid field on purpose + + +# -- configured values of a group ----------------------------------------------------------------- + + +def _configured_core(make_core: MakeCore) -> RouterCore: + return make_core( + model_list=[ + deployment( + "cfg", + "c1", + model="openai/gpt-4o", + model_info={ + "max_input_tokens": 1000, + "mode": "embedding", + "display_name": "Nice", + "service_tiers": ["flex"], + }, + ), + deployment( + "cfg", "c2", model="openai/gpt-4o-mini", model_info={"max_input_tokens": 5000, "max_output_tokens": 77} + ), + deployment("blank", "b1", model_info={"mode": " ", "display_name": ""}), + deployment("bad", "x1", model_info={"max_input_tokens": "lots"}), + deployment("openai/*", "w", model="openai/*"), + ], + model_group_alias={"cfg-alias": "cfg"}, + ) + + +def test_token_limits_come_from_the_first_deployment_of_the_group(make_core: MakeCore) -> None: + assert _configured_core(make_core).registry().group("cfg", field="token_limits") == (1000, None) + + +@pytest.mark.parametrize("name", ["nope", "openai/gpt-x", "cfg-alias"]) +def test_token_limits_of_an_unknown_wildcard_or_alias_name_are_none(make_core: MakeCore, name: str) -> None: + assert _configured_core(make_core).registry().group(name, field="token_limits") == (None, None) + + +def test_a_malformed_token_limit_reads_as_absent(make_core: MakeCore) -> None: + assert _configured_core(make_core).registry().group("bad", field="token_limits") == (None, None) + + +@pytest.mark.parametrize( + ("name", "mode", "display_name"), + [("cfg", "embedding", "Nice"), ("blank", None, None), ("nope", None, None), ("cfg-alias", None, None)], +) +def test_mode_and_display_name_are_the_first_deployments_non_blank_values( + make_core: MakeCore, name: str, mode: str | None, display_name: str | None +) -> None: + registry = _configured_core(make_core).registry() + + assert registry.group(name, field="mode") == mode + assert registry.group(name, field="display_name") == display_name + + +def test_listing_info_reports_the_widest_limits_and_each_backend_model_once(make_core: MakeCore) -> None: + info = _configured_core(make_core).registry().group("cfg", field="listing_info") + + assert info is not None + assert info.cost_map_keys == ("openai/gpt-4o", "openai/gpt-4o-mini") + assert (info.max_input_tokens, info.max_output_tokens) == (5000, 77) + + +@pytest.mark.parametrize("name", ["nope", "openai/gpt-x", "cfg-alias"]) +def test_listing_info_of_an_unknown_wildcard_or_alias_name_is_none(make_core: MakeCore, name: str) -> None: + assert _configured_core(make_core).registry().group(name, field="listing_info") is None + + +def test_service_tiers_are_each_routable_deployments_value_in_order(make_core: MakeCore) -> None: + registry = _configured_core(make_core).registry() + + assert registry.group("cfg", field="service_tiers") == (["flex"], None) + assert registry.group("cfg-alias", field="service_tiers") == (["flex"], None) + assert registry.group("nope", field="service_tiers") == () + + +def test_service_tiers_leave_out_blocked_deployments_and_other_teams(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment("st", "s1", model_info={"service_tiers": ["a"], "blocked": True}), + deployment("st", "s2", model_info={"service_tiers": ["b"]}), + deployment("st", "s3", model_info={"service_tiers": ["c"], "team_id": "TA"}), + ] + ) + + assert core.registry().group("st", field="service_tiers") == (["b"],) + assert core.registry().group("st", field="service_tiers", team_id="TA") == (["b"], ["c"]) + assert core.registry().group("st", field="service_tiers", team_id="TB") == (["b"],) + + +def test_max_input_tokens_are_each_routable_deployments_own_value_in_order(make_core: MakeCore) -> None: + registry = _configured_core(make_core).registry() + + assert registry.group("cfg", field="max_input_tokens") == (1000, 5000) + assert registry.group("cfg-alias", field="max_input_tokens") == (1000, 5000) + assert registry.group("bad", field="max_input_tokens") == (None,) + assert registry.group("blank", field="max_input_tokens") == (None,) + assert registry.group("nope", field="max_input_tokens") == () + + +def test_max_input_tokens_leave_out_blocked_deployments_and_other_teams(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment("mt", "m1", model_info={"max_input_tokens": 1, "blocked": True}), + deployment("mt", "m2", model_info={"max_input_tokens": 2}), + deployment("mt", "m3", model_info={"max_input_tokens": 3, "team_id": "TA"}), + ] + ) + + assert core.registry().group("mt", field="max_input_tokens") == (2,) + assert core.registry().group("mt", field="max_input_tokens", team_id="TA") == (2, 3) + + +def test_max_input_tokens_skip_a_value_a_saved_model_info_copied_from_the_cost_map(make_core: MakeCore) -> None: + catalog_limit = GetModelCostMap.loaded_model_cost_map()["gpt-4o"]["max_input_tokens"] + core = make_core( + model_list=[ + deployment( + "echo", "e1", model="openai/gpt-4o", model_info={"key": "gpt-4o", "max_input_tokens": catalog_limit} + ), + deployment( + "echo", "e2", model="openai/gpt-4o", model_info={"key": "gpt-4o", "max_input_tokens": catalog_limit - 1} + ), + ] + ) + + assert core.registry().group("echo", field="max_input_tokens") == (None, catalog_limit - 1) + + +# -- deployment model info -------------------------------------------------------------------------- + + +def test_deployment_model_info_merges_custom_pricing_over_the_builtin_entry(make_core: MakeCore) -> None: + model_id = unique_id() + core = make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3)]) + + info = core.registry().deployment_model_info(model_id, "openai/gpt-4o") + + assert info is not None + builtin = litellm.get_model_info("openai/gpt-4o") + assert info["input_cost_per_token"] == 1e-3 + assert info["max_input_tokens"] == builtin["max_input_tokens"] + + +def test_deployment_model_info_without_custom_pricing_is_the_builtin_entry(make_core: MakeCore) -> None: + model_id = unique_id() + core = make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o")]) + + info = core.registry().deployment_model_info(model_id, "openai/gpt-4o") + + assert info is not None + builtin = litellm.get_model_info("openai/gpt-4o") + assert (info["input_cost_per_token"], info["output_cost_per_token"]) == ( + builtin["input_cost_per_token"], + builtin["output_cost_per_token"], + ) + + +def test_deployment_model_info_of_an_unmapped_model_is_its_custom_entry(make_core: MakeCore) -> None: + model_id = unique_id() + core = make_core( + model_list=[ + deployment( + "unm", + model_id, + model="openai/unmapped-registry-thing", + input_cost_per_token=5e-6, + model_info={"max_input_tokens": 1000}, + ) + ] + ) + + info = core.registry().deployment_model_info(model_id, "openai/unmapped-registry-thing") + + assert info is not None + assert (info["input_cost_per_token"], info["max_input_tokens"]) == (5e-6, 1000) + + +def test_deployment_model_info_of_an_unknown_id_and_model_is_none(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + assert core.registry().deployment_model_info("missing-id", "nope/nothing-at-all") is None + + +def test_cached_deployment_model_info_equals_the_uncached_one(make_core: MakeCore) -> None: + model_id = unique_id() + core = make_core(model_list=[deployment("gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3)]) + + assert core.registry().deployment_model_info(model_id, "openai/gpt-4o", cached=True) == ( + core.registry().deployment_model_info(model_id, "openai/gpt-4o") + ) + + +def test_discovered_model_info_is_empty_without_a_refresh(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + assert dict(core.registry().discovered_model_info("a")) == {} + assert dict(core.registry().discovered_model_info(None)) == {} + + +# -- router model info ------------------------------------------------------------------------------ + + +def test_router_model_info_by_id_is_the_builtin_entry_with_the_deployments_model_info_on_top( + make_core: MakeCore, +) -> None: + model_id = unique_id() + core = make_core( + model_list=[ + deployment( + "gpt", model_id, model="openai/gpt-4o", input_cost_per_token=1e-3, model_info={"max_input_tokens": 99} + ) + ] + ) + + info = core.registry().router_model_info(None, "gpt", id=model_id) + + builtin = litellm.get_model_info("openai/gpt-4o") + assert info["key"] == builtin["key"] + assert info["max_input_tokens"] == 99 + assert info["max_output_tokens"] == builtin["max_output_tokens"] + + +def test_router_model_info_of_a_passed_deployment_dict(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + info = core.registry().router_model_info(deployment("x", "x", model="openai/gpt-4o-mini"), "x") + + assert info["key"] == litellm.get_model_info("openai/gpt-4o-mini")["key"] + + +def test_router_model_info_reads_the_base_model(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment( + "az", "z", model="azure/my-deployment", api_base="https://x", model_info={"base_model": "azure/gpt-4o"} + ) + ] + ) + + info = core.registry().router_model_info(None, "az", id="z") + + assert info["max_input_tokens"] == litellm.get_model_info("azure/gpt-4o")["max_input_tokens"] + + +def test_router_model_info_of_a_wildcard_deployment_reads_the_requested_model(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("openai/*", "w", model="openai/*")]) + + info = core.registry().router_model_info(None, "openai/gpt-4o-mini", id="w") + + assert info["key"] == litellm.get_model_info("openai/gpt-4o-mini")["key"] + + +def test_router_model_info_for_an_unknown_id_and_no_deployment_raises(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + with pytest.raises(ValueError, match="Deployment not found"): + core.registry().router_model_info(None, "gpt", id="missing") + + +def test_router_model_info_of_a_deployment_without_litellm_params_raises(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + with pytest.raises(ValueError, match="Deployment missing valid litellm_params"): + core.registry().router_model_info({"model_name": "x", "litellm_params": {}, "model_info": {"id": "q"}}, "x") + + +def test_router_model_info_of_an_unmapped_model_is_the_zeroed_stub_construction_registered(make_core: MakeCore) -> None: + """Construction registers every backend model in the cost map, so an unmapped one reads as free, not missing.""" + core = make_core(model_list=[deployment("gpt", "a", model="openai/not-a-known-registry-model")]) + + info = core.registry().router_model_info(None, "gpt", id="a") + + assert info["key"] == "openai/not-a-known-registry-model" + assert (info["input_cost_per_token"], info["output_cost_per_token"], info["max_input_tokens"]) == (0, 0, None) + assert info["litellm_provider"] == "openai" + + +# -- usage this minute ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_group_usage_is_none_until_something_is_counted(make_core: MakeCore) -> None: + core = _usage_core(make_core) + + assert await core.health().usage("gpt") == (None, None) + assert await core.health().usage("nope") == (None, None) + + +@pytest.mark.asyncio +async def test_group_usage_sums_each_deployments_counters_for_this_minute(make_core: MakeCore) -> None: + core = _usage_core(make_core) + await _count(core, RouterCacheEnum.TPM, "a", "openai/gpt-4o", 30) + await _count(core, RouterCacheEnum.TPM, "b", "anthropic/claude-3-haiku-20240307", 4) + await _count(core, RouterCacheEnum.RPM, "b", "anthropic/claude-3-haiku-20240307", 2) + + assert await core.health().usage("gpt") == (34, 2) + assert await core.health().usage("al") == (34, 2) + + +@pytest.mark.asyncio +async def test_io_token_usage_sums_input_and_output_counters(make_core: MakeCore) -> None: + core = _usage_core(make_core) + assert await core.get_model_group_io_token_usage("io") == (None, None) # pyright: ignore[reportAttributeAccessIssue] # public on the core + await _count(core, RouterCacheEnum.ITPM, "i", "openai/gpt-4o", 7) + + assert await core.get_model_group_io_token_usage("io") == (7, None) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + +@pytest.mark.asyncio +async def test_remaining_usage_is_each_limit_minus_this_minutes_usage(make_core: MakeCore) -> None: + core = _usage_core(make_core) + await _count(core, RouterCacheEnum.TPM, "a", "openai/gpt-4o", 30) + await _count(core, RouterCacheEnum.RPM, "b", "anthropic/claude-3-haiku-20240307", 2) + + assert await core.health().usage("gpt", kind="remaining") == { + "x-ratelimit-remaining-tokens": 1070, + "x-ratelimit-limit-tokens": 1100, + "x-ratelimit-remaining-requests": 13, + "x-ratelimit-limit-requests": 15, + } + + +@pytest.mark.asyncio +async def test_remaining_usage_of_an_io_limited_group_reports_input_and_output_token_headers( + make_core: MakeCore, +) -> None: + core = _usage_core(make_core) + await _count(core, RouterCacheEnum.ITPM, "i", "openai/gpt-4o", 7) + + remaining = await core.health().usage("io", kind="remaining") + + assert remaining["x-ratelimit-limit-input-tokens"] == 300 + assert remaining["x-ratelimit-remaining-input-tokens"] == 293 + assert remaining["x-ratelimit-limit-output-tokens"] == 40 + assert remaining["x-ratelimit-remaining-output-tokens"] == 40 + assert "x-ratelimit-limit-tokens" not in remaining + + +@pytest.mark.asyncio +@pytest.mark.parametrize("group", ["plain", "nope"]) +async def test_remaining_usage_of_a_group_without_limits_is_empty(make_core: MakeCore, group: str) -> None: + assert await _usage_core(make_core).health().usage(group, kind="remaining") == {} + + +@pytest.mark.asyncio +async def test_an_unknown_usage_kind_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown usage kind 'bogus'"): + await _usage_core(make_core).health().usage("gpt", kind="bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid kind on purpose + + +# -- set_response_headers --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_response_headers_name_the_group_and_its_remaining_limits(make_core: MakeCore) -> None: + core = _usage_core(make_core) + await _count(core, RouterCacheEnum.TPM, "a", "openai/gpt-4o", 30) + response = completion_response() + + returned = await core.set_response_headers(response, model_group="gpt") # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert returned is response + assert response._hidden_params["additional_headers"] == { + "x-litellm-model-group": "gpt", + "x-ratelimit-remaining-tokens": 1070, + "x-ratelimit-limit-tokens": 1100, + "x-ratelimit-remaining-requests": 15, + "x-ratelimit-limit-requests": 15, + } + + +@pytest.mark.asyncio +async def test_response_headers_without_a_group_carry_only_a_none_group(make_core: MakeCore) -> None: + returned = await _usage_core(make_core).set_response_headers(completion_response(), model_group=None) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert returned._hidden_params["additional_headers"] == {"x-litellm-model-group": None} + + +@pytest.mark.asyncio +async def test_response_headers_of_a_group_without_limits_carry_only_the_group(make_core: MakeCore) -> None: + returned = await _usage_core(make_core).set_response_headers(completion_response(), model_group="plain") # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert returned._hidden_params["additional_headers"] == {"x-litellm-model-group": "plain"} + + +@pytest.mark.asyncio +async def test_response_headers_of_no_response_are_none(make_core: MakeCore) -> None: + assert await _usage_core(make_core).set_response_headers(None, model_group="gpt") is None # pyright: ignore[reportAttributeAccessIssue] # public on the core + + +@pytest.mark.asyncio +async def test_response_headers_stamp_the_model_id_of_the_deployment_in_the_request(make_core: MakeCore) -> None: + returned = await _usage_core(make_core).set_response_headers( # pyright: ignore[reportAttributeAccessIssue] # public on the core + completion_response(), model_group="plain", request_kwargs={"metadata": {"model_info": {"id": "p"}}} + ) + + assert returned._hidden_params["model_id"] == "p" + + +# -- budgets per deployment ------------------------------------------------------------------------- + + +def test_no_deployment_budget_means_no_budget_limiter(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + assert core.registry().plugins("budget_limiter") is None + + +@pytest.mark.asyncio +async def test_adding_a_deployment_with_a_budget_installs_the_budget_limiter(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + core.add_deployment( # pyright: ignore[reportAttributeAccessIssue] # public on the core + Deployment(**deployment("gpt", "budgeted", max_budget=1.0, budget_duration="1d")) # pyright: ignore[reportArgumentType] # a model_list entry + ) + + limiter = core.registry().plugins("budget_limiter") + assert limiter is not None + assert type(limiter).__name__ == "RouterBudgetLimiting" + + +def test_a_team_without_ptu_shares_has_no_ptu_ceiling(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + assert core.health().team_ptu_ceiling("TA", "gpt") is None diff --git a/tests/unit/test_router/core/test_registry_characterization_writes.py b/tests/unit/test_router/core/test_registry_characterization_writes.py new file mode 100644 index 00000000000..0558a16fe9d --- /dev/null +++ b/tests/unit/test_router/core/test_registry_characterization_writes.py @@ -0,0 +1,516 @@ +""" +Registry and settings writes beyond the version bookkeeping in ``test_registry_changesets.py``: +what ``add_deployment``, ``set_model_list``, upserts and deletes leave behind in every index, how +``update_settings`` / ``ChangeSet(settings=...)`` coerce and filter values, and the ``settings()`` +and ``registry()`` views. +""" + +import warnings + +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import Deployment, RetryPolicy +from tests.unit.test_router.core.registry_helpers import ( + MESSAGES, + MakeCore, + ids, + names_and_ids, + team_deployment, + typed, + unique_id, +) +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment + + +def _core(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + """gpt (a, b), emb (e), the openai/* wildcard (w), team TA's ``pub`` (t1, internal name ``int``).""" + return make_core( + model_list=[ + deployment("gpt", "a"), + deployment("gpt", "b"), + deployment("emb", "e"), + deployment("openai/*", "w", model="openai/*"), + team_deployment("int", "t1", "TA", "pub"), + ], + **arguments, + ) + + +# -- add_deployment --------------------------------------------------------------------------------- + + +def test_add_deployment_returns_the_deployment_it_serves(make_core: MakeCore) -> None: + core = _core(make_core) + + added = core.add_deployment(typed("new", "n")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert isinstance(added, Deployment) + assert (added.model_name, added.model_info.id, added.litellm_params.model) == ("new", "n", "openai/n") + + +def test_an_added_deployment_is_appended_and_names_a_new_group(make_core: MakeCore) -> None: + core = _core(make_core) + + core.add_deployment(typed("new", "n")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert core.registry().names(kind="ids") == ["a", "b", "e", "w", "t1", "n"] + assert "new" in core.registry().names(kind="group_set") + assert core.registry().index("model_name")["new"] == [5] + assert core.registry().names(kind="deployment_names")[-1] == "openai/n" + + +def test_an_added_deployment_without_an_id_gets_the_id_its_model_info_generated(make_core: MakeCore) -> None: + core = _core(make_core) + candidate = Deployment(model_name="new", litellm_params={"model": "openai/q", "api_key": "k"}) # pyright: ignore[reportArgumentType] # litellm_params as a dict + + added = core.add_deployment(candidate) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert added is not None + assert added.model_info.id == candidate.model_info.id + assert core.registry().has_deployment(str(candidate.model_info.id)) + + +@pytest.mark.asyncio +async def test_an_added_deployment_serves_requests_for_its_group(make_core: MakeCore, host: ScriptedHost) -> None: + core = _core(make_core) + + core.add_deployment(typed("new", "n")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + await core.acompletion(model="new", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["n"] + + +def test_an_added_wildcard_deployment_serves_matching_names(make_core: MakeCore) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + + core.add_deployment(typed("anthropic/*", "x", model="anthropic/*")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert names_and_ids(core.registry().deployments("anthropic/claude-x")) == [("anthropic/claude-x", "x")] + assert core.settings().provider_default_deployment_ids == ["x"] + + +def test_an_added_team_deployment_is_team_indexed(make_core: MakeCore) -> None: + core = _core(make_core) + + core.add_deployment(Deployment(**team_deployment("int-2", "t2", "TB", "pub-b"))) # pyright: ignore[reportAttributeAccessIssue, reportArgumentType] # public on the core + + assert core.registry().index("team_model")[("TB", "pub-b")] == [5] + assert core.registry().names(kind="team_public") == frozenset({"pub", "pub-b"}) + assert ids(core.registry().deployments("pub-b", team_id="TB")) == ["t2"] + + +def test_adding_a_deployment_with_an_unknown_provider_raises_and_adds_nothing(make_core: MakeCore) -> None: + core = _core(make_core) + version = core.registry().version + + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): + core.add_deployment(typed("bad", "bad", model="notaprovider/x")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert core.registry().names(kind="ids") == ["a", "b", "e", "w", "t1"] + assert "bad" not in core.registry().names(kind="group_set") + assert core.registry().version == version + + +def test_an_added_deployment_registers_its_custom_pricing(make_core: MakeCore) -> None: + core = _core(make_core) + model_id = unique_id() + + core.add_deployment(typed("priced", model_id, model="openai/gpt-4o", input_cost_per_token=3e-3)) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert litellm.model_cost[model_id]["input_cost_per_token"] == 3e-3 + + +# -- upsert ----------------------------------------------------------------------------------------- + + +def test_a_changed_upsert_moves_the_deployment_to_the_end_of_the_list(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(upsert=[typed("gpt", "a", api_base="https://changed.example")])) + + assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"] + assert core.registry().index("model_name") == {"gpt": [0, 4], "emb": [1], "openai/*": [2], "int": [3]} + assert ids(core.registry().deployments("gpt")) == ["b", "a"] + + +def test_an_upsert_that_renames_the_group_moves_the_deployment_between_groups(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(upsert=[typed("emb", "a")])) + + assert ids(core.registry().deployments("gpt")) == ["b"] + assert ids(core.registry().deployments("emb")) == ["e", "a"] + + +def test_an_upsert_that_empties_a_group_drops_the_group_name(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(upsert=[typed("emb2", "e")])) + + assert core.registry().names(kind="group_set") == {"gpt", "openai/*", "int", "emb2"} + assert core.registry().deployments("emb") == [] + + +def test_a_failing_upsert_of_an_existing_deployment_raises_and_leaves_it_removed(make_core: MakeCore) -> None: + """Odd but current: without ignore_invalid_deployments the old deployment is popped and not restored.""" + core = _core(make_core) + + with pytest.raises(litellm.BadRequestError): + core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")])) + + assert core.registry().names(kind="ids") == ["b", "e", "w", "t1"] + + +def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous_deployment( + make_core: MakeCore, +) -> None: + core = _core(make_core, ignore_invalid_deployments=True) + + result = core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")])) + + assert list(result.upserted) == [None] + restored = core.registry().deployment("a") + assert restored is not None + assert restored.litellm_params.model == "openai/a" + assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"] + + +def test_the_deprecated_upsert_deployment_warns_and_returns_the_deployment(make_core: MakeCore) -> None: + core = _core(make_core) + + with pytest.warns(DeprecationWarning, match=r"upsert_deployment is deprecated; use .*apply\(ChangeSet\(upsert="): + upserted = core.upsert_deployment(typed("gpt", "c")) # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + assert upserted is not None + assert upserted.model_info.id == "c" + + +# -- delete ----------------------------------------------------------------------------------------- + + +def test_delete_returns_the_removed_model_list_entry(make_core: MakeCore) -> None: + core = _core(make_core) + + result = core.apply(ChangeSet(delete=["e"])) + + (removed,) = result.deleted + assert isinstance(removed, dict) + assert (removed["model_name"], removed["model_info"]["id"]) == ("emb", "e") + + +def test_deleting_the_last_deployment_of_a_group_drops_the_group(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["e"])) + + assert "emb" not in core.registry().names(kind="group_set") + assert core.registry().deployments("emb") == [] + assert core.registry().index("model_name") == {"gpt": [0, 1], "openai/*": [2], "int": [3]} + + +def test_deleting_a_deployment_reindexes_the_ones_after_it(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["a"])) + + for deployment_id in ("b", "e", "w", "t1"): + found = core.registry().deployment(deployment_id) + assert found is not None + assert found.model_info.id == deployment_id + assert core.registry().index("team_model") == {("TA", "pub"): [3]} + + +def test_deleting_a_team_deployment_drops_its_team_index_and_public_name(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["t1"])) + + assert core.registry().index("team_model") == {} + assert core.registry().names(kind="team_public") == frozenset() + assert core.registry().deployments("pub", team_id="TA") == [] + + +def test_deleting_a_wildcard_deployment_stops_it_serving_matching_names(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["w"])) + + assert core.registry().deployments("openai/gpt-x") == [] + + +def test_deleting_a_wildcard_deployment_keeps_it_among_provider_default_deployment_ids(make_core: MakeCore) -> None: + """Odd but current: the provider default id list is append-only.""" + core = _core(make_core) + + core.apply(ChangeSet(delete=["w"])) + + assert core.settings().provider_default_deployment_ids == ["w"] + + +@pytest.mark.asyncio +async def test_a_deleted_deployment_is_no_longer_called(make_core: MakeCore, host: ScriptedHost) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["a"])) + for _ in range(3): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["b", "b", "b"] + + +def test_the_deprecated_delete_deployment_warns_and_returns_none_for_an_unknown_id(make_core: MakeCore) -> None: + core = _core(make_core) + + with pytest.warns(DeprecationWarning, match=r"delete_deployment is deprecated; use .*apply\(ChangeSet\(delete="): + assert core.delete_deployment("missing") is None # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + +# -- set_model_list --------------------------------------------------------------------------------- + + +def test_set_model_list_replaces_every_deployment_and_group(make_core: MakeCore) -> None: + core = _core(make_core) + + core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert core.registry().names(kind="ids") == ["n"] + assert core.registry().names(kind="group_set") == {"new"} + assert core.registry().index("model_name") == {"new": [0]} + assert core.registry().index("team_model") == {} + assert core.registry().names(kind="team_public") == frozenset() + + +def test_set_model_list_keeps_the_aliases(make_core: MakeCore) -> None: + core = _core(make_core, model_group_alias={"fast": "new"}) + + core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert ids(core.registry().deployments("fast")) == ["n"] + + +def test_set_model_list_leaves_the_given_list_unchanged(make_core: MakeCore) -> None: + core = _core(make_core) + given = [{"model_name": "new", "litellm_params": {"model": "openai/n", "api_key": "k"}}] + + core.set_model_list(given) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert given == [{"model_name": "new", "litellm_params": {"model": "openai/n", "api_key": "k"}}] + + +def test_set_model_list_keeps_the_old_wildcard_routes(make_core: MakeCore) -> None: + """Odd but current: the wildcard pattern router is not reset, so a replaced wildcard still matches.""" + core = _core(make_core) + + core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert names_and_ids(core.registry().deployments("openai/gpt-x")) == [("openai/gpt-x", "w")] + assert core.settings().provider_default_deployment_ids == ["w"] + + +def test_set_model_list_appends_to_the_deployment_names(make_core: MakeCore) -> None: + """Odd but current: deployment_names keep the replaced deployments' models.""" + core = make_core(model_list=[deployment("gpt", "a")]) + + core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert core.registry().names(kind="deployment_names") == ["openai/a", "openai/n"] + + +@pytest.mark.asyncio +async def test_after_set_model_list_requests_route_to_the_new_deployments( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = _core(make_core) + + core.set_model_list([deployment("gpt", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["n"] + + +# -- update_settings / ChangeSet(settings=...) ------------------------------------------------------ + + +@pytest.mark.parametrize( + ("name", "given", "stored"), + [ + ("timeout", "5", 5), + ("num_retries", "3", 3), + ("retry_after", 2.9, 2), + ("allowed_fails", "4", 4), + ("cooldown_time", 7.5, 7), + ], +) +def test_integer_settings_are_cast_to_int(make_core: MakeCore, name: str, given: object, stored: int) -> None: + core = _core(make_core) + + core.apply(ChangeSet(settings={name: given})) + + assert core.settings().as_dict()[name] == stored + + +def test_a_non_integer_value_for_an_integer_setting_raises(make_core: MakeCore) -> None: + core = _core(make_core) + + with pytest.raises(ValueError, match="invalid literal for int"): + core.apply(ChangeSet(settings={"timeout": "abc"})) + + +def test_a_setting_that_is_not_runtime_updatable_is_ignored(make_core: MakeCore) -> None: + core = _core(make_core) + before = dict(core.settings().as_dict()) + + core.apply(ChangeSet(settings={"bogus": 1, "model_list": []})) + + assert dict(core.settings().as_dict()) == before + assert core.registry().names(kind="ids") == ["a", "b", "e", "w", "t1"] + + +def test_a_retry_policy_dict_is_stored_as_a_retry_policy(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(settings={"retry_policy": {"RateLimitErrorRetries": 2}})) + + stored = core.settings().as_dict()["retry_policy"] + assert isinstance(stored, RetryPolicy) + assert stored.RateLimitErrorRetries == 2 + + +@pytest.mark.parametrize(("given", "stored"), [("true", True), (True, True), ("false", False), ("yes-ish", False)]) +def test_mid_stream_fallback_continuation_reads_flag_strings(make_core: MakeCore, given: object, stored: bool) -> None: + core = _core(make_core) + + core.apply(ChangeSet(settings={"enable_mid_stream_fallback_continuation": given})) + + assert core.settings().as_dict()["enable_mid_stream_fallback_continuation"] is stored + + +def test_model_group_alias_from_settings_is_visible_to_group_info(make_core: MakeCore) -> None: + core = _core(make_core) + assert core.registry().group("quick") is None + + core.apply(ChangeSet(settings={"model_group_alias": {"quick": "emb"}})) + + info = core.registry().group("quick") + assert info is not None + assert info.model_group == "quick" + + +def test_the_deprecated_update_settings_warns_and_applies(make_core: MakeCore) -> None: + core = _core(make_core) + + with pytest.warns(DeprecationWarning, match=r"update_settings is deprecated"): + core.update_settings(cooldown_time=13) # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + assert core.settings().cooldown_time == 13 + + +# -- the settings() and registry() views ------------------------------------------------------------ + + +def test_settings_as_dict_lists_the_runtime_updatable_settings(make_core: MakeCore) -> None: + core = _core(make_core, model_group_alias={"fast": "gpt"}, num_retries=2, cooldown_time=5) + + settings = core.settings().as_dict() + + assert set(settings) == { + "routing_strategy_args", + "routing_strategy", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "model_group_retry_policy", + "retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", + "enable_tag_filtering", + "tag_routing_prefix", + "fallback_on_unsupported_params", + "routing_groups", + } + assert (settings["routing_strategy"], settings["num_retries"], settings["cooldown_time"]) == ( + "simple-shuffle", + 2, + 5, + ) + assert settings["model_group_alias"] == {"fast": "gpt"} + assert settings["routing_groups"] == [] + + +def test_settings_defaults_on_a_fresh_core(make_core: MakeCore) -> None: + settings = _core(make_core).settings() + + assert settings.routing_strategy == "simple-shuffle" + assert settings.allowed_fails == 3 + assert settings.fallbacks is None + assert settings.enable_tag_filtering is False + assert settings.provider_default_deployment_ids == ["w"] + + +@pytest.mark.parametrize(("name", "expected"), [("cooldown_time", True), ("bogus", False), ("num_retries", False)]) +def test_settings_is_set_names_only_settings_fields(make_core: MakeCore, name: str, expected: bool) -> None: + assert _core(make_core).settings().is_set(name) is expected + + +def test_registry_view_is_reused(make_core: MakeCore) -> None: + core = _core(make_core) + + assert core.registry() is core.registry() + + +def test_registry_all_is_the_live_model_list(make_core: MakeCore) -> None: + core = _core(make_core) + listed = core.registry().all() + + core.add_deployment(typed("new", "n")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert ids(listed) == ["a", "b", "e", "w", "t1", "n"] + + +def test_an_unknown_index_name_raises(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="unknown index 'bogus'"): + _core(make_core).registry().index("bogus") # pyright: ignore[reportCallIssue, reportArgumentType] # an invalid name on purpose + + +# -- aapply(refresh_model_info) --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_refreshing_model_info_with_nothing_to_discover_changes_nothing(make_core: MakeCore) -> None: + core = _core(make_core) + version = core.registry().version + + result = await core.aapply(ChangeSet(refresh_model_info=True)) # pyright: ignore[reportAttributeAccessIssue] # aapply is on every core + + assert (result.version, list(result.upserted), list(result.deleted)) == (version, [], []) + assert core.registry().version == version + assert dict(core.registry().discovered_model_info("a")) == {} + + +@pytest.mark.asyncio +async def test_aapply_applies_upserts_and_deletes_like_apply(make_core: MakeCore) -> None: + core = _core(make_core) + + result = await core.aapply(ChangeSet(upsert=[typed("gpt", "c")], delete=["e"])) # pyright: ignore[reportAttributeAccessIssue] # aapply is on every core + + assert [None if d is None else d.model_info.id for d in result.upserted] == ["c"] + assert core.registry().names(kind="ids") == ["a", "b", "w", "t1", "c"] + + +@pytest.mark.asyncio +async def test_the_deprecated_arefresh_model_info_warns(make_core: MakeCore) -> None: + core = _core(make_core) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await core.arefresh_model_info() # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + assert any("arefresh_model_info is deprecated" in str(w.message) for w in caught) diff --git a/tests/unit/test_router/core/test_request_scoped_outcomes.py b/tests/unit/test_router/core/test_request_scoped_outcomes.py new file mode 100644 index 00000000000..ebe162dc090 --- /dev/null +++ b/tests/unit/test_router/core/test_request_scoped_outcomes.py @@ -0,0 +1,466 @@ +""" +What a router records once its outcome callbacks left the process lists, on both cores: +``observe(request_scoped=True)``, the old ``serve_through_request_logging()`` and ``retire()`` take +them off, and ``observe(logging_obj)`` puts them back on one request's logging object. What the +Python router writes outside those callbacks goes on: ``make_call``'s usage, a refused pre-call +check's cooldown and a failed fallback hop's cooldown. +""" + +import asyncio +from collections.abc import Callable, Iterable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router import Router +from litellm.router_core import RouterCore +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.retries_health_helpers import ( + MESSAGES, + MakeCore, + attempt, + attempt_sync, + cooled_down, + fresh, +) +from tests.unit.test_router.core.scripted_host import Call, deployment, rate_limit, server_error +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts, request_logging + +USAGE: Final = {"prompt_tokens": 4, "completion_tokens": 8, "total_tokens": 12} + + +class RequestLoggingHost(EndpointHost): + """``EndpointHost`` that logs on a request's logging object as litellm does: its first failure + alone runs the failure callbacks, and what ``_core_bind_to_request_logging`` put on the object + runs beside the process lists. A stream logs a failure through its own copy of the object, so + only an attempt's failure marks it.""" + + __slots__ = () + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLogging): + return await super()._answer_async(op, args, kwargs) + details: Final = logging_obj.model_call_details + details["current_attempt_api_call_start_time"] = object() + try: + return await super()._answer_async(op, args, kwargs) + except Exception: + if details.get("has_logged_async_failure") is not True: + details["has_logged_async_failure"] = True + details["async_failure_logged_attempt"] = details.get("current_attempt_api_call_start_time") + raise + + async def log_async(self, call: Call, error: BaseException | None, response: object) -> None: + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, LiteLLMLogging): + await super().log_async(call, error, response) + return + if error is not None and logging_obj.model_call_details.get("has_logged_async_failure") is True: + return + await super().log_async(call, error, response) + logged: Final = self._logging_kwargs(call, error, response) + if error is not None: + for callback in _functions(logging_obj.dynamic_failure_callbacks): + _ = callback(logged, None, 0.0, 0.0) + for callback in _functions(logging_obj.dynamic_async_failure_callbacks): + await callback(logged, None, 0.0, 0.0) # pyright: ignore[reportGeneralTypeIssues] # an async callback + return + for callback in _functions(logging_obj.dynamic_async_success_callbacks): + await callback(logged, response, 0.0, 0.0) # pyright: ignore[reportGeneralTypeIssues] # an async callback + + def log_sync(self, call: Call, error: BaseException | None, response: object) -> None: + super().log_sync(call, error, response) + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if error is None or not isinstance(logging_obj, LiteLLMLogging): + return + logged: Final = self._logging_kwargs(call, error, response) + for callback in _functions(logging_obj.dynamic_failure_callbacks): + _ = callback(logged, None, 0.0, 0.0) + + +def _functions(callbacks: Iterable[object] | None) -> tuple[Callable[..., object], ...]: + """The function callbacks of a logging object's list; a ``CustomLogger`` logs on its own hooks.""" + return tuple( + callback for callback in callbacks or () if callable(callback) and not isinstance(callback, CustomLogger) + ) + + +@pytest.fixture +def host() -> RequestLoggingHost: + return RequestLoggingHost() + + +def observed(core: RouterCore) -> None: + _ = core.observe(request_scoped=True) # pyright: ignore[reportCallIssue] # every core takes it + + +def through_the_old_member(core: RouterCore) -> None: + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # as Router.from_config attaches it + with pytest.warns(DeprecationWarning, match=r"Router\.serve_through_request_logging is deprecated"): + router.serve_through_request_logging() + + +def retired(core: RouterCore) -> None: + core.retire() + + +RELEASES: Final = pytest.mark.parametrize( + "release", [observed, through_the_old_member, retired], ids=["observe", "old-member", "retire"] +) + + +def weighted_pair(make_core: MakeCore, **limits: object) -> RouterCore: # kwargs-ok: both deployments' params + """``d1`` takes every pick while it is not cooling down, and one failure cools it.""" + return make_core( + model_list=[deployment("g", "d1", weight=1, **limits), deployment("g", "d2", weight=0, **limits)], + num_retries=0, + allowed_fails=0, + ) + + +def successes(core: RouterCore, deployment_id: str) -> object: + return core.health().store.get_cache(f"{deployment_id}:successes", local_only=True) + + +async def settle() -> None: + """Let the recording tasks a stream's end scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +# --- a router still on the process lists ------------------------------------------------------ + + +async def test_a_router_on_the_lists_cools_a_failing_deployment_and_moves_on( + make_core: MakeCore, host: RequestLoggingHost +) -> None: + core: Final = weighted_pair(make_core) + host.answer("openai/d1", fresh(server_error)) + + first: Final = await attempt(core, disable_fallbacks=True) + second: Final = await attempt(core, disable_fallbacks=True) + + assert isinstance(first, litellm.InternalServerError) + assert not isinstance(second, Exception) + assert host.deployments_called() == ["d1", "d2"] + assert list(cooled_down(core, ["d1", "d2"])) == ["d1"] + assert core.health().failures_this_minute("d1") == 1 + assert successes(core, "d2") == 1 + + +async def test_a_binding_changes_nothing_on_a_router_on_the_lists( + make_core: MakeCore, host: RequestLoggingHost +) -> None: + core: Final = weighted_pair(make_core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + _ = attempt_sync(core, litellm_logging_obj=bound) + + assert successes(core, "d1") == 1 + + +# --- released, with no binding ------------------------------------------------------------------ + + +@RELEASES +async def test_a_failing_deployment_never_cools_and_the_next_call_returns_to_it( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core) + host.answer("openai/d1", fresh(server_error)) + release(core) + + first: Final = await attempt(core, disable_fallbacks=True) + second: Final = await attempt(core, disable_fallbacks=True) + + assert isinstance(first, litellm.InternalServerError) + assert isinstance(second, litellm.InternalServerError) + assert host.deployments_called() == ["d1", "d1"] + assert cooled_down(core, ["d1", "d2"]) == {} + assert core.health().failures_this_minute("d1") == 0 + assert await core.health().usage("g") == (None, None) + + +@RELEASES +async def test_a_retry_goes_back_to_the_failing_deployment( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", weight=1), deployment("g", "d2", weight=0)], num_retries=1 + ) + host.answer("openai/d1", fresh(rate_limit)) + release(core) + + answered: Final = await attempt(core) + + assert isinstance(answered, litellm.RateLimitError) + assert host.deployments_called() == ["d1", "d1"] + assert cooled_down(core, ["d1", "d2"]) == {} + + +@RELEASES +def test_a_sync_failure_records_nothing( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core) + host.answer("openai/d1", fresh(server_error)) + release(core) + + answered: Final = attempt_sync(core) + + assert isinstance(answered, litellm.InternalServerError) + assert cooled_down(core, ["d1", "d2"]) == {} + assert core.health().failures_this_minute("d1") == 0 + + +@RELEASES +async def test_a_success_counts_its_usage_and_no_success( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core, rpm=100) + release(core) + + answered: Final = await attempt(core) + + assert not isinstance(answered, Exception) + assert successes(core, "d1") is None + assert await core.health().usage("g") == (10, 1) + + +@RELEASES +async def test_a_sync_success_counts_its_usage_and_no_success( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core, rpm=100) + release(core) + + answered: Final = attempt_sync(core) + + assert not isinstance(answered, Exception) + assert successes(core, "d1") is None + assert await core.health().usage("g") == (10, 1) + + +async def test_an_endpoint_outside_make_call_counts_no_usage(make_core: MakeCore, host: RequestLoggingHost) -> None: + """``amoderation`` makes one attempt, outside ``make_call``: only the success callback counts it.""" + core: Final = make_core(model_list=[deployment("g", "d1", rpm=100)], num_retries=0) + observed(core) + + _ = await core.amoderation(model="g", input="x") # pyright: ignore[reportAttributeAccessIssue] # every core serves it + + assert await core.health().usage("g") == (None, None) + + +@RELEASES +async def test_a_stream_counts_its_request_when_it_opens_and_nothing_when_it_ends( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + host.answer("openai/d1", chat_stream("hello", usage=USAGE, logged=True)) + core: Final = weighted_pair(make_core, rpm=100) + release(core) + + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + _ = await collect_texts(stream) + await settle() + + assert successes(core, "d1") is None + assert await core.health().usage("g") == (0, 1) + + +@RELEASES +async def test_a_stream_failing_after_content_records_nothing( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + host.answer("openai/d1", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True)) + core: Final = weighted_pair(make_core) + release(core) + + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + with pytest.raises(litellm.RateLimitError, match="mid stream"): + _ = await collect_texts(stream) + await settle() + + assert cooled_down(core, ["d1", "d2"]) == {} + assert core.health().failures_this_minute("d1") == 0 + + +# --- what goes on outside the callbacks ------------------------------------------------------------ + + +class RefusesEveryDeployment(CustomLogger): + async def async_pre_call_check(self, deployment: object, parent_otel_span: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # any deployment + raise rate_limit() + + +@RELEASES +async def test_a_refused_pre_call_check_still_cools_the_deployment( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core) + release(core) + host.callback_list("callbacks").append(RefusesEveryDeployment()) + + _ = await attempt(core, disable_fallbacks=True) + + assert list(cooled_down(core, ["d1", "d2"])) == ["d1"] + assert core.health().failures_this_minute("d1") == 0 + + +async def test_a_failed_fallback_hop_on_a_shared_logging_object_is_still_counted( + make_core: MakeCore, host: RequestLoggingHost +) -> None: + """The first failure logs through no callback, yet blocks the object for the hops, and a blocked + hop records through ``_trigger_cooldown_for_failed_deployment``.""" + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b"), deployment("c", "c")], + fallbacks=[{"a": ["b"]}, {"b": ["c"]}], + num_retries=0, + ) + observed(core) + host.answer("*", fresh(server_error)) + + _ = await attempt(core, "a", litellm_logging_obj=request_logging()) + + assert core.health().failures_this_minute("a") == 0 + assert core.health().failures_this_minute("b") == 1 + assert core.health().failures_this_minute("c") == 1 + + +async def test_failed_fallback_hops_with_no_logging_object_are_not_counted( + make_core: MakeCore, host: RequestLoggingHost +) -> None: + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b")], fallbacks=[{"a": ["b"]}], num_retries=0 + ) + observed(core) + host.answer("*", fresh(server_error)) + + _ = await attempt(core, "a") + + assert core.health().failures_this_minute("a") == 0 + assert core.health().failures_this_minute("b") == 0 + + +# --- a binding, for its request alone ----------------------------------------------------------- + + +@pytest.mark.parametrize("release", [observed, retired], ids=["observe", "retire"]) +async def test_a_binding_records_a_failure_for_that_request_only( + make_core: MakeCore, host: RequestLoggingHost, release: Callable[[RouterCore], None] +) -> None: + core: Final = weighted_pair(make_core) + host.answer("openai/d1", fresh(server_error)) + release(core) + + _ = await attempt(core, disable_fallbacks=True, litellm_logging_obj=request_logging()) + assert cooled_down(core, ["d1", "d2"]) == {} + assert core.health().failures_this_minute("d1") == 0 + + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + _ = await attempt(core, disable_fallbacks=True, litellm_logging_obj=bound) + + assert list(cooled_down(core, ["d1", "d2"])) == ["d1"] + assert core.health().failures_this_minute("d1") == 1 + assert await core.health().usage("g") == (None, 1) + + +async def test_a_binding_records_an_async_success(make_core: MakeCore, host: RequestLoggingHost) -> None: + core: Final = weighted_pair(make_core) + observed(core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + _ = await attempt(core, litellm_logging_obj=bound) + + assert successes(core, "d1") == 1 + + +async def test_a_binding_leaves_a_sync_success_uncounted(make_core: MakeCore, host: RequestLoggingHost) -> None: + """``_core_bind_to_request_logging`` does not bind ``sync_deployment_callback_on_success``.""" + core: Final = weighted_pair(make_core, rpm=100) + observed(core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + _ = attempt_sync(core, litellm_logging_obj=bound) + + assert successes(core, "d1") is None + assert await core.health().usage("g") == (10, 1) + + +async def test_a_binding_records_a_sync_failure_with_no_request(make_core: MakeCore, host: RequestLoggingHost) -> None: + core: Final = weighted_pair(make_core) + host.answer("openai/d1", fresh(server_error)) + observed(core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + _ = attempt_sync(core, litellm_logging_obj=bound) + + assert list(cooled_down(core, ["d1", "d2"])) == ["d1"] + assert core.health().failures_this_minute("d1") == 1 + assert await core.health().usage("g") == (None, None) + + +async def test_a_binding_records_a_stream_that_ends(make_core: MakeCore, host: RequestLoggingHost) -> None: + host.answer("openai/d1", chat_stream("hello", usage=USAGE, logged=True)) + core: Final = weighted_pair(make_core, rpm=100) + observed(core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + stream: Final = await core.acompletion( # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + model="g", messages=MESSAGES, stream=True, litellm_logging_obj=bound + ) + _ = await collect_texts(stream) + await settle() + + assert successes(core, "d1") == 1 + assert await core.health().usage("g") == (12, 1) + + +async def test_a_binding_records_a_stream_failing_after_content(make_core: MakeCore, host: RequestLoggingHost) -> None: + host.answer("openai/d1", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True)) + core: Final = weighted_pair(make_core) + observed(core) + bound: Final = request_logging() + _ = core.observe(bound) # pyright: ignore[reportCallIssue] # every core takes it + + stream: Final = await core.acompletion( # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + model="g", messages=MESSAGES, stream=True, litellm_logging_obj=bound + ) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + _ = await collect_texts(stream) + await settle() + + assert list(cooled_down(core, ["d1", "d2"])) == ["d1"] + assert core.health().failures_this_minute("d1") == 1 + # No limit on the deployment, so the open counted nothing: the failure is the one request. + assert await core.health().usage("g") == (None, 1) + + +# --- retire ---------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("strategy", ["latency-based-routing", "least-busy", "usage-based-routing-v2"]) +async def test_a_retired_router_still_picks_with_its_strategy( + make_core: MakeCore, host: RequestLoggingHost, strategy: str +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", rpm=100), deployment("g", "d2", rpm=100)], + routing_strategy=strategy, + num_retries=0, + ) + core.retire() + + answered: Final = await attempt(core) + + assert not isinstance(answered, Exception) + assert len(host.deployments_called()) == 1 diff --git a/tests/unit/test_router/core/test_retries_health_characterization_cooldowns.py b/tests/unit/test_router/core/test_retries_health_characterization_cooldowns.py new file mode 100644 index 00000000000..4aeb11c58ec --- /dev/null +++ b/tests/unit/test_router/core/test_retries_health_characterization_cooldowns.py @@ -0,0 +1,845 @@ +""" +Characterization of deployment health in a router core: when a failure cools a deployment down (by status, +group size, allowed_fails and allowed_fails_policy at router and deployment level), what the failure and +success callbacks count, blocked deployments and health-check-driven filtering. +""" + +import time +from typing import Final + +import pytest + +import litellm +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS +from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure +from litellm.types.router import AllowedFailsPolicy, Deployment +from tests.unit.test_router.core.retries_health_helpers import ( + BackgroundSuccessLoggingHost, + ErrorBuilder, + MakeCore, + api_connection, + api_status, + attempt, + cooled_down, + fresh, + not_found, + permission_denied, + service_unavailable, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + bad_request, + deployment, + rate_limit, + server_error, + timeout, +) + + +def group( + *ids: str, **params: object +) -> list[dict[str, object]]: # kwargs-ok: litellm_params # mutable-ok: a model_list + return [deployment("g", deployment_id, **params) for deployment_id in ids] + + +def unhealthy(*ids: str, age: float = 0.0) -> dict[str, dict[str, object]]: # mutable-ok: health states + return {i: {"is_healthy": False, "timestamp": time.time() - age, "reason": "probe failed"} for i in ids} + + +def failure_kwargs( + model_info: object, exception: Exception, **extra: object +) -> dict[str, object]: # kwargs-ok: logging kwargs # mutable-ok: litellm's logging kwargs + litellm_params: Final = {"metadata": {"model_group": "g"}, "model_info": model_info} + litellm_params.update(extra.pop("litellm_params", {})) # pyright: ignore[reportCallIssue, reportArgumentType] # test extras + return {"exception": exception, "litellm_params": litellm_params, **extra} + + +# --- which failures cool a deployment down ------------------------------------------------------- + + +@pytest.mark.parametrize("builder", [auth_error, not_found]) +@pytest.mark.asyncio +async def test_a_lone_deployment_cools_down_on_a_401_or_404( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=0) + host.answer("openai/a", fresh(builder)) + + first: Final = await attempt(core) + second: Final = await attempt(core) + + assert type(first) is type(builder()) + assert set(cooled_down(core, ["a"])) == {"a"} + assert isinstance(second, litellm.RouterRateLimitError) + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_404_cools_a_group_deployment_down_at_once(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0) + host.answer("*", fresh(not_found)) + + await attempt(core) + + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion") + + +@pytest.mark.parametrize("builder", [permission_denied, bad_request, api_status(422), api_status(409)]) +@pytest.mark.asyncio +async def test_other_4xx_errors_never_cool_down(make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, allowed_fails=0) + host.answer("*", fresh(builder)) + + for _ in range(4): + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.parametrize("builder", [server_error, timeout, service_unavailable]) +@pytest.mark.asyncio +async def test_by_default_a_group_deployment_cools_down_on_its_fifth_failure_this_minute( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0) + host.answer("*", fresh(builder)) + + while not cooled_down(core, ["a", "b"]): + await attempt(core) + + called: Final = host.deployments_called("acompletion") + (cooled,) = cooled_down(core, ["a", "b"]) + assert called[-1] == cooled + assert called.count(cooled) == DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS + + +@pytest.mark.asyncio +async def test_successes_this_minute_keep_the_failure_rate_under_the_default_threshold(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b")) + for _ in range(5): + core.sync_deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + {"litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}}, None, 0.0, 0.0 + ) + + outcomes: Final = [core.health().record_failure("a", server_error(), 500) for _ in range(6)] + + assert outcomes == [False] * 5 + [True] + + +@pytest.mark.parametrize("allowed_fails", [3, 1]) +@pytest.mark.asyncio +async def test_an_explicit_allowed_fails_counts_failures_even_at_the_default_value( + make_core: MakeCore, allowed_fails: int +) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=allowed_fails) + + outcomes: Final = [core.health().record_failure("a", server_error(), 500) for _ in range(allowed_fails + 1)] + + assert outcomes == [False] * allowed_fails + [True] + + +@pytest.mark.asyncio +async def test_a_connection_error_never_cools_down(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, allowed_fails=0) + host.answer("*", fresh(api_connection)) + + for _ in range(3): + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.parametrize(("ids", "cools"), [(("a", "b"), True), (("a",), False)], ids=["group", "lone"]) +@pytest.mark.asyncio +async def test_a_402_cools_a_group_deployment_but_not_a_lone_one( + make_core: MakeCore, host: ScriptedHost, ids: tuple[str, ...], cools: bool +) -> None: + core: Final = make_core(model_list=group(*ids), num_retries=0) + host.answer("*", fresh(api_status(402))) + + await attempt(core) + + assert bool(cooled_down(core, list(ids))) is cools + + +@pytest.mark.asyncio +async def test_a_lone_deployment_with_a_fallback_group_cools_down_under_explicit_allowed_fails( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("f", "z")], + num_retries=0, + allowed_fails=0, + fallbacks=[{"g": ["f"]}], + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert set(cooled_down(core, ["a", "z"])) == {"a"} + assert host.deployments_called("acompletion") == ["a", "z"] + + +@pytest.mark.asyncio +async def test_a_lone_deployment_with_a_fallback_group_is_not_cooled_by_a_429_under_defaults( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("f", "z")], num_retries=0, fallbacks=[{"g": ["f"]}] + ) + host.answer("openai/a", fresh(rate_limit)) + + await attempt(core) + + assert cooled_down(core, ["a", "z"]) == {} + + +@pytest.mark.asyncio +async def test_record_failure_never_cools_down_when_cooldowns_are_disabled(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0, disable_cooldowns=True) + + cooled: Final = core.health().record_failure("a", rate_limit(), 429) + + assert cooled is False + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") == 1 + + +@pytest.mark.parametrize( + ("deployment_id", "status"), [("missing", 429), ("a", ""), ("a", 400)], ids=["unknown", "empty", "400"] +) +@pytest.mark.asyncio +async def test_record_failure_without_a_cooldown(make_core: MakeCore, deployment_id: str, status: object) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + + assert core.health().record_failure(deployment_id, rate_limit(), status) is False # pyright: ignore[reportArgumentType] # status as given + assert core.health().failures_this_minute(deployment_id) == 1 + + +@pytest.mark.asyncio +async def test_a_router_cooldown_time_of_zero_falls_back_to_the_default( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, cooldown_time=0) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert core.settings().cooldown_time == DEFAULT_COOLDOWN_TIME_SECONDS + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == DEFAULT_COOLDOWN_TIME_SECONDS + + +# --- allowed_fails_policy ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_router_allowed_fails_policy_counts_every_error_type_in_one_counter(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=group("a", "b"), + allowed_fails=5, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + ) + + outcomes: Final = [ + core.health().record_failure("a", server_error(), 500), + core.health().record_failure("a", server_error(), 500), + core.health().record_failure("a", rate_limit(), 429), + ] + + assert outcomes == [False, False, True] + + +@pytest.mark.parametrize( + ("field", "error", "status"), + [ + ("AuthenticationErrorAllowedFails", auth_error, 401), + ("TimeoutErrorAllowedFails", timeout, 408), + ("RateLimitErrorAllowedFails", rate_limit, 429), + ("InternalServerErrorAllowedFails", server_error, 500), + ("ServiceUnavailableErrorAllowedFails", service_unavailable, 503), + ("NotFoundErrorAllowedFails", not_found, 404), + ], +) +@pytest.mark.asyncio +async def test_router_allowed_fails_policy_threshold_per_error_type( + make_core: MakeCore, field: str, error: ErrorBuilder, status: int +) -> None: + core: Final = make_core( + model_list=group("a", "b"), allowed_fails=10, allowed_fails_policy=AllowedFailsPolicy(**{field: 2}) + ) + + outcomes: Final = [core.health().record_failure("a", error(), status) for _ in range(3)] + + assert outcomes == [False, False, True] + + +@pytest.mark.asyncio +async def test_router_allowed_fails_policy_for_bad_requests_does_not_cool_down(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=group("a", "b"), allowed_fails_policy=AllowedFailsPolicy(BadRequestErrorAllowedFails=0) + ) + + outcomes: Final = [core.health().record_failure("a", bad_request(), 400) for _ in range(3)] + + assert outcomes == [False, False, False] + + +@pytest.mark.asyncio +async def test_deployment_allowed_fails_policy_counts_each_error_type_separately(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=group( + "a", + "b", + model_info={ + "allowed_fails_policy": {"RateLimitErrorAllowedFails": 1, "InternalServerErrorAllowedFails": 1} + }, + ) + ) + + outcomes: Final = [ + core.health().record_failure("a", rate_limit(), 429), + core.health().record_failure("a", server_error(), 500), + core.health().record_failure("a", rate_limit(), 429), + ] + + assert outcomes == [False, False, True] + + +@pytest.mark.asyncio +async def test_deployment_allowed_fails_policy_wins_over_the_router_policy(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "a", model_info={"allowed_fails_policy": {"RateLimitErrorAllowedFails": 2}}), + deployment("g", "b"), + ], + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=0), + ) + + a_outcomes: Final = [core.health().record_failure("a", rate_limit(), 429) for _ in range(3)] + b_outcome: Final = core.health().record_failure("b", rate_limit(), 429) + + assert a_outcomes == [False, False, True] + assert b_outcome is True + + +@pytest.mark.asyncio +async def test_deployment_policy_not_naming_the_error_falls_back_to_router_allowed_fails(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=group("a", "b", model_info={"allowed_fails_policy": {"RateLimitErrorAllowedFails": 9}}) + ) + + outcomes: Final = [core.health().record_failure("a", server_error(), 500) for _ in range(4)] + + assert outcomes == [False, False, False, True] + + +@pytest.mark.parametrize(("ids", "cools"), [(("a", "b"), True), (("a",), False)], ids=["group", "lone"]) +@pytest.mark.asyncio +async def test_deployment_allowed_fails_in_model_info(make_core: MakeCore, ids: tuple[str, ...], cools: bool) -> None: + core: Final = make_core(model_list=group(*ids, model_info={"allowed_fails": 1})) + + outcomes: Final = [core.health().record_failure("a", server_error(), 500) for _ in range(2)] + + assert outcomes == [False, cools] + + +@pytest.mark.asyncio +async def test_deployment_allowed_fails_policy_cools_down_a_lone_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", model_info={"allowed_fails_policy": {"TimeoutErrorAllowedFails": 0}})) + + assert core.health().record_failure("a", timeout(), 408) is True + + +@pytest.mark.asyncio +async def test_deployment_cooldown_time_in_model_info_sets_the_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=group( + "a", "b", model_info={"cooldown_time": 17, "allowed_fails_policy": {"RateLimitErrorAllowedFails": 0}} + ), + num_retries=0, + ) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == 17 + + +# --- deployment_callback_on_failure ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_failure_callback_counts_the_failure_and_cools_down(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + + cooled: Final = core.deployment_callback_on_failure(failure_kwargs({"id": "a"}, server_error()), None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert cooled is True + assert core.health().failures_this_minute("a") == 1 + assert set(cooled_down(core, ["a", "b"])) == {"a"} + + +@pytest.mark.parametrize("model_info", [None, "a", {}, {"name": "no id"}], ids=["none", "not-a-dict", "empty", "no-id"]) +@pytest.mark.asyncio +async def test_failure_callback_without_a_deployment_id_does_nothing(make_core: MakeCore, model_info: object) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + + cooled: Final = core.deployment_callback_on_failure(failure_kwargs(model_info, server_error()), None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert cooled is False + assert core.health().failures_this_minute("a") == 0 + + +@pytest.mark.asyncio +async def test_failure_callback_ignores_an_advisor_orchestration_failure(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + error: Final = server_error() + mark_advisor_orchestration_failure(error) + + cooled: Final = core.deployment_callback_on_failure(failure_kwargs({"id": "a"}, error), None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert cooled is False + assert core.health().failures_this_minute("a") == 0 + + +@pytest.mark.parametrize( + "extra", + [ + { + "litellm_params": { + "metadata": {"model_group": "g", "internal_call_origin": "background_response_status_poll"} + } + }, + {"call_type": "avideo_status"}, + ], + ids=["background-poll", "video-lookup"], +) +@pytest.mark.asyncio +async def test_failure_callback_ignores_a_404_that_is_not_about_the_deployment( + make_core: MakeCore, extra: dict[str, object] +) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + + cooled: Final = core.deployment_callback_on_failure( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + failure_kwargs({"id": "a"}, not_found(), **extra), None, 0.0, 0.0 + ) + + assert cooled is False + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.asyncio +async def test_failure_callback_ignores_a_408_from_the_callers_own_timeout(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + started: Final = time.time() + from datetime import datetime, timedelta + + start: Final = datetime.fromtimestamp(started) + kwargs: Final = failure_kwargs( + {"id": "a"}, + timeout(), + litellm_params={"client_side_timeout": True, "timeout": 1.0}, + start_time=start, + end_time=start + timedelta(seconds=2), + ) + + cooled: Final = core.deployment_callback_on_failure(kwargs, None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert cooled is False + assert core.health().failures_this_minute("a") == 0 + + +@pytest.mark.parametrize( + ("model_info", "litellm_params", "expected"), + [ + ({"id": "a", "cooldown_time": 3}, {"cooldown_time": 4}, 3), + ({"id": "a"}, {"cooldown_time": 4}, 4), + ({"id": "a", "cooldown_time": -1}, {}, 60), + ], + ids=["model-info-first", "litellm-params", "negative-ignored"], +) +@pytest.mark.asyncio +async def test_failure_callback_cooldown_time_resolution( + make_core: MakeCore, model_info: dict[str, object], litellm_params: dict[str, object], expected: float +) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0, cooldown_time=60) + + core.deployment_callback_on_failure( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + failure_kwargs(model_info, server_error(), litellm_params=litellm_params), None, 0.0, 0.0 + ) + + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == expected + + +# --- success callbacks and usage counters -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sync_success_callback_counts_a_success_and_returns_its_key(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b")) + kwargs: Final = {"litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}} + + key: Final = core.sync_deployment_callback_on_success(kwargs, None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + core.sync_deployment_callback_on_success(kwargs, None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert key == "a:successes" + assert core.health().store.get_cache("a:successes", local_only=True) == 2 + + +@pytest.mark.asyncio +async def test_sync_success_callback_turns_an_integer_id_into_a_string_key(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a")) + + key: Final = core.sync_deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + {"litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": 7}}}, None, 0.0, 0.0 + ) + + assert key == "7:successes" + + +@pytest.mark.parametrize( + "kwargs", + [ + {"litellm_params": {"metadata": None, "model_info": {"id": "a"}}}, + {"litellm_params": {"metadata": {}, "model_info": {"id": "a"}}}, + {"litellm_params": {"metadata": {"model_group": "g"}, "model_info": None}}, + { + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}, + }, + ], + ids=["no-metadata", "no-group", "no-model-info", "batch-retrieve"], +) +@pytest.mark.asyncio +async def test_sync_success_callback_counts_nothing_without_a_group_and_id( + make_core: MakeCore, kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=group("a")) + + assert core.sync_deployment_callback_on_success(kwargs, None, 0.0, 0.0) is None # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + assert core.health().store.get_cache("a:successes", local_only=True) is None + + +@pytest.mark.parametrize( + "kwargs", + [ + {"call_type": "_arealtime"}, + {"call_type": "aretrieve_batch"}, + {"litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}}, + ], + ids=["realtime", "batch-retrieve", "no-standard-logging-object"], +) +@pytest.mark.asyncio +async def test_async_success_callback_skips_calls_it_does_not_track( + make_core: MakeCore, kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=group("a", rpm=100)) + full: Final = { + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}, + "standard_logging_object": {"total_tokens": 10}, + **kwargs, + } + if "litellm_params" in kwargs: + del full["standard_logging_object"] + + result: Final = await core.deployment_callback_on_success(full, None, 0.0, 0.0) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + assert result is None + assert core.health().store.get_cache("a:successes", local_only=True) is None + assert await core.health().usage("g") == (None, None) + + +@pytest.mark.asyncio +async def test_async_success_callback_for_an_unknown_deployment_counts_nothing(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", rpm=100)) + + await core.deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + { + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "zz"}}, + "standard_logging_object": {"total_tokens": 10}, + }, + None, + 0.0, + 0.0, + ) + + assert core.health().store.get_cache("zz:successes", local_only=True) is None + + +@pytest.mark.asyncio +async def test_async_success_callback_writes_tokens_and_a_request_for_a_tracked_deployment( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=group("a", rpm=100)) + + await core.deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + { + "litellm_params": { + "metadata": {"model_group": "g", "deployment": "openai/a"}, + "model_info": {"id": "a"}, + }, + "standard_logging_object": {"total_tokens": 12}, + }, + None, + 0.0, + 0.0, + ) + + assert core.health().store.get_cache("a:successes", local_only=True) == 1 + assert await core.health().usage("g") == (12, 1) + + +@pytest.mark.asyncio +async def test_async_success_callback_adds_only_tokens_not_yet_counted(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", rpm=100)) + + await core.deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + { + "litellm_params": { + "metadata": { + "model_group": "g", + "deployment": "openai/a", + "_litellm_router_usage_counted_tokens": 8, + }, + "model_info": {"id": "a"}, + }, + "standard_logging_object": {"total_tokens": 12}, + }, + None, + 0.0, + 0.0, + ) + + assert await core.health().usage("g") == (4, 0) + + +@pytest.mark.asyncio +async def test_async_success_callback_counts_successes_for_an_untracked_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a")) + + await core.deployment_callback_on_success( # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + { + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}, + "standard_logging_object": {"total_tokens": 12}, + }, + None, + 0.0, + 0.0, + ) + + assert core.health().store.get_cache("a:successes", local_only=True) == 1 + assert await core.health().usage("g") == (None, None) + + +@pytest.mark.parametrize("limit", [{"rpm": 100}, {"tpm": 1000}], ids=["rpm", "tpm"]) +@pytest.mark.asyncio +async def test_a_successful_request_counts_its_tokens_and_one_request_once( + make_core: MakeCore, host: ScriptedHost, limit: dict[str, int] +) -> None: + core: Final = make_core(model_list=group("a", **limit)) + + await attempt(core) + if isinstance(host, BackgroundSuccessLoggingHost): + await host.drain() + + assert await core.health().usage("g") == (10, 1) + + +@pytest.fixture +def host(request: pytest.FixtureRequest) -> ScriptedHost: + """Background success logging for the usage test above, the plain host for the rest.""" + if request.node.name.startswith("test_a_successful_request_counts_its_tokens"): # pyright: ignore[reportAny] # pytest node + return BackgroundSuccessLoggingHost() + return ScriptedHost() + + +@pytest.mark.asyncio +async def test_a_failed_request_counts_one_request_and_no_tokens(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", rpm=100), num_retries=0) + host.answer("openai/a", fresh(bad_request)) + + await attempt(core) + + assert await core.health().usage("g") == (None, 1) + assert core.health().failures_this_minute("a") == 1 + + +@pytest.mark.asyncio +async def test_an_untracked_deployment_reports_no_usage(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a")) + + await attempt(core) + + assert await core.health().usage("g") == (None, None) + + +@pytest.mark.asyncio +async def test_a_successful_request_counts_one_success_this_minute(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a")) + + await attempt(core) + await attempt(core) + + assert core.health().store.get_cache("a:successes", local_only=True) == 2 + assert core.health().failures_this_minute("a") == 0 + + +# --- blocked deployments ------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_blocked_deployment_is_never_called(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model_info={"blocked": True}), deployment("g", "b")]) + + for _ in range(4): + await attempt(core) + + assert host.deployments_called("acompletion") == ["b"] * 4 + assert core.health().fully_blocked_model_names() == set() + + +@pytest.mark.asyncio +async def test_a_group_with_every_deployment_blocked_raises_no_deployments_available( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b", model_info={"blocked": True})) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RouterRateLimitError) + assert "No deployments available for selected model" in str(error) + assert host.calls == [] + assert core.health().fully_blocked_model_names() == {"g"} + + +def test_is_blocked_and_all_blocked_read_model_info(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a")) + blocked: Final = Deployment(**deployment("g", "a", model_info={"blocked": True})) # pyright: ignore[reportArgumentType] # a model_list entry + open_: Final = Deployment(**deployment("g", "b")) # pyright: ignore[reportArgumentType] # a model_list entry + + assert core.health().is_blocked(blocked) is True + assert core.health().is_blocked(open_) is False + assert core.health().all_blocked([blocked.model_dump()]) is True # pyright: ignore[reportArgumentType] # deployment dicts + assert core.health().all_blocked([blocked.model_dump(), open_.model_dump()]) is False # pyright: ignore[reportArgumentType] # deployment dicts + + +# --- health-check-driven filtering --------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_health_check_routing_skips_deployments_marked_unhealthy(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), enable_health_check_routing=True) + core.health().health_checks.set_deployment_health_states(unhealthy("a")) # pyright: ignore[reportArgumentType] # health states + + for _ in range(4): + await attempt(core) + + assert host.deployments_called("acompletion") == ["b"] * 4 + assert await core.health().fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_health_check_routing_uses_every_deployment_when_all_are_unhealthy( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), enable_health_check_routing=True) + core.health().health_checks.set_deployment_health_states(unhealthy("a", "b")) # pyright: ignore[reportArgumentType] # health states + + for _ in range(12): + await attempt(core) + + assert set(host.deployments_called("acompletion")) == {"a", "b"} + assert await core.health().fully_unhealthy_model_names() == {"g"} + + +@pytest.mark.parametrize( + ("arguments", "age"), + [ + ({}, 0.0), + ( + { + "enable_health_check_routing": True, + "allowed_fails_policy": AllowedFailsPolicy(TimeoutErrorAllowedFails=1), + }, + 0.0, + ), + ({"enable_health_check_routing": True, "health_check_staleness_threshold": 10}, 30.0), + ({"enable_health_check_routing": True, "background_health_check_model_groups": ["other"]}, 0.0), + ], + ids=["routing-off", "allowed-fails-policy", "stale", "group-not-listed"], +) +@pytest.mark.asyncio +async def test_health_check_state_is_ignored( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], age: float +) -> None: + core: Final = make_core(model_list=group("a", "b"), **arguments) + core.health().health_checks.set_deployment_health_states(unhealthy("a", age=age)) # pyright: ignore[reportArgumentType] # health states + + for _ in range(12): + await attempt(core) + + assert set(host.deployments_called("acompletion")) == {"a", "b"} + + +@pytest.mark.asyncio +async def test_listed_health_check_group_is_filtered_even_with_an_allowed_fails_policy( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=group("a", "b"), + enable_health_check_routing=True, + background_health_check_model_groups=["g"], + allowed_fails_policy=AllowedFailsPolicy(TimeoutErrorAllowedFails=1), + ) + core.health().health_checks.set_deployment_health_states(unhealthy("a")) # pyright: ignore[reportArgumentType] # health states + + for _ in range(4): + await attempt(core) + + assert host.deployments_called("acompletion") == ["b"] * 4 + + +@pytest.mark.asyncio +async def test_an_unhealthy_deployment_stays_excluded_when_the_healthy_one_is_cooling_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), enable_health_check_routing=True, num_retries=0) + core.health().health_checks.set_deployment_health_states(unhealthy("a")) # pyright: ignore[reportArgumentType] # health states + host.answer("openai/b", fresh(rate_limit)) + + await attempt(core) + error: Final = await attempt(core) + + assert set(cooled_down(core, ["a", "b"])) == {"b"} + assert isinstance(error, litellm.RouterRateLimitError) + assert "cooldown_list=['b']" in str(error) + assert host.deployments_called("acompletion") == ["b"] + + +def test_a_cooldown_recorded_outside_an_event_loop_raises_after_writing_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a", "b"), allowed_fails=0) + + with pytest.raises(RuntimeError, match="no running event loop"): + core.health().record_failure("a", server_error(), 500) + + assert set(cooled_down(core, ["a", "b"])) == {"a"} + + +@pytest.mark.asyncio +async def test_a_request_cooldown_time_reaches_the_call_and_sets_the_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0) + host.answer("*", fresh(rate_limit)) + + await attempt(core, cooldown_time=11) + + assert host.calls[0].kwargs["cooldown_time"] == 11 + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == 11 diff --git a/tests/unit/test_router/core/test_retries_health_characterization_failure_exits.py b/tests/unit/test_router/core/test_retries_health_characterization_failure_exits.py new file mode 100644 index 00000000000..63f261f1f09 --- /dev/null +++ b/tests/unit/test_router/core/test_retries_health_characterization_failure_exits.py @@ -0,0 +1,365 @@ +""" +Characterization of the failures a router core does not hold against a deployment, on every path a +failure is recorded from: the early exits of ``deployment_callback_on_failure`` (nothing counted, no +cooldown), the per-caller exits of ``_should_run_cooldown_logic`` (counted, no cooldown), and what else +decides a cooldown: the ``retry-after`` header, ``litellm.allowed_fails`` and a team deployment's siblings. +""" + +import asyncio +import email.utils +from datetime import datetime, timedelta, timezone +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.router_core import RouterCore +from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure +from litellm.types.router import AllowedFailsPolicy +from litellm.types.utils import BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN +from tests.unit.test_router.core.retries_health_helpers import ( + ErrorBuilder, + MakeCore, + attempt, + attempt_sync, + cooled_down, + fresh, + not_found, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + deployment, + rate_limit, + server_error, +) +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts, collect_texts_sync + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +TOKEN_EXCHANGE: Final = "http://127.0.0.1:9/token" +PER_USER_COPILOT: Final = { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "ghc", + "github_copilot_auth_type": "per_user_oauth", +} + + +def group( + *ids: str, **params: object +) -> list[dict[str, object]]: # kwargs-ok: litellm_params # mutable-ok: a model_list + return [deployment("g", deployment_id, **params) for deployment_id in ids] + + +def selection_failure() -> Exception: + error: Final = rate_limit() + mark_deployment_selection_failure(error) + return error + + +def caller_auth_error() -> Exception: + return litellm.CallerCredentialAuthenticationError( + message="not connected", llm_provider="github_copilot", model="gpt-4o" + ) + + +def caller_rate_limit() -> Exception: + return litellm.CallerCredentialRateLimitError( + message="token exchange limited", llm_provider="github_copilot", model="gpt-4o" + ) + + +def rate_limited_with(headers: dict[str, str]) -> litellm.RateLimitError: # mutable-ok: httpx's header mapping + return litellm.RateLimitError( + message="slow down", llm_provider="openai", model="gpt-4o", response=httpx.Response(429, headers=headers) + ) + + +async def settle() -> None: + """Let the logging and recording tasks a finished stream scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +# --- U1: deployment_callback_on_failure's early exits ----------------------------------------------- + + +@pytest.mark.parametrize( + ("builder", "params"), + [(selection_failure, {}), (auth_error, {"token_exchange_endpoint": TOKEN_EXCHANGE})], + ids=["selection-failure", "caller-scoped-oauth"], +) +@pytest.mark.asyncio +async def test_an_attempt_failure_outside_the_deployment_is_neither_counted_nor_cooled( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, params: dict[str, object] +) -> None: + core: Final = make_core(model_list=group("a", "b", **params), num_retries=0, allowed_fails=0, cooldown_time=60) + host.answer("*", fresh(builder)) + + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 0 + + +@pytest.mark.parametrize( + ("builder", "params"), + [(selection_failure, {}), (auth_error, {"token_exchange_endpoint": TOKEN_EXCHANGE})], + ids=["selection-failure", "caller-scoped-oauth"], +) +def test_a_sync_attempt_failure_outside_the_deployment_is_neither_counted_nor_cooled( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, params: dict[str, object] +) -> None: + core: Final = make_core(model_list=group("a", "b", **params), num_retries=0, allowed_fails=0, cooldown_time=60) + host.answer("*", fresh(builder)) + + attempt_sync(core) + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 0 + + +@pytest.mark.asyncio +async def test_a_background_poll_404_on_an_attempt_is_neither_counted_nor_cooled( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, allowed_fails=0) + host.answer("*", fresh(not_found)) + + await attempt(core, metadata={INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN}) + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 0 + + +@pytest.mark.asyncio +async def test_a_stream_failure_outside_the_deployment_still_counts_its_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", error=selection_failure(), logged=True)) + core: Final = make_core( + model_list=[deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + num_retries=0, + allowed_fails=0, + ) + + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + with pytest.raises(litellm.RateLimitError): + await collect_texts(stream) + await settle() + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") == 0 + # `async_deployment_callback_on_failure` makes none of the failure callback's exits. + assert await core.health().usage("g") == (0, 2) + + +def test_a_sync_stream_failure_outside_the_deployment_is_neither_counted_nor_cooled( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", error=selection_failure(), logged=True, sync=True)) + core: Final = make_core( + model_list=[deployment("g", "a", order=1), deployment("g", "b", order=2)], num_retries=0, allowed_fails=0 + ) + + stream: Final = core.completion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # every core serves completion + with pytest.raises(litellm.RateLimitError): + collect_texts_sync(stream) + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") == 0 + + +@pytest.mark.asyncio +async def test_the_failure_hook_counts_a_failure_outside_the_deployment_as_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, allowed_fails=0) + logged: Final = { + "exception": selection_failure(), + "litellm_params": {"metadata": {"model_group": "g"}, "model_info": {"id": "a"}}, + } + + assert core.deployment_callback_on_failure(logged, None, None, None) is False # pyright: ignore[reportAttributeAccessIssue] # every core serves it + assert core.health().failures_this_minute("a") == 0 + + +# --- N13: _should_run_cooldown_logic's per-caller exits --------------------------------------------- + + +@pytest.mark.parametrize("builder", [caller_auth_error, caller_rate_limit], ids=["authentication", "rate-limit"]) +@pytest.mark.asyncio +async def test_a_caller_credential_error_is_counted_but_never_cools_a_shared_deployment( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, allowed_fails=0) + host.answer("*", fresh(builder)) + + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 1 + + +@pytest.mark.parametrize("builder", [auth_error, server_error], ids=["401", "500"]) +@pytest.mark.asyncio +async def test_a_per_user_copilot_deployment_is_counted_but_never_cools_down( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder +) -> None: + core: Final = make_core( + model_list=[deployment("g", "gh", weight=1, **PER_USER_COPILOT), deployment("g", "other", weight=0)], + num_retries=0, + allowed_fails=0, + ) + host.answer("*", fresh(builder)) + + await attempt(core) + + assert cooled_down(core, ["gh", "other"]) == {} + assert core.health().failures_this_minute("gh") == 1 + + +# --- S3: the retry-after header ---------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("headers", "expected_seconds"), + [ + ({"retry-after-ms": "3000", "retry-after": "8"}, 8), + ({"retry-after-ms": "7000"}, 5), + ({"retry-after": "2.5"}, 5), + ({"retry-after": "soon"}, 5), + ({"retry-after": "-3"}, 5), + ({"retry-after": " 7 "}, 7), + ], + ids=["seconds-over-ms", "ms-ignored", "float-ignored", "unreadable", "negative", "padded-int"], +) +@pytest.mark.asyncio +async def test_a_cooldown_takes_its_time_from_retry_after_read_as_an_int_only( + make_core: MakeCore, host: ScriptedHost, headers: dict[str, str], expected_seconds: int +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, cooldown_time=5) + host.answer("*", lambda **_: rate_limited_with(headers)) + + await attempt(core) + + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == expected_seconds + + +@pytest.mark.asyncio +async def test_a_cooldown_reads_a_retry_after_date(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=0, cooldown_time=5) + until: Final = email.utils.format_datetime(datetime.now(timezone.utc) + timedelta(seconds=120), usegmt=True) + host.answer("*", lambda **_: rate_limited_with({"retry-after": until})) + + await attempt(core) + + (entry,) = cooled_down(core, ["a", "b"]).values() + cooldown_time: Final = entry["cooldown_time"] + assert isinstance(cooldown_time, int | float) and 100 <= cooldown_time <= 120 + + +# --- U5: litellm.allowed_fails ----------------------------------------------------------------------- + + +def weighted() -> list[dict[str, object]]: # mutable-ok: a model_list + return [deployment("g", "a", weight=1), deployment("g", "b", weight=0)] + + +@pytest.mark.asyncio +async def test_a_write_to_the_process_allowed_fails_starts_the_count_at_the_copied_value( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + core: Final = make_core(model_list=weighted(), num_retries=0) + monkeypatch.setattr(litellm, "allowed_fails", 0) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert core.settings().allowed_fails == 3 + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.asyncio +async def test_a_process_allowed_fails_put_back_ends_the_count( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + core: Final = make_core(model_list=weighted(), num_retries=0) + host.answer("*", fresh(rate_limit)) + monkeypatch.setattr(litellm, "allowed_fails", 0) + await attempt(core) + assert cooled_down(core, ["a", "b"]) == {} + + monkeypatch.setattr(litellm, "allowed_fails", 3) + await attempt(core) + + assert list(cooled_down(core, ["a", "b"])) == ["a"] + + +@pytest.mark.asyncio +async def test_the_allowed_fails_count_is_the_process_value_copied_at_init( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "allowed_fails", 1) + host.set_dynamic(allowed_fails=1) + core: Final = make_core( + model_list=weighted(), num_retries=0, allowed_fails_policy=AllowedFailsPolicy(TimeoutErrorAllowedFails=0) + ) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + assert cooled_down(core, ["a", "b"]) == {} + + await attempt(core) + assert list(cooled_down(core, ["a", "b"])) == ["a"] + + +# --- U10: a team deployment's siblings -------------------------------------------------------------- + + +def team_deployment( + name: str, deployment_id: str, **info: object +) -> dict[str, object]: # kwargs-ok: model_info # mutable-ok: a model_list entry + return deployment( + name, + deployment_id, + weight=1 if deployment_id == "ta1" else 0, + model_info={"team_id": "T1", "team_public_model_name": "tp", **info}, + ) + + +async def call_team_model(core: RouterCore) -> object: + return await attempt(core, "tp", metadata={"user_api_key_team_id": "T1"}) + + +@pytest.mark.asyncio +async def test_a_team_deployment_with_a_sibling_under_its_public_name_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[team_deployment("t1a", "ta1"), team_deployment("t1b", "ta2")], num_retries=0) + host.answer("*", fresh(rate_limit)) + + await call_team_model(core) + + assert list(cooled_down(core, ["ta1", "ta2"])) == ["ta1"] + + +@pytest.mark.parametrize("others", [(), (("t1b", "ta2", {"blocked": True}),)], ids=["alone", "blocked-sibling"]) +@pytest.mark.asyncio +async def test_a_team_deployment_with_no_routable_sibling_stays_a_lone_deployment( + make_core: MakeCore, host: ScriptedHost, others: tuple[tuple[str, str, dict[str, object]], ...] +) -> None: + core: Final = make_core( + model_list=[ + team_deployment("t1a", "ta1"), + *(team_deployment(name, deployment_id, **info) for name, deployment_id, info in others), + ], + num_retries=0, + ) + host.answer("*", fresh(rate_limit)) + + await call_team_model(core) + + assert cooled_down(core, ["ta1", "ta2"]) == {} diff --git a/tests/unit/test_router/core/test_retries_health_characterization_retries.py b/tests/unit/test_router/core/test_retries_health_characterization_retries.py new file mode 100644 index 00000000000..5c6c885a59f --- /dev/null +++ b/tests/unit/test_router/core/test_retries_health_characterization_retries.py @@ -0,0 +1,683 @@ +""" +Characterization of a router core's retry loop: which errors retry, how many times, how long it sleeps, +what it records on the request and the final error, and the public retry helpers. +""" + +from typing import Final + +import pytest + +import litellm +from litellm.constants import INITIAL_RETRY_DELAY, JITTER, MAX_RETRY_DELAY +from litellm.exceptions import GuardrailRaisedException +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure +from litellm.types.router import RetryPolicy +from litellm.types.utils import Usage +from tests.unit.test_router.core.retries_health_helpers import ( + ErrorBuilder, + MakeCore, + MetadataRecorder, + api_connection, + api_status, + attempt, + attempt_sync, + bad_gateway, + cooled_down, + fresh, + headers_of, + http_response, + not_found, + permission_denied, + plain_error, + rate_limited, + service_unavailable, + unprocessable, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + agentic_followup_failure, + auth_error, + bad_request, + content_policy, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) +from tests.unit.test_router.core.streaming_helpers import request_logging + + +def group( + *ids: str, **params: object +) -> list[dict[str, object]]: # kwargs-ok: litellm_params # mutable-ok: a model_list + return [deployment("g", deployment_id, **params) for deployment_id in ids] + + +def within(slept: float, base: float) -> bool: + return base <= slept < base + JITTER + + +# --- which errors retry, by error class and status ------------------------------------------------ + + +@pytest.mark.parametrize( + ("builder", "lone_calls", "pair_calls"), + [ + (not_found, 1, 1), + (permission_denied, 1, 3), + (unprocessable, 1, 1), + (service_unavailable, 3, 3), + (bad_gateway, 3, 3), + (api_connection, 3, 3), + (api_status(409), 3, 3), + (plain_error, 3, 3), + ], + ids=["404", "403", "422", "503", "502", "connection", "409", "no-status"], +) +@pytest.mark.parametrize("size", ["lone", "pair"]) +@pytest.mark.asyncio +async def test_retry_decision_per_error_class_and_group_size( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, lone_calls: int, pair_calls: int, size: str +) -> None: + core: Final = make_core(model_list=group("a") if size == "lone" else group("a", "b"), num_retries=2) + host.answer("*", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == (lone_calls if size == "lone" else pair_calls) + + +@pytest.mark.asyncio +async def test_an_error_without_a_litellm_type_is_retried_but_not_stamped( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + host.answer("openai/a", fresh(plain_error)) + + error: Final = await attempt(core) + + assert isinstance(error, ValueError) + assert getattr(error, "num_retries", None) is None + assert getattr(error, "max_retries", None) is None + assert "LiteLLM Retried" not in str(error) + + +@pytest.mark.asyncio +async def test_a_403_retry_moves_to_a_deployment_that_has_not_refused_yet( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b", "c"), num_retries=4) + host.answer("*", fresh(permission_denied)) + + error: Final = await attempt(core) + + called: Final = host.deployments_called("acompletion") + assert isinstance(error, litellm.PermissionDeniedError) + assert len(called) == 5 + assert sorted(called[:3]) == ["a", "b", "c"] + assert cooled_down(core, ["a", "b", "c"]) == {} + + +@pytest.mark.asyncio +async def test_a_401_on_each_deployment_cools_each_down_and_stops_when_none_is_left( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b", "c"), num_retries=4) + host.answer("*", fresh(auth_error)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.AuthenticationError) + assert sorted(host.deployments_called("acompletion")) == ["a", "b", "c"] + assert set(cooled_down(core, ["a", "b", "c"])) == {"a", "b", "c"} + + +@pytest.mark.asyncio +async def test_a_non_retryable_error_mid_loop_stops_the_retries(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=4) + host.answer("openai/a", fresh(server_error), fresh(bad_request)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.BadRequestError) + assert len(host.calls_to("acompletion")) == 2 + assert getattr(error, "num_retries", None) is None + + +@pytest.mark.asyncio +async def test_a_retry_policy_keeps_retrying_through_a_later_non_retryable_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a"), retry_policy=RetryPolicy(InternalServerErrorRetries=3)) + host.answer("openai/a", fresh(server_error), fresh(bad_request)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.BadRequestError) + assert len(host.calls_to("acompletion")) == 4 + assert getattr(error, "num_retries", None) == 3 + + +@pytest.mark.asyncio +async def test_a_guardrail_block_is_raised_without_retrying(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=3) + host.answer("*", lambda **_: GuardrailRaisedException(guardrail_name="pii", message="blocked by guardrail")) + + error: Final = await attempt(core) + + assert isinstance(error, GuardrailRaisedException) + assert len(host.calls_to("acompletion")) == 1 + assert host.sleeps == [] + + +@pytest.mark.asyncio +async def test_a_failed_agentic_followup_is_raised_without_retrying(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=3) + host.answer("*", lambda **_: agentic_followup_failure(server_error())) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.InternalServerError) + assert len(host.calls_to("acompletion")) == 1 + assert host.sleeps == [] + + +@pytest.mark.asyncio +async def test_a_retry_that_fails_its_agentic_followup_ends_the_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=3) + host.answer("*", server_error(), lambda **_: agentic_followup_failure(server_error()), None) + + error: Final = await attempt(core) + + assert isinstance(error, Exception) + assert is_agentic_followup_failure(error) + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_a_retry_carries_the_superseded_attempt_usage(make_core: MakeCore, host: ScriptedHost) -> None: + logging_obj: Final = request_logging() + usage: Final = Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2) + + def failed_after_usage(**_: object) -> Exception: + logging_obj.model_call_details["combined_usage_object"] = usage + return server_error() + + core: Final = make_core(model_list=group("a", "b"), num_retries=1) + host.answer("*", failed_after_usage, None) + + await attempt(core, litellm_logging_obj=logging_obj) + + assert logging_obj.model_call_details["failed_attempts_usage"] == ((usage, 0.0),) + + +# --- retry policies, more error types -------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("policy", "builder", "expected_calls"), + [ + (RetryPolicy(ServiceUnavailableErrorRetries=2), service_unavailable, 3), + (RetryPolicy(InternalServerErrorRetries=2), service_unavailable, 1 + 2), + (RetryPolicy(DefaultRetries=0), api_status(409), 1), + (RetryPolicy(DefaultRetries=2), unprocessable, 3), + (RetryPolicy(BadRequestErrorRetries=1), content_policy, 2), + (RetryPolicy(BadRequestErrorRetries=1, ContentPolicyViolationErrorRetries=0), content_policy, 1), + ], + ids=[ + "service-unavailable", + "503-falls-back-to-internal-server-error", + "default-for-409", + "default-for-422", + "content-policy-inherits-bad-request", + "content-policy-specific-wins", + ], +) +@pytest.mark.asyncio +async def test_retry_policy_field_chosen_for_each_error( + make_core: MakeCore, host: ScriptedHost, policy: RetryPolicy, builder: ErrorBuilder, expected_calls: int +) -> None: + core: Final = make_core(model_list=group("a"), retry_policy=policy) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == expected_calls + + +@pytest.mark.parametrize( + ("policy", "builder"), + [ + (RetryPolicy(AuthenticationErrorRetries=2), auth_error), + (RetryPolicy(NotFoundErrorRetries=2), not_found), + (RetryPolicy(DefaultRetries=1), not_found), + ], + ids=["auth", "not-found", "default-for-404"], +) +@pytest.mark.asyncio +async def test_retry_policy_retries_a_401_or_404_into_the_cooled_lone_deployment( + make_core: MakeCore, host: ScriptedHost, policy: RetryPolicy, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=group("a"), retry_policy=policy) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RouterRateLimitError) + assert "All deployments for selected model are in cooldown" in str(error) + assert len(host.calls_to("acompletion")) == 1 + + +@pytest.mark.parametrize( + ("policy", "builder"), + [(RetryPolicy(AuthenticationErrorRetries=2), auth_error), (RetryPolicy(NotFoundErrorRetries=2), not_found)], + ids=["auth", "not-found"], +) +@pytest.mark.asyncio +async def test_retry_policy_retries_a_401_or_404_when_cooldowns_are_off( + make_core: MakeCore, host: ScriptedHost, policy: RetryPolicy, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=group("a"), retry_policy=policy, disable_cooldowns=True) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == 3 + + +@pytest.mark.asyncio +async def test_retry_policy_wins_over_the_deployment_num_retries(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=group("a", num_retries=4), retry_policy=RetryPolicy(InternalServerErrorRetries=1) + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_request_model_group_retry_policy_overrides_the_router_one( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=group("a"), model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=0)} + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core, model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=3)}) + + assert len(host.calls_to("acompletion")) == 4 + + +@pytest.mark.asyncio +async def test_model_group_retry_policy_given_as_a_dict(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=group("a"), num_retries=0, model_group_retry_policy={"g": {"InternalServerErrorRetries": 2}} + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 3 + + +@pytest.mark.parametrize( + ("error", "model_group", "expected"), + [ + (server_error(), "g", 1), + (server_error(), "h", 4), + (server_error(), None, 4), + (rate_limit(), "g", None), + (rate_limit(), "h", 2), + (bad_request(), "h", 7), + (context_window(), "h", 7), + (not_found(), "h", 7), + ], +) +def test_get_num_retries_from_retry_policy( + make_core: MakeCore, error: Exception, model_group: str | None, expected: int | None +) -> None: + core: Final = make_core( + model_list=group("a"), + retry_policy=RetryPolicy(InternalServerErrorRetries=4, RateLimitErrorRetries=2, DefaultRetries=7), + model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=1)}, + ) + + assert core.get_num_retries_from_retry_policy(error, model_group=model_group) == expected # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + +def test_get_num_retries_from_retry_policy_without_any_policy_is_none(make_core: MakeCore) -> None: + core: Final = make_core(model_list=group("a"), num_retries=3) + + assert core.get_num_retries_from_retry_policy(server_error(), model_group="g") is None # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + + +# --- should_retry_this_error --------------------------------------------------------------------- + +HEALTHY_ONE: Final = [{"model_info": {"id": "a"}}] +ALL_TWO: Final = [{"model_info": {"id": "a"}}, {"model_info": {"id": "b"}}] + + +@pytest.mark.parametrize( + ("error", "arguments", "retries"), + [ + (server_error(), {"healthy_deployments": HEALTHY_ONE, "all_deployments": HEALTHY_ONE}, True), + (server_error(), {"healthy_deployments": [], "all_deployments": HEALTHY_ONE}, False), + (server_error(), {}, False), + (timeout(), {"healthy_deployments": HEALTHY_ONE}, True), + (plain_error(), {"healthy_deployments": HEALTHY_ONE}, True), + (plain_error(), {"healthy_deployments": None}, False), + (bad_request(), {"healthy_deployments": HEALTHY_ONE}, False), + (context_window(), {"healthy_deployments": HEALTHY_ONE, "context_window_fallbacks": []}, False), + (content_policy(), {"healthy_deployments": HEALTHY_ONE, "content_policy_fallbacks": []}, False), + (not_found(), {"healthy_deployments": ALL_TWO, "all_deployments": ALL_TWO}, False), + (unprocessable(), {"healthy_deployments": ALL_TWO, "all_deployments": ALL_TWO}, False), + (auth_error(), {"healthy_deployments": HEALTHY_ONE, "all_deployments": HEALTHY_ONE}, False), + (auth_error(), {"healthy_deployments": HEALTHY_ONE, "all_deployments": ALL_TWO}, True), + (auth_error(), {"healthy_deployments": [], "all_deployments": ALL_TWO}, False), + (permission_denied(), {"healthy_deployments": HEALTHY_ONE, "all_deployments": HEALTHY_ONE}, False), + (permission_denied(), {"healthy_deployments": HEALTHY_ONE, "all_deployments": ALL_TWO}, True), + (rate_limit(), {"healthy_deployments": HEALTHY_ONE, "regular_fallbacks": [{"g": ["h"]}]}, True), + (rate_limit(), {"healthy_deployments": [], "regular_fallbacks": [{"g": ["h"]}]}, False), + (rate_limit(), {"healthy_deployments": [], "regular_fallbacks": []}, False), + (api_status(409)(), {"healthy_deployments": HEALTHY_ONE}, True), + (service_unavailable(), {"healthy_deployments": HEALTHY_ONE}, True), + ], +) +def test_should_retry_this_error_returns_true_or_raises_the_error( + make_core: MakeCore, error: Exception, arguments: dict[str, object], retries: bool +) -> None: + core: Final = make_core(model_list=group("a", "b")) + + if retries: + assert core.should_retry_this_error(error=error, **arguments) is True # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + else: + with pytest.raises(type(error)) as raised: + core.should_retry_this_error(error=error, **arguments) # pyright: ignore[reportAttributeAccessIssue] # PythonCore's public member + assert raised.value is error + + +# --- mock rate limit errors ---------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_mock_rate_limit_error_raises_without_calling_the_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=0) + + error: Final = await attempt(core, mock_testing_rate_limit_error=True) + + assert isinstance(error, litellm.RateLimitError) + assert "This is a mock exception for model=g, to trigger a rate limit error." in str(error) + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_mock_rate_limit_error_is_retried_and_the_retry_reaches_the_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + + response: Final = await attempt(core, mock_testing_rate_limit_error=True) + + assert not isinstance(response, Exception) + assert host.deployments_called("acompletion") == ["a"] + assert "mock_testing_rate_limit_error" not in host.calls[0].kwargs + assert headers_of(response)["x-litellm-attempted-retries"] == 1 + assert headers_of(response)["x-litellm-max-retries"] == 2 + + +@pytest.mark.asyncio +async def test_mock_rate_limit_error_carries_a_lone_deployments_num_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", num_retries=1), num_retries=5) + + response: Final = await attempt(core, mock_testing_rate_limit_error=True) + + assert headers_of(response)["x-litellm-max-retries"] == 1 + + +@pytest.mark.parametrize("value", [False, None]) +@pytest.mark.asyncio +async def test_mock_rate_limit_error_only_fires_when_true( + make_core: MakeCore, host: ScriptedHost, value: object +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=0) + + response: Final = await attempt(core, mock_testing_rate_limit_error=value) + + assert not isinstance(response, Exception) + assert host.deployments_called("acompletion") == ["a"] + + +# --- sleeps -------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("num_retries", [1, 2, 3]) +@pytest.mark.asyncio +async def test_lone_deployment_sleeps_once_more_than_it_retries( + make_core: MakeCore, host: ScriptedHost, num_retries: int +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=num_retries) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + bases: Final = [INITIAL_RETRY_DELAY * 2**n for n in range(num_retries + 1)] + assert len(host.sleeps) == num_retries + 1 + assert all(within(slept, base) for slept, base in zip(host.sleeps, bases)) + + +@pytest.mark.asyncio +async def test_backoff_is_capped_at_max_retry_delay(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=6) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + expected: Final = [min(INITIAL_RETRY_DELAY * 2**n, MAX_RETRY_DELAY) for n in range(7)] + assert all(within(slept, base) for slept, base in zip(host.sleeps, expected)) + assert within(host.sleeps[-1], MAX_RETRY_DELAY) + + +@pytest.mark.asyncio +async def test_retry_after_setting_above_max_retry_delay_is_capped(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=1, retry_after=20) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert all(within(slept, MAX_RETRY_DELAY) for slept in host.sleeps) + + +@pytest.mark.asyncio +async def test_a_retry_after_header_of_zero_falls_back_to_backoff(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=1) + host.answer("openai/a", lambda **_: rate_limited(retry_after="0")) + + await attempt(core) + + assert within(host.sleeps[0], INITIAL_RETRY_DELAY) + + +@pytest.mark.asyncio +async def test_retry_after_header_on_any_error_type_sets_the_sleep(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=1) + host.answer( + "openai/a", + lambda **_: litellm.InternalServerError( + message="boom", + llm_provider="openai", + model="gpt-4o", + response=http_response(500, {"retry-after": "4"}), + ), + ) + + await attempt(core) + + assert all(within(slept, 4) for slept in host.sleeps) + + +@pytest.mark.asyncio +async def test_litellm_response_headers_on_the_error_set_the_sleep(make_core: MakeCore, host: ScriptedHost) -> None: + def failure(**_: object) -> Exception: + error: Final = server_error() + error.litellm_response_headers = {"retry-after": "6"} # pyright: ignore[reportAttributeAccessIssue] # what provider handlers attach + return error + + core: Final = make_core(model_list=group("a"), num_retries=1) + host.answer("openai/a", failure) + + await attempt(core) + + assert all(within(slept, 6) for slept in host.sleeps) + + +@pytest.mark.asyncio +async def test_retry_after_header_is_ignored_while_siblings_are_healthy( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", "b"), num_retries=2, allowed_fails=10) + host.answer("*", fresh(server_error)) + + await attempt(core) + + assert host.sleeps == [0, 0, 0] + + +# --- what the retry loop writes on the request ------------------------------------------------- + + +@pytest.mark.asyncio +async def test_each_attempt_carries_its_retry_count_and_the_failures_before_it( + make_core: MakeCore, host: ScriptedHost +) -> None: + recorder: Final = MetadataRecorder() + core: Final = make_core(model_list=group("a", "b"), num_retries=2, allowed_fails=10) + host.answer("*", recorder.then(server_error), recorder.then(server_error), recorder.then(None)) + + response: Final = await attempt(core) + + assert not isinstance(response, Exception) + snapshots: Final = recorder.snapshots + assert [s["attempted_retries"] for s in snapshots] == [0, 1, 2] + assert [s["max_retries"] for s in snapshots] == [2, 2, 2] + assert [s["model_group_size"] for s in snapshots] == [2, 2, 2] + assert [s["request_retry_count"] for s in snapshots] == [None, 1, 2] + assert snapshots[0]["previous_models"] is None + trail: Final = snapshots[2]["previous_models"] + assert isinstance(trail, tuple) + assert [entry["attempted_retries"] for entry in trail] == [0, 1] + assert [entry["exception_type"] for entry in trail] == ["InternalServerError"] * 2 + assert all(entry["model_group"] == "g" for entry in trail) + called: Final = host.deployments_called("acompletion") + assert [entry["deployment_id"] for entry in trail] == called[:2] + assert [entry["model"] for entry in trail] == [f"openai/{d}" for d in called[:2]] + assert all(entry["exception_string"] == "litellm.InternalServerError: boom" for entry in trail) + + +@pytest.mark.asyncio +async def test_max_retries_on_the_request_follows_the_deployment_num_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + recorder: Final = MetadataRecorder() + core: Final = make_core(model_list=group("a", num_retries=1), num_retries=3) + host.answer("openai/a", recorder.then(server_error), recorder.then(None)) + + response: Final = await attempt(core) + + assert [s["max_retries"] for s in recorder.snapshots] == [3, 1] + assert headers_of(response)["x-litellm-max-retries"] == 1 + + +@pytest.mark.asyncio +async def test_exhausted_deployment_num_retries_are_stamped_on_the_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a", num_retries=2), num_retries=5) + host.answer("openai/a", fresh(server_error)) + + error: Final = await attempt(core) + + assert getattr(error, "num_retries", None) == 2 + assert getattr(error, "max_retries", None) == 2 + + +@pytest.mark.parametrize("bad_value", ["many", None]) +@pytest.mark.asyncio +async def test_a_deployment_num_retries_that_is_not_a_number_falls_back_to_the_router( + make_core: MakeCore, host: ScriptedHost, bad_value: object +) -> None: + core: Final = make_core(model_list=group("a", num_retries=bad_value), num_retries=1) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +# --- sync and other endpoints ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_sync_completion_retries_like_the_async_one(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + host.answer("openai/a", fresh(server_error)) + + error: Final = attempt_sync(core) + + assert isinstance(error, litellm.InternalServerError) + assert host.deployments_called("completion") == ["a", "a", "a"] + assert len(host.sleeps) == 3 + assert getattr(error, "num_retries", None) == 2 + + +@pytest.mark.asyncio +async def test_sync_completion_success_after_a_retry_reports_retry_headers( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + host.answer("openai/a", fresh(server_error), None) + + response: Final = attempt_sync(core) + + assert headers_of(response)["x-litellm-attempted-retries"] == 1 + assert headers_of(response)["x-litellm-max-retries"] == 2 + + +@pytest.mark.parametrize("op", ["aembedding", "embedding"]) +@pytest.mark.asyncio +async def test_embeddings_retry_with_the_router_num_retries(make_core: MakeCore, host: ScriptedHost, op: str) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + host.answer("openai/a", fresh(server_error)) + + async def embed() -> object: + if op == "aembedding": + return await core.aembedding(model="g", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # every core serves aembedding + return core.embedding(model="g", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # every core serves embedding + + with pytest.raises(litellm.InternalServerError): + await embed() + + assert host.deployments_called(op) == ["a", "a", "a"] + + +@pytest.mark.asyncio +async def test_embedding_success_after_a_retry_reports_retry_headers(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=group("a"), num_retries=2) + host.answer("openai/a", fresh(timeout), None) + + response: Final = await core.aembedding(model="g", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # every core serves aembedding + + assert headers_of(response)["x-litellm-attempted-retries"] == 1 + assert headers_of(response)["x-litellm-max-retries"] == 2 diff --git a/tests/unit/test_router/core/test_selection.py b/tests/unit/test_router/core/test_selection.py index 4b4c320c428..b33316ff380 100644 --- a/tests/unit/test_router/core/test_selection.py +++ b/tests/unit/test_router/core/test_selection.py @@ -26,7 +26,7 @@ from tests.unit.test_router.core.scripted_host import ( ) MESSAGES: Final = [{"role": "user", "content": "hello there, router"}] -PICKS: Final = 20 +PICKS: Final = 60 # a shuffle misses a deployment in 20 picks about once in a thousand runs MakeCore = Callable[..., RouterCore] @@ -622,3 +622,40 @@ async def test_cooling_down_one_group_leaves_other_groups_pickable(make_core: Ma with pytest.raises(RouterRateLimitError): await core.select("g") assert await _picked_ids(core, "h") <= {"c", "d"} + + +# --- custom strategies --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_registered_strategy_picks_every_deployment_an_endpoint_call_reaches( + make_core: MakeCore, host: ScriptedHost +) -> None: + from litellm.types.router import CustomRoutingStrategyBase + + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b"), deployment("g", "c")]) + seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: the request each pick was asked for + + class HighestId(CustomRoutingStrategyBase): + async def async_get_available_deployment( + self, + model: str, + messages: list[dict[str, str]] | None = None, # mutable-ok: the strategy interface's own type + input: str | list | None = None, # mutable-ok: the strategy interface's own type + specific_deployment: bool | None = False, + request_kwargs: dict | None = None, # mutable-ok: the strategy interface's own type + ) -> dict: # mutable-ok: the strategy interface's own type + seen.append(dict(request_kwargs or {})) + deployments: Final = core.get_model_list(model_name=model) # pyright: ignore[reportAttributeAccessIssue] # both cores serve the Python core's lookup + return max(deployments, key=lambda d: d["model_info"]["id"]) + + def get_available_deployment(self, model: str) -> dict: # mutable-ok: the strategy interface's own type + raise NotImplementedError + + core.register_plugin(HighestId()) + for _ in range(PICKS): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert set(host.deployments_called("acompletion")) == {"c"} + # The strategy picks from the whole group: no other core narrowed the pick before it. + assert len(seen) == PICKS + assert not any("_retry_skipped_deployment_ids" in request for request in seen) diff --git a/tests/unit/test_router/core/test_selection_characterization_groups.py b/tests/unit/test_router/core/test_selection_characterization_groups.py new file mode 100644 index 00000000000..465118e1c65 --- /dev/null +++ b/tests/unit/test_router/core/test_selection_characterization_groups.py @@ -0,0 +1,747 @@ +""" +Characterization of which deployments a router core considers for a request before a strategy +picks: routing groups and priority routing groups, tag filtering, region filtering, model access +groups, team-only models, default deployments and default fallbacks, paused (blocked) +deployments, pass-through selection, the healthy listing and the request-scoped narrowing keys. +""" + +import datetime +import re +from collections.abc import Mapping +from typing import Final + +import pydantic +import pytest + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.router_api import ChangeSet +from litellm.types.router import RouterRateLimitError +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, ids_of, picked_ids + +SPREAD_PICKS: Final = 60 + +NO_DEPLOYMENTS_MESSAGE: Final = ( + "No deployments available for selected model, Try again in 5 seconds. Passed model={model}. " + "pre-call-checks={checks}, cooldown_list=[]" +) + +NO_HEALTHY_MESSAGE: Final = ( + "litellm.BadRequestError: You passed in model={model}. There are no healthy deployments for this model" +) + +GROUP_MODELS: Final = [deployment("m1", "a"), deployment("m2", "b"), deployment("m3", "c")] + + +def _group( + name: str, models: list[str], strategy: str, **extra: object +) -> dict[str, object]: # mutable-ok: a routing_groups entry # kwargs-ok: extra RoutingGroup fields + return {"group_name": name, "models": models, "routing_strategy": strategy, **extra} + + +def _priority(name: str, priorities: dict[str, int]) -> dict[str, object]: # mutable-ok: a routing_groups entry + return _group(name, list(priorities), "priority", model_priorities=priorities) + + +# --- routing groups ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_routing_group_name_is_callable_and_unions_its_members(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=list(GROUP_MODELS), routing_groups=[_group("grp", ["m1", "m2"], "simple-shuffle")] + ) + assert await picked_ids(core, "grp", times=SPREAD_PICKS) == {"a", "b"} + assert ids_of(await core.select("grp", kind="healthy")) == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_routing_group_call_uses_the_groups_strategy(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_group("grp", ["m1", "m2"], "least-busy")]) + assert await picked_ids(core, "grp", messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_a_member_called_by_name_uses_its_groups_strategy(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("m1", "b")], + routing_groups=[_group("grp", ["m1"], "least-busy")], + ) + assert await picked_ids(core, "m1", messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_models_outside_every_group_keep_the_top_level_strategy(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("m1", "a"), deployment("free", "x"), deployment("free", "y")], + routing_groups=[_group("grp", ["m1"], "least-busy")], + ) + assert await picked_ids(core, "free", times=SPREAD_PICKS, messages=MESSAGES) == {"x", "y"} + + +@pytest.mark.asyncio +async def test_an_alias_to_a_routing_group_resolves_to_the_group(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=list(GROUP_MODELS), + model_group_alias={"al": "grp"}, + routing_groups=[_group("grp", ["m1", "m2"], "simple-shuffle")], + ) + assert await picked_ids(core, "al", times=SPREAD_PICKS) == {"a", "b"} + + +@pytest.mark.asyncio +async def test_a_group_named_like_a_model_is_shadowed_by_the_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_group("m1", ["m2"], "least-busy")]) + assert await picked_ids(core, "m1") == {"a"} + + +@pytest.mark.asyncio +async def test_a_group_whose_members_have_no_deployments_has_no_healthy_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_group("empty", ["zz"], "least-busy")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("empty") + assert str(raised.value) == NO_HEALTHY_MESSAGE.format(model="empty") + + +@pytest.mark.asyncio +async def test_a_routing_group_completion_reports_the_group_as_model_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_group("grp", ["m1"], "simple-shuffle")]) + response: Final = await core.acompletion(model="grp", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a"] + assert host.calls[-1].model_group == "grp" + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "grp" + + +def test_routing_group_names_are_listed_among_the_routable_names(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_group("grp", ["m1", "m2"], "least-busy")]) + # The group name is listed twice today. + assert sorted(core.registry().names(kind="groups")) == ["grp", "grp", "m1", "m2", "m3"] + + +@pytest.mark.asyncio +async def test_routing_groups_applied_at_runtime_become_callable(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS)) + with pytest.raises(litellm.BadRequestError): + await core.select("grp") + core.apply(ChangeSet(settings={"routing_groups": [_group("grp", ["m1", "m2"], "least-busy")]})) + assert await picked_ids(core, "grp", messages=MESSAGES) == {"a"} + + +@pytest.mark.parametrize( + ("groups", "message"), + [ + ( + [_group("x", ["m1"], "least-busy"), _group("x", ["m2"], "least-busy")], + "routing_groups: group names must be unique, duplicate group_name 'x'.", + ), + ( + [_group("default", ["m1"], "least-busy")], + "routing_groups: 'default' is reserved for the implicit fallback group.", + ), + ([_group("", ["m1"], "least-busy")], "routing_groups: group_name must be non-empty."), + ( + [_group("x", ["m1"], "least-busy"), _group("y", ["m1"], "simple-shuffle")], + "routing_groups: model_name 'm1' appears in 'x' and 'y'. Each model may belong to at most one group.", + ), + ([_group("x", ["m1"], "nope")], "Invalid routing_strategy: 'nope'."), + ([_priority("m1", {"m2": 1})], "Priority routing group names must not shadow a model or alias"), + ], + ids=["duplicate", "reserved-default", "empty-name", "overlap", "bad-strategy", "priority-shadow"], +) +def test_invalid_routing_groups_are_rejected_at_construction( + make_core: MakeCore, groups: list[dict[str, object]], message: str +) -> None: + with pytest.raises(ValueError, match=re.escape(message)): + make_core(model_list=list(GROUP_MODELS), routing_groups=groups) + + +@pytest.mark.parametrize( + ("group", "message"), + [ + (_group("p", ["m1", "m2"], "priority", model_priorities={"m1": 1}), "model_priorities must contain exactly"), + ( + _group("p", ["m1"], "priority", model_priorities={"m1": 1}, routing_strategy_args={"x": 1}), + "Priority routing groups use model_priorities, not routing_strategy_args", + ), + (_group("p", ["m1"], "least-busy", model_priorities={"m1": 1}), "model_priorities requires routing_strategy"), + (_group("p", ["m1"], "priority", model_priorities={"m1": 0}), "greater than or equal to 1"), + ( + _group("p", ["m1", "m1"], "priority", model_priorities={"m1": 1}), + "Priority routing groups require nonempty, distinct models", + ), + ], + ids=["missing-priority", "strategy-args", "priorities-without-priority", "zero-priority", "duplicate-member"], +) +def test_invalid_priority_groups_fail_validation_at_construction( + make_core: MakeCore, group: dict[str, object], message: str +) -> None: + with pytest.raises(pydantic.ValidationError, match=re.escape(message)): + make_core(model_list=list(GROUP_MODELS), routing_groups=[group]) + + +# --- priority routing groups --------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_priority_group_picks_only_its_lowest_priority_members(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[*GROUP_MODELS, deployment("m2", "b2")], + routing_groups=[_priority("p", {"m1": 2, "m2": 1, "m3": 2})], + ) + assert await picked_ids(core, "p", times=SPREAD_PICKS) == {"b", "b2"} + assert sorted(ids_of(await core.select("p", kind="healthy"))) == ["b", "b2"] + + +@pytest.mark.asyncio +async def test_a_priority_group_pick_carries_the_member_priority_as_order(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_priority("p", {"m1": 3, "m2": 1})]) + chosen: Final = await core.select("p") + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + assert chosen["model_name"] == "m2" # pyright: ignore[reportIndexIssue] + assert chosen["litellm_params"]["order"] == 1 # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_a_priority_group_member_called_directly_ignores_the_priority(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(GROUP_MODELS), routing_groups=[_priority("p", {"m1": 3, "m2": 1})]) + chosen: Final = await core.select("m1") + assert chosen["model_info"]["id"] == "a" # pyright: ignore[reportIndexIssue] + assert "order" not in chosen["litellm_params"] # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_a_priority_group_fails_over_to_the_next_priority_and_stays_there_while_the_first_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=list(GROUP_MODELS), + routing_groups=[_priority("p", {"m1": 1, "m2": 2})], + allowed_fails=0, + num_retries=0, + cooldown_time=60, + ) + host.answer("openai/a", rate_limit()) + await core.acompletion(model="p", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a", "b"] + assert await picked_ids(core, "p") == {"b"} + + +# --- tag filtering ----------------------------------------------------------------------------------- + +TAGGED: Final = [ + deployment("g", "free", tags=["free"]), + deployment("g", "paid", tags=["paid"]), + deployment("g", "dflt", tags=["default"]), + deployment("g", "untagged"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tags", "expected"), + [(["free"], {"free"}), (["paid"], {"paid"}), (["free", "paid"], {"free", "paid"})], +) +async def test_tag_filtering_picks_deployments_carrying_a_request_tag( + make_core: MakeCore, tags: list[str], expected: set[str] +) -> None: + core: Final = make_core(model_list=list(TAGGED), enable_tag_filtering=True) + assert await picked_ids(core, "g", times=SPREAD_PICKS, request_kwargs={"metadata": {"tags": tags}}) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", [{}, {"metadata": {"tags": ["unknown"]}}], ids=["untagged", "unmatched"]) +async def test_tag_filtering_sends_untagged_or_unmatched_requests_to_default_tagged_deployments( + make_core: MakeCore, request_kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=list(TAGGED), enable_tag_filtering=True) + assert await picked_ids(core, "g", request_kwargs=request_kwargs) == {"dflt"} + + +@pytest.mark.asyncio +async def test_tag_filtering_without_a_default_rejects_an_unmatched_tag(make_core: MakeCore) -> None: + core: Final = make_core(model_list=TAGGED[:2], enable_tag_filtering=True) + with pytest.raises(ValueError, match="Not allowed to access model") as raised: + await core.select("g", {"metadata": {"tags": ["unknown"]}}) + assert str(raised.value) == ( + "Not allowed to access model due to tags configuration. Passed model=g and tags=['unknown']" + ) + + +@pytest.mark.asyncio +async def test_tag_filtering_without_a_default_serves_an_untagged_request_from_every_deployment( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=TAGGED[:2], enable_tag_filtering=True) + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"free", "paid"} + + +@pytest.mark.asyncio +async def test_request_tags_are_ignored_without_tag_filtering(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TAGGED)) + picked: Final = await picked_ids(core, "g", times=SPREAD_PICKS, request_kwargs={"metadata": {"tags": ["free"]}}) + assert picked == {"free", "paid", "dflt", "untagged"} + + +@pytest.mark.asyncio +async def test_a_request_can_enable_tag_filtering_for_itself(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TAGGED)) + request: Final = {"metadata": {"tags": ["free"]}, "enable_tag_filtering": True} + assert await picked_ids(core, "g", request_kwargs=request) == {"free"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("match_any", "expected"), [(True, {"ab", "a"}), (False, {"ab"})]) +async def test_tag_filtering_match_any_decides_whether_one_shared_tag_is_enough( + make_core: MakeCore, match_any: bool, expected: set[str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "ab", tags=["a", "b"]), deployment("g", "a", tags=["a"])], + enable_tag_filtering=True, + tag_filtering_match_any=match_any, + ) + picked: Final = await picked_ids(core, "g", times=SPREAD_PICKS, request_kwargs={"metadata": {"tags": ["a", "b"]}}) + assert picked == expected + + +@pytest.mark.asyncio +async def test_tags_in_litellm_metadata_are_read_too(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TAGGED), enable_tag_filtering=True) + assert await picked_ids(core, "g", request_kwargs={"litellm_metadata": {"tags": ["paid"]}}) == {"paid"} + + +@pytest.mark.asyncio +async def test_completion_with_tags_calls_the_tagged_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=list(TAGGED), enable_tag_filtering=True) + for _ in range(5): + await core.acompletion(model="g", messages=MESSAGES, metadata={"tags": ["paid"]}) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["paid"] * 5 + + +# --- region filtering -------------------------------------------------------------------------------- + +REGIONAL: Final = [ + deployment("g", "eu", region_name="eu"), + deployment("g", "us", region_name="us"), + deployment("g", "unknown"), +] + + +@pytest.mark.asyncio +async def test_allowed_model_region_keeps_only_deployments_in_that_region(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(REGIONAL), enable_pre_call_checks=True) + assert await picked_ids(core, "g", request_kwargs={"allowed_model_region": "eu"}, messages=MESSAGES) == {"eu"} + + +@pytest.mark.asyncio +async def test_allowed_model_region_is_ignored_without_request_input(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(REGIONAL), enable_pre_call_checks=True) + picked: Final = await picked_ids(core, "g", times=SPREAD_PICKS, request_kwargs={"allowed_model_region": "eu"}) + assert picked == {"eu", "us", "unknown"} + + +@pytest.mark.asyncio +async def test_allowed_model_region_is_ignored_without_pre_call_checks(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(REGIONAL)) + picked: Final = await picked_ids( + core, "g", times=SPREAD_PICKS, request_kwargs={"allowed_model_region": "eu"}, messages=MESSAGES + ) + assert picked == {"eu", "us", "unknown"} + + +@pytest.mark.asyncio +async def test_an_allowed_model_region_no_deployment_is_in_rate_limits(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(REGIONAL), enable_pre_call_checks=True) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", {"allowed_model_region": "ap"}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g", checks=True) + + +# --- model access groups ------------------------------------------------------------------------------- + +ACCESS_GROUPED: Final = [ + deployment("g", "in_a", model_info={"access_groups": ["grpA"]}), + deployment("g", "in_b", model_info={"access_groups": ["grpB"]}), +] + + +def _auth( + bucket: str = "metadata", **fields: object +) -> dict[str, object]: # mutable-ok: request kwargs # kwargs-ok: UserAPIKeyAuth fields + return {bucket: {"user_api_key_auth": UserAPIKeyAuth(**fields)}} # pyright: ignore[reportArgumentType] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "expected"), + [ + (_auth(models=["grpA"]), {"in_a"}), + (_auth(models=["grpB"]), {"in_b"}), + (_auth(team_models=["grpB"]), {"in_b"}), + (_auth("litellm_metadata", models=["grpA"]), {"in_a"}), + ], + ids=["key-group-a", "key-group-b", "team-group-b", "litellm-metadata"], +) +async def test_an_access_group_only_caller_is_narrowed_to_its_groups_deployments( + make_core: MakeCore, request_kwargs: dict[str, object], expected: set[str] +) -> None: + core: Final = make_core(model_list=list(ACCESS_GROUPED)) + assert await picked_ids(core, "g", request_kwargs=request_kwargs) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "models", [["g"], ["*"], ["all-proxy-models"], [], ["unrelated"]], ids=["direct", "star", "all", "none", "other"] +) +async def test_a_caller_not_limited_to_access_groups_can_reach_every_deployment( + make_core: MakeCore, models: list[str] +) -> None: + core: Final = make_core(model_list=list(ACCESS_GROUPED)) + picked: Final = await picked_ids(core, "g", times=SPREAD_PICKS, request_kwargs=_auth(models=models)) + assert picked == {"in_a", "in_b"} + + +# --- team-only models ------------------------------------------------------------------------------------ + +TEAM_ONLY: Final = [ + deployment("internal-x", "tx", model_info={"team_id": "team-x", "team_public_model_name": "pub"}), + deployment("internal-y", "ty", model_info={"team_id": "team-y", "team_public_model_name": "pub"}), +] + + +@pytest.mark.asyncio +async def test_a_team_id_in_litellm_metadata_selects_the_team_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAM_ONLY)) + assert await picked_ids(core, "pub", request_kwargs={"litellm_metadata": {"user_api_key_team_id": "team-y"}}) == { + "ty" + } + + +@pytest.mark.asyncio +async def test_a_proxy_admin_without_a_team_is_refused_a_public_name_several_teams_share(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TEAM_ONLY)) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("pub", _auth(user_role="proxy_admin")) + assert str(raised.value) == ( + "litellm.BadRequestError: Model name 'pub' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ) + + +@pytest.mark.asyncio +async def test_a_proxy_admin_without_a_team_finds_one_teams_public_name_but_cannot_use_it(make_core: MakeCore) -> None: + core: Final = make_core(model_list=TEAM_ONLY[:1]) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("pub", _auth(user_role="proxy_admin")) + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="pub", checks=False) + + +# --- default deployment, default fallbacks ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_an_unknown_model_goes_to_a_catch_all_deployment_under_its_own_name(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("*", "catch_all", model="openai/*")]) + chosen: Final = await core.select("brand-new") + assert chosen["model_info"]["id"] == "catch_all" # pyright: ignore[reportIndexIssue] + assert chosen["litellm_params"]["model"] == "openai/brand-new" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_an_unknown_model_goes_to_the_first_default_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("h", "b")], default_fallbacks=["h", "g"]) + chosen: Final = await core.select("unknown") + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + await core.acompletion(model="unknown", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["b"] + + +@pytest.mark.asyncio +async def test_an_unknown_model_goes_to_the_process_default_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(default_fallbacks=["g"]) + core: Final = make_core(model_list=[deployment("g", "a")]) + chosen: Final = await core.select("unknown") + assert chosen["model_info"]["id"] == "a" # pyright: ignore[reportIndexIssue] + + +# --- paused (blocked) deployments ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_a_blocked_deployment_is_never_picked_for_its_group(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model_info={"blocked": True}), deployment("g", "b")]) + assert await picked_ids(core, "g") == {"b"} + assert ids_of(await core.select("g", kind="healthy")) == ["b"] + + +@pytest.mark.asyncio +async def test_selecting_a_blocked_deployment_by_id_raises_service_unavailable(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model_info={"blocked": True}), deployment("g", "b")]) + with pytest.raises(litellm.ServiceUnavailableError) as raised: + await core.select("a") + # The message names the deployment's litellm model, not the id that was asked for. + assert str(raised.value) == ( + "litellm.ServiceUnavailableError: Model 'openai/a' is currently paused and cannot accept requests." + ) + assert raised.value.status_code == 503 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["select", "acompletion", "completion"]) +async def test_a_group_whose_every_deployment_is_blocked_rate_limits( + make_core: MakeCore, host: ScriptedHost, entry: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", model_info={"blocked": True})]) + call: Final = core.select("g") if entry == "select" else core.call(entry, {"model": "g", "messages": MESSAGES}) + with pytest.raises(RouterRateLimitError) as raised: + await call + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g", checks=False) + assert host.calls == [] + + +# --- pass-through selection ------------------------------------------------------------------------------------ + +PASS_THROUGH: Final = [deployment("g", "pt", use_in_pass_through=True), deployment("g", "plain")] + + +@pytest.mark.asyncio +async def test_pass_through_selection_picks_only_pass_through_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH)) + assert await picked_ids(core, "g", kind="pass_through") == {"pt"} + + +@pytest.mark.asyncio +async def test_pass_through_selection_by_id_returns_a_pass_through_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH)) + chosen: Final = await core.select("pt", kind="pass_through") + assert chosen["model_info"]["id"] == "pt" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_pass_through_selection_by_id_refuses_a_plain_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH)) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("plain", kind="pass_through") + assert str(raised.value) == ( + "litellm.BadRequestError: Deployment plain does not support pass-through endpoint (use_in_pass_through=False)" + ) + + +@pytest.mark.asyncio +async def test_pass_through_selection_without_pass_through_deployments_raises_bad_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("g", kind="pass_through") + assert str(raised.value) == ( + "litellm.BadRequestError: Model g has no deployment configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true in the deployment configuration" + ) + + +@pytest.mark.asyncio +async def test_async_pass_through_selection_words_the_missing_deployment_error_differently( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.async_get_available_deployment_for_pass_through("g", {}) # pyright: ignore[reportAttributeAccessIssue] + assert str(raised.value) == ( + "litellm.BadRequestError: Model g has no deployments configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true to the deployment configuration" + ) + + +@pytest.mark.asyncio +async def test_pass_through_selection_of_an_unknown_model_has_no_healthy_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH)) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("nope", kind="pass_through") + assert str(raised.value) == NO_HEALTHY_MESSAGE.format(model="nope") + + +@pytest.mark.asyncio +async def test_pass_through_selection_skips_a_cooling_pass_through_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "pt1", use_in_pass_through=True), + deployment("g", "pt2", use_in_pass_through=True), + ], + allowed_fails=0, + num_retries=0, + cooldown_time=60, + ) + host.answer("openai/pt1", rate_limit()) + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="pt1", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert await picked_ids(core, "g", kind="pass_through") == {"pt2"} + + +def test_deprecated_pass_through_pick_matches_select(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(PASS_THROUGH)) + assert core.get_available_deployment_for_pass_through("g")["model_info"]["id"] == "pt" # pyright: ignore[reportAttributeAccessIssue] + + +# --- the healthy listing -------------------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_healthy_listing_keeps_only_the_lowest_order(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", order=1), deployment("g", "b", order=2)]) + assert ids_of(await core.select("g", kind="healthy")) == ["a"] + + +@pytest.mark.asyncio +async def test_the_healthy_listing_for_a_deployment_id_is_that_single_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + healthy: Final = await core.select("b", kind="healthy") + assert isinstance(healthy, Mapping) + assert healthy["model_info"]["id"] == "b" + + +@pytest.mark.asyncio +async def test_the_healthy_listing_of_an_unknown_model_raises_bad_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("nope", kind="healthy") + assert str(raised.value) == NO_HEALTHY_MESSAGE.format(model="nope") + + +@pytest.mark.asyncio +async def test_the_healthy_listing_leaves_out_cooling_deployments(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", rate_limit()) + with pytest.raises(litellm.RateLimitError): + await core.acompletion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert ids_of(await core.select("g", kind="healthy")) == ["b"] + + +# --- request-scoped narrowing keys ----------------------------------------------------------------------------------- + +THREE: Final = [deployment("g", "a"), deployment("g", "b"), deployment("g", "c")] + + +@pytest.mark.asyncio +async def test_target_order_picks_that_order_instead_of_the_lowest(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", order=1), deployment("g", "b", order=2)]) + assert await picked_ids(core, "g", request_kwargs={"_target_order": 2}) == {"b"} + assert ids_of(await core.select("g", {"_target_order": 2}, kind="healthy")) == ["b"] + + +@pytest.mark.asyncio +async def test_a_target_order_no_deployment_has_rate_limits(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", order=1)]) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", {"_target_order": 3}) + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g", checks=False) + + +@pytest.mark.asyncio +async def test_excluded_deployment_ids_are_never_picked(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + assert await picked_ids(core, "g", request_kwargs={"_excluded_deployment_ids": ["a", "b"]}) == {"c"} + + +@pytest.mark.asyncio +async def test_excluding_every_deployment_rate_limits(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", {"_excluded_deployment_ids": ["a", "b", "c"]}) + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g", checks=False) + + +@pytest.mark.asyncio +async def test_retry_skipped_deployment_ids_are_not_picked_while_others_remain(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + assert await picked_ids(core, "g", request_kwargs={"_retry_skipped_deployment_ids": ("a", "b")}) == {"c"} + + +@pytest.mark.asyncio +async def test_skipping_every_deployment_on_retry_keeps_them_all(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + healthy: Final = await core.select("g", {"_retry_skipped_deployment_ids": ("a", "b", "c")}, kind="healthy") + assert ids_of(healthy) == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_retry_skipped_deployment_ids_given_as_a_list_are_ignored(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + healthy: Final = await core.select("g", {"_retry_skipped_deployment_ids": ["a", "b"]}, kind="healthy") + assert ids_of(healthy) == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + request: Final[dict[str, object]] = { + "_target_order": None, + "_excluded_deployment_ids": ["a"], + "_retry_skipped_deployment_ids": ("b",), + } + assert ids_of(await core.select("g", request, kind="healthy")) == ["c"] + assert request == {} + + +# --- access windows (reserved deployments) ---------------------------------------------------------------------------- + + +def _window_for(team_ids: list[str]) -> dict[str, object]: # mutable-ok: an access_windows entry + now: Final = datetime.datetime.now(datetime.timezone.utc) + return { + "start": (now - datetime.timedelta(hours=1)).strftime("%H:%M"), + "end": (now + datetime.timedelta(hours=1)).strftime("%H:%M"), + "timezone": "UTC", + "team_ids": team_ids, + } + + +def _reserved(window: dict[str, object]) -> dict[str, object]: # mutable-ok: a model_list entry + return deployment("g", "reserved", model_info={"access_windows": [window]}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", [{"metadata": {"user_api_key_team_id": "team-b"}}, {}]) +async def test_a_deployment_reserved_for_another_team_is_hidden_during_its_window( + make_core: MakeCore, request_kwargs: dict[str, object] +) -> None: + core: Final = make_core(model_list=[_reserved(_window_for(["team-a"])), deployment("g", "open")]) + assert await picked_ids(core, "g", request_kwargs=request_kwargs) == {"open"} + + +@pytest.mark.asyncio +async def test_the_reserving_team_reaches_its_reserved_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[_reserved(_window_for(["team-a"]))]) + assert await picked_ids(core, "g", request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}) == {"reserved"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "named"), [("g", "g"), ("reserved", "openai/reserved")], ids=["group", "id"]) +async def test_only_reserved_deployments_left_raises_bad_request_naming_the_window_end( + make_core: MakeCore, model: str, named: str +) -> None: + window: Final = _window_for(["team-a"]) + core: Final = make_core(model_list=[_reserved(window)]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select(model, {"metadata": {"user_api_key_team_id": "team-b"}}) + assert str(raised.value) == ( + f"litellm.BadRequestError: Deployment {named} is reserved for another team until {window['end']} UTC" + ) + + +# --- specific_deployment ------------------------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["openai/unknown", "g"], ids=["unknown-litellm-model", "group-name"]) +async def test_specific_deployment_for_a_name_no_litellm_model_matches_rate_limits( + make_core: MakeCore, model: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(RouterRateLimitError) as raised: + await core.select(model, specific_deployment=True) # pyright: ignore[reportCallIssue] + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model=model, checks=False) diff --git a/tests/unit/test_router/core/test_selection_characterization_prerouting.py b/tests/unit/test_router/core/test_selection_characterization_prerouting.py new file mode 100644 index 00000000000..317fbf4be46 --- /dev/null +++ b/tests/unit/test_router/core/test_selection_characterization_prerouting.py @@ -0,0 +1,505 @@ +""" +Characterization of what runs before a router core's strategy picks: pre-routing strategies (the +complexity router, chosen among tag-scoped markers), the strategy marker deployments they are +registered from, tier ``litellm_params`` pinned onto the request, alias-marker params forwarded to +the routed call, the Claude Code session router binding, and the ``Router(plugins=[...])`` +routing-plugin pipeline. +""" + +import asyncio +import re +from collections.abc import Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.router import RoutingContext +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, picked_ids + +HARD_MESSAGES: Final = [ + { + "role": "user", + "content": ( + "Design a distributed consensus algorithm, prove its correctness step by step, analyze the time " + "complexity, and implement it in Rust with a kubernetes deployment and database sharding strategy. " + "Think step by step and reason about edge cases." + ), + } +] + +TIERS: Final = {"SIMPLE": "small", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"} +ALL_BIG: Final = {"SIMPLE": "big", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"} + +ONLY_MARKERS_MESSAGE: Final = ( + "litellm.BadRequestError: You passed in model={model}. Every deployment for it is a strategy router marker " + "(auto_router/...), which is not a callable model, and no pre-routing strategy selected a deployment for " + "this request" +) + +SESSION_ID: Final = "0f8fad5b-d9cb-469f-a165-70867728950e" + + +def _complexity_router( + name: str = "smart", + tiers: Mapping[str, object] = TIERS, + deployment_id: str = "marker", + **params: object, # kwargs-ok: extra marker litellm_params +) -> dict[str, object]: # mutable-ok: a model_list entry + return { + "model_name": name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": dict(tiers)}, + "complexity_router_default_model": "big", + **params, + }, + "model_info": {"id": deployment_id}, + } + + +def _tiered( + **marker_params: object, +) -> list[dict[str, object]]: # mutable-ok: a model_list # kwargs-ok: marker litellm_params + return [deployment("small", "s"), deployment("big", "b"), _complexity_router(**marker_params)] + + +def _sync_entry(core: RouterCore, entry: str, model: str) -> object: + if entry == "completion": + return core.completion(model=model, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + return core.get_available_deployment(model, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + + +def _metadata(request: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = request.get("metadata") + assert isinstance(metadata, Mapping) + return metadata + + +# --- the complexity router ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("messages", "expected"), [(MESSAGES, "s"), (HARD_MESSAGES, "b")], ids=["simple", "hard"]) +async def test_a_complexity_router_picks_the_tier_deployment_for_the_prompt( + make_core: MakeCore, messages: list[dict[str, str]], expected: str +) -> None: + core: Final = make_core(model_list=_tiered()) + chosen: Final = await core.select("smart", {}, messages=messages) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == expected # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + request: Final[dict[str, object]] = {} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + metadata: Final = _metadata(request) + decision: Final = metadata["routing_decision"] + assert isinstance(decision, Mapping) + assert decision["router_model_name"] == "smart" + assert decision["router_type"] == "complexity" + assert decision["router_config_id"] == "marker" + assert decision["routed_model"] == "small" + assert decision["tier"] == "SIMPLE" + assert metadata["pre_routing_selected_model"] == "small" + + +@pytest.mark.asyncio +async def test_completion_through_a_complexity_router_reports_the_router_name_as_model_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_tiered()) + await core.acompletion(model="smart", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["s"] + assert host.calls[-1].model == "openai/s" + assert host.calls[-1].model_group == "smart" + + +@pytest.mark.asyncio +async def test_an_alias_to_a_complexity_router_routes_through_it(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"}) + request: Final[dict[str, object]] = {} + chosen: Final = await core.select("al", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] + assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue] + await core.acompletion(model="al", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[-1].model_group == "al" + + +@pytest.mark.asyncio +async def test_a_complexity_router_takes_precedence_over_plain_deployments_of_its_name(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*_tiered(), deployment("smart", "plain")]) + assert await picked_ids(core, "smart", times=10, messages=MESSAGES) == {"s"} + + +@pytest.mark.asyncio +async def test_a_complexity_router_tier_naming_an_unknown_model_falls_back_to_the_default_model( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("big", "b"), _complexity_router(tiers={**TIERS, "SIMPLE": "missing"})] + ) + request: Final[dict[str, object]] = {} + chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + decision: Final = _metadata(request)["routing_decision"] + assert isinstance(decision, Mapping) + assert decision["cause"] == "health_default_fallback" + assert decision["routed_model"] == "big" + + +def test_a_complexity_router_without_tiers_or_a_default_model_is_rejected(make_core: MakeCore) -> None: + with pytest.raises(ValueError, match="complexity_router_default_model is required for complexity-router"): + make_core(model_list=[deployment("smart", "marker", model="auto_router/complexity_router")]) + + +# --- strategy markers are never callable ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_healthy_listing_of_a_router_name_refuses_the_marker(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("smart", {}, kind="healthy") + assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart") + + +@pytest.mark.asyncio +async def test_pass_through_selection_of_a_router_name_refuses_the_marker(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("smart", {}, kind="pass_through", messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart") + + +@pytest.mark.asyncio +async def test_selecting_a_marker_by_its_id_refuses_it_naming_its_litellm_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("marker", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="auto_router/complexity_router") + + +@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"]) +def test_the_sync_path_never_runs_pre_routing_and_refuses_the_marker( + make_core: MakeCore, host: ScriptedHost, entry: str +) -> None: + core: Final = make_core(model_list=_tiered()) + with pytest.raises(litellm.BadRequestError) as raised: + _sync_entry(core, entry, "smart") + assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart") + assert host.calls == [] + + +# --- choosing among tag-scoped markers ----------------------------------------------------------------- + +TAG_SCOPED: Final = [ + deployment("small", "s"), + deployment("big", "b"), + _complexity_router(deployment_id="cheap_marker", tags=["cheap"]), + _complexity_router(tiers=ALL_BIG, deployment_id="premium_marker", tags=["premium"]), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "expected"), + [({"metadata": {"tags": ["cheap"]}}, "s"), ({"metadata": {"tags": ["premium"]}}, "b"), ({}, "s")], + ids=["cheap-tag", "premium-tag", "untagged-takes-the-first"], +) +async def test_request_tags_pick_among_markers_of_one_name( + make_core: MakeCore, request_kwargs: dict[str, object], expected: str +) -> None: + core: Final = make_core(model_list=list(TAG_SCOPED)) + chosen: Final = await core.select("smart", request_kwargs, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == expected # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_an_untagged_request_takes_the_default_tagged_marker(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("small", "s"), + deployment("big", "b"), + _complexity_router(deployment_id="cheap_marker", tags=["cheap"]), + _complexity_router(tiers=ALL_BIG, deployment_id="default_marker", tags=["default"]), + ] + ) + chosen: Final = await core.select("smart", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + + +SCOPED_WITH_PLAIN: Final = [ + deployment("small", "s"), + deployment("big", "b"), + _complexity_router(deployment_id="cheap_marker", tags=["cheap"]), + deployment("smart", "plain"), +] + + +@pytest.mark.asyncio +async def test_with_tag_filtering_an_unmatched_tag_skips_the_markers_for_plain_tag_selection( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=list(SCOPED_WITH_PLAIN), enable_tag_filtering=True) + with pytest.raises( + ValueError, match=re.escape("Not allowed to access model due to tags configuration. Passed model=smart") + ): + await core.select("smart", {"metadata": {"tags": ["other"]}}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core.select("smart", {"metadata": {"tags": ["cheap"]}}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_without_tag_filtering_an_unmatched_tag_still_takes_the_first_marker(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(SCOPED_WITH_PLAIN)) + chosen: Final = await core.select("smart", {"metadata": {"tags": ["other"]}}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] + + +# --- tier litellm_params and alias-marker params ------------------------------------------------------------- + +PINNED_TIERS: Final = { + **TIERS, + "SIMPLE": {"model_name": "small", "litellm_params": {"temperature": 0.1, "max_tokens": 7}}, +} + + +@pytest.mark.asyncio +async def test_tier_litellm_params_override_the_callers_params_on_the_call( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS)) + await core.acompletion(model="smart", messages=MESSAGES, temperature=0.9, max_completion_tokens=100) # pyright: ignore[reportAttributeAccessIssue] + sent: Final = host.calls[-1].kwargs + assert host.deployments_called() == ["s"] + assert sent["temperature"] == 0.1 + assert sent["max_tokens"] == 7 + assert "max_completion_tokens" not in sent + + +@pytest.mark.asyncio +async def test_tier_litellm_params_are_pinned_onto_the_request_by_select(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS)) + request: Final[dict[str, object]] = {"temperature": 0.9, "max_completion_tokens": 100} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["temperature"] == 0.1 + assert request["max_tokens"] == 7 + assert "max_completion_tokens" not in request + + +@pytest.mark.asyncio +async def test_tier_litellm_params_apply_only_to_their_tier(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS)) + await core.acompletion(model="smart", messages=HARD_MESSAGES, temperature=0.9) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["b"] + assert host.calls[-1].kwargs["temperature"] == 0.9 + assert "max_tokens" not in host.calls[-1].kwargs + + +@pytest.mark.asyncio +async def test_marker_params_fill_gaps_on_the_routed_call_but_never_override_the_caller( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_tiered(temperature=0.3)) + await core.acompletion(model="smart", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[-1].kwargs["temperature"] == 0.3 + await core.acompletion(model="smart", messages=MESSAGES, temperature=0.9) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[-1].kwargs["temperature"] == 0.9 + + +@pytest.mark.asyncio +async def test_marker_credentials_and_pricing_are_never_forwarded(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_tiered(api_base="http://marker.example", input_cost_per_token=0.5)) + await core.acompletion(model="smart", messages=HARD_MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + sent: Final = host.calls[-1].kwargs + assert host.deployments_called() == ["b"] + assert sent.get("api_base") is None + assert "input_cost_per_token" not in sent + + +@pytest.mark.asyncio +async def test_a_deployments_own_param_beats_the_forwarded_marker_param( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[ + deployment("small", "s", aws_region_name="deployment-region"), + deployment("big", "b"), + _complexity_router(aws_region_name="marker-region"), + ] + ) + await core.acompletion(model="smart", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[-1].kwargs["aws_region_name"] == "deployment-region" + await core.acompletion(model="smart", messages=HARD_MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls[-1].kwargs["aws_region_name"] == "marker-region" + + +# --- the Claude Code session router binding -------------------------------------------------------------------- + + +def _session_request( + agent_id: str | None = None, caller: str = "caller-hash" +) -> dict[str, object]: # mutable-ok: request kwargs + headers: Final[dict[str, str]] = {"x-claude-code-session-id": SESSION_ID} + if agent_id is not None: + headers["x-claude-code-agent-id"] = agent_id + return {"proxy_server_request": {"headers": headers}, "metadata": {"user_api_key_hash": caller}} + + +@pytest.mark.asyncio +async def test_a_subagent_request_follows_the_router_its_session_started_on(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue] + subagent: Final = _session_request(agent_id="agent-1") + chosen: Final = await core.select("big", subagent, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] + assert _metadata(subagent)["model_group"] == "smart" + + +@pytest.mark.asyncio +async def test_a_session_binding_is_scoped_to_the_caller(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue] + other_caller: Final = _session_request(agent_id="agent-1", caller="other-hash") + chosen: Final = await core.select("big", other_caller, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_a_main_agent_request_is_not_redirected_by_its_session_binding(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core.select("big", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_a_subagent_request_without_a_session_binding_keeps_its_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_tiered()) + chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("session_id", ["short", "has spaces in it"]) +async def test_an_invalid_session_id_never_binds(make_core: MakeCore, session_id: str) -> None: + core: Final = make_core(model_list=_tiered()) + first: Final = _session_request() + first["proxy_server_request"] = {"headers": {"x-claude-code-session-id": session_id}} + await core.select("smart", first, messages=MESSAGES) # pyright: ignore[reportCallIssue] + subagent: Final = _session_request(agent_id="agent-1") + subagent["proxy_server_request"] = { + "headers": {"x-claude-code-session-id": session_id, "x-claude-code-agent-id": "agent-1"} + } + chosen: Final = await core.select("big", subagent, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] + + +# --- the routing-plugin pipeline ------------------------------------------------------------------------------ + + +class Narrowing: + """Keeps only ``keep`` among the candidate models and records what it was handed.""" + + def __init__(self, keep: frozenset[str]) -> None: + self.keep: Final = keep + self.seen: list[list[str]] = [] # mutable-ok: the candidate lists each run saw + + async def run(self, context: RoutingContext) -> RoutingContext: + self.seen.append(list(context.candidate_models)) + context.candidate_models = [model for model in context.candidate_models if model in self.keep] + context.signals["narrowing"] = {"kept": sorted(self.keep)} + return context + + +TWO: Final = [deployment("g", "a"), deployment("g", "b")] + + +@pytest.mark.asyncio +async def test_a_routing_plugin_narrows_the_candidates_to_the_models_it_keeps(make_core: MakeCore) -> None: + plugin: Final = Narrowing(frozenset({"openai/b"})) + core: Final = make_core(model_list=list(TWO), plugins=[plugin]) + assert await picked_ids(core, "g", messages=MESSAGES) == {"b"} + assert plugin.seen[0] == ["openai/a", "openai/b"] + + +@pytest.mark.asyncio +async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))]) + request: Final[dict[str, object]] = {} + await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request == { + "metadata": { + "routing_plugin_signals": {"narrowing": {"kept": ["openai/b"]}}, + "_routing_plugin_candidate_models": ["openai/b"], + } + } + + +@pytest.mark.asyncio +async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a", "openai/b"}))]) + request: Final[dict[str, object]] = {} + await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request == {"metadata": {"routing_plugin_signals": {"narrowing": {"kept": ["openai/a", "openai/b"]}}}} + + +@pytest.mark.asyncio +async def test_routing_plugins_run_in_order_each_seeing_the_previous_narrowing(make_core: MakeCore) -> None: + first: Final = Narrowing(frozenset({"openai/b", "openai/c"})) + second: Final = Narrowing(frozenset({"openai/c"})) + core: Final = make_core(model_list=[*TWO, deployment("g", "c")], plugins=[first, second]) + assert await picked_ids(core, "g", times=1, messages=MESSAGES) == {"c"} + assert second.seen == [["openai/b", "openai/c"]] + + +@pytest.mark.asyncio +async def test_a_routing_plugin_that_narrows_to_nothing_fails_the_pick(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset())]) + with pytest.raises(ValueError, match="No deployments left") as raised: + await core.select("g", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value) == "No deployments left after routing-plugin filtering. candidate_models=[]" + + +SYNC_PLUGIN_MESSAGE_START: Final = ( + "Router(plugins=[...]) is configured, but this call resolved to the synchronous deployment-selection path" +) + + +@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"]) +def test_routing_plugins_refuse_the_sync_selection_path(make_core: MakeCore, host: ScriptedHost, entry: str) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a"}))]) + with pytest.raises(ValueError, match=re.escape(SYNC_PLUGIN_MESSAGE_START)) as raised: + _sync_entry(core, entry, "g") + assert str(raised.value).startswith(SYNC_PLUGIN_MESSAGE_START) + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_routing_plugins_refuse_usage_based_routing_v1_even_from_async_callers(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a"}))], routing_strategy="usage-based-routing" + ) + with pytest.raises(ValueError, match=re.escape(SYNC_PLUGIN_MESSAGE_START)) as raised: + await core.select("g", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value).startswith(SYNC_PLUGIN_MESSAGE_START) + + +@pytest.mark.asyncio +async def test_a_routing_plugin_does_not_narrow_a_pick_by_deployment_id(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))]) + chosen: Final = await core.select("a", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert chosen["model_info"]["id"] == "a" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_completion_with_a_routing_plugin_calls_a_kept_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a"}))]) + await asyncio.gather(*(core.acompletion(model="g", messages=MESSAGES) for _ in range(5))) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called() == ["a"] * 5 diff --git a/tests/unit/test_router/core/test_selection_characterization_session_repin.py b/tests/unit/test_router/core/test_selection_characterization_session_repin.py new file mode 100644 index 00000000000..f022e267586 --- /dev/null +++ b/tests/unit/test_router/core/test_selection_characterization_session_repin.py @@ -0,0 +1,119 @@ +""" +Characterization of ``session_affinity_repin_on_failover``: a session pinned to a deployment that +fails is served by another deployment of the group, and a completed success there moves the +session's pin to it when the flag is on. With it off, the pin stays where it was. +""" + +import asyncio +from collections.abc import Callable + +import pytest + +from litellm.router_core import RouterCore +from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck +from tests.unit.test_router.core.scripted_host import ScriptedHost, completion_response, deployment, rate_limit +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGES, + chat_stream, + collect_texts, + install_orjson_stub_if_missing, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] + +_SESSION = "session-1" +_CALLER = "caller-1" +_METADATA = {"session_id": _SESSION, "user_api_key_hash": _CALLER} + + +def _affinity_check(core: RouterCore) -> DeploymentAffinityCheck: + return next( + callback + for callback in core.optional_callbacks or () # pyright: ignore[reportAttributeAccessIssue, reportUnknownVariableType, reportUnknownMemberType] + if isinstance(callback, DeploymentAffinityCheck) + ) + + +async def _pinned(check: DeploymentAffinityCheck) -> object: + key = DeploymentAffinityCheck.get_session_affinity_cache_key("g", _SESSION, _CALLER) + pinned: object = await check.cache.async_get_cache(key) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return pinned # pyright: ignore[reportUnknownVariableType] # see above + + +async def _complete(core: RouterCore) -> object: + response: object = await core.acompletion( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + model="g", messages=[{"role": "user", "content": "hi"}], metadata=dict(_METADATA) + ) + return response # pyright: ignore[reportUnknownVariableType] # see above + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("repin", "next_served_by"), [(True, "b"), (False, "a")]) +async def test_a_failover_success_repins_the_session_only_with_repin_on_failover( + make_core: MakeCore, host: ScriptedHost, repin: bool, next_served_by: str +) -> None: + core = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + optional_pre_call_checks=["session_affinity"], + session_affinity_repin_on_failover=repin, + num_retries=1, + ) + check = _affinity_check(core) + key = DeploymentAffinityCheck.get_session_affinity_cache_key("g", _SESSION, _CALLER) + await check.cache.async_set_cache(key, {"model_id": "a"}) # pyright: ignore[reportUnknownMemberType] + host.answer("openai/a", rate_limit(), completion_response()) + host.answer("openai/b", completion_response()) + + await _complete(core) + + # The pinned deployment fails and cools down; the group's other deployment serves the retry. + assert host.deployments_called("acompletion") == ["a", "b"] + assert await _pinned(check) == {"model_id": "b" if repin else "a"} + + # Once the first deployment is healthy again, the session goes where it is pinned. + core.health().cooldowns.cooldown_store.flush_cache() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + await _complete(core) + + assert host.deployments_called("acompletion")[-1] == next_served_by + + +async def _settle() -> None: + """Let the logging and recording tasks a finished stream scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("repin", [True, False]) +async def test_a_failover_stream_repins_the_session_when_it_ends( + make_core: MakeCore, host: ScriptedHost, repin: bool +) -> None: + core = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + optional_pre_call_checks=["session_affinity"], + session_affinity_repin_on_failover=repin, + num_retries=1, + ) + check = _affinity_check(core) + key = DeploymentAffinityCheck.get_session_affinity_cache_key("g", _SESSION, _CALLER) + await check.cache.async_set_cache(key, {"model_id": "a"}) # pyright: ignore[reportUnknownMemberType] + host.answer("openai/a", rate_limit()) + host.answer( + "openai/b", + chat_stream("hello", usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, logged=True), + ) + + stream: object = await core.acompletion( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + model="g", messages=MESSAGES, stream=True, metadata=dict(_METADATA) + ) + await _settle() + # The pin moves only once the stream has been read to its end. + assert await _pinned(check) == {"model_id": "a"} + + assert await collect_texts(stream) == "hello" # pyright: ignore[reportUnknownArgumentType] # see above + await _settle() + + assert host.deployments_called("acompletion") == ["a", "b"] + assert await _pinned(check) == {"model_id": "b" if repin else "a"} diff --git a/tests/unit/test_router/core/test_selection_characterization_strategies.py b/tests/unit/test_router/core/test_selection_characterization_strategies.py new file mode 100644 index 00000000000..3999c24857f --- /dev/null +++ b/tests/unit/test_router/core/test_selection_characterization_strategies.py @@ -0,0 +1,394 @@ +""" +Characterization of how a router core's routing strategies pick a deployment: strategy validation, +simple-shuffle weighting, the stateful strategies without traffic, per-request strategy overrides, +runtime strategy changes and custom strategies registered with ``register_plugin``. +""" + +import re +from typing import Final + +import pydantic +import pytest + +import litellm +from litellm.router_api import ChangeSet +from litellm.types.router import CustomRoutingStrategyBase, RouterRateLimitError, RoutingStrategy +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, picked_ids + +SPREAD_PICKS: Final = 60 +"""Enough uniform picks between two deployments that missing one is a 2**-59 event.""" + +INVALID_STRATEGY_MESSAGE: Final = ( + "Invalid routing_strategy: '{name}'. Valid options: ['simple-shuffle', 'lar1', 'least-busy', " + "'latency-based-routing', 'cost-based-routing', 'usage-based-routing-v2', 'usage-based-routing', " + "'provider-budget-routing']. Check 'router_settings.routing_strategy' in your config.yaml or the " + "'routing_strategy' parameter if using the Router SDK directly." +) + +NO_DEPLOYMENTS_MESSAGE: Final = ( + "No deployments available for selected model, Try again in 5 seconds. Passed model=g. " + "pre-call-checks=False, cooldown_list=[]" +) + + +def _two(**params: object) -> list[dict[str, object]]: # mutable-ok: a model_list # kwargs-ok: shared litellm_params + return [deployment("g", "a", **params), deployment("g", "b", **params)] + + +# --- strategy validation --------------------------------------------------------------------- + + +@pytest.mark.parametrize("routing_strategy", ["bogus", "priority", "Simple-Shuffle"]) +def test_an_unknown_routing_strategy_is_rejected_at_construction(make_core: MakeCore, routing_strategy: str) -> None: + with pytest.raises(ValueError, match="Invalid routing_strategy") as raised: + make_core(model_list=_two(), routing_strategy=routing_strategy) + assert str(raised.value) == INVALID_STRATEGY_MESSAGE.format(name=routing_strategy) + + +@pytest.mark.asyncio +async def test_lar1_is_an_accepted_strategy_that_picks_within_the_group(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*_two(), deployment("other", "c")], routing_strategy="lar1") + assert await picked_ids(core, "g", messages=MESSAGES) <= {"a", "b"} + + +@pytest.mark.asyncio +async def test_provider_budget_routing_without_budgets_has_no_selector_and_rate_limits_every_pick( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_two(), routing_strategy="provider-budget-routing") + with pytest.raises(RouterRateLimitError) as raised: + await core.select("g", messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE + with pytest.raises(RouterRateLimitError): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [("a", "b"), ("b", "a")]) +async def test_a_routing_strategy_enum_member_is_accepted_like_its_string( + make_core: MakeCore, ids: tuple[str, str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", deployment_id) for deployment_id in ids], + routing_strategy=RoutingStrategy.LEAST_BUSY, + ) + assert await picked_ids(core, "g", messages=MESSAGES) == {ids[0]} + + +@pytest.mark.asyncio +async def test_select_with_an_unknown_kind_raises_value_error(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + with pytest.raises(ValueError, match=re.escape("unknown select kind 'nope'")): + await core.select("g", kind="nope") # pyright: ignore[reportCallIssue, reportArgumentType] + + +# --- simple-shuffle -------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_simple_shuffle_spreads_picks_over_unweighted_deployments(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"a", "b"} + + +@pytest.mark.asyncio +async def test_simple_shuffle_spreads_picks_when_every_weight_is_zero(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two(weight=0, rpm=0, tpm=0)) + assert await picked_ids(core, "g", times=SPREAD_PICKS) == {"a", "b"} + + +@pytest.mark.asyncio +async def test_simple_shuffle_falls_through_all_zero_weights_to_rpm(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight=0, rpm=5), deployment("g", "b", weight=0, rpm=0)]) + assert await picked_ids(core, "g") == {"a"} + + +@pytest.mark.asyncio +async def test_simple_shuffle_falls_through_unset_weight_and_rpm_to_tpm(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", tpm=0), deployment("g", "b", tpm=100)]) + assert await picked_ids(core, "g") == {"b"} + + +@pytest.mark.asyncio +async def test_rpm_takes_precedence_over_tpm(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", rpm=1, tpm=0), deployment("g", "b", rpm=0, tpm=1000)]) + assert await picked_ids(core, "g") == {"a"} + + +@pytest.mark.asyncio +async def test_simple_shuffle_with_a_non_numeric_weight_raises_type_error_at_pick(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight="heavy"), deployment("g", "b")]) + with pytest.raises(TypeError, match="Deployment weight must be numeric"): + await core.select("g") + + +@pytest.mark.asyncio +async def test_request_router_weights_are_keyed_by_the_alias_target_group(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two(), model_group_alias={"alias": "g"}) + assert await picked_ids(core, "alias", request_kwargs={"_router_weights": {"g": {"b": 1.0}}}) == {"b"} + + +@pytest.mark.asyncio +async def test_request_router_weights_for_another_group_are_ignored(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight=1), deployment("g", "b", weight=0)]) + assert await picked_ids(core, "g", request_kwargs={"_router_weights": {"other": {"b": 1.0}}}) == {"a"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("weights", [{"g": {"a": 0.0, "b": 0.0}}, {"g": {"a": -1.0}}, {"g": {"a": "1"}}]) +async def test_invalid_request_router_weights_fail_the_pick_with_a_validation_error( + make_core: MakeCore, weights: dict[str, object] +) -> None: + core: Final = make_core(model_list=_two()) + with pytest.raises(pydantic.ValidationError): + await core.select("g", {"_router_weights": weights}) + + +@pytest.mark.asyncio +async def test_an_empty_request_router_weights_group_falls_back_to_deployment_weights(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight=0), deployment("g", "b", weight=2)]) + assert await picked_ids(core, "g", request_kwargs={"_router_weights": {"g": {}}}) == {"b"} + + +@pytest.mark.asyncio +async def test_request_router_weights_give_unlisted_deployments_zero_weight(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[*_two(), deployment("g", "c")]) + assert await picked_ids(core, "g", request_kwargs={"_router_weights": {"g": {"c": 0.5}}}) == {"c"} + + +# --- stateful strategies without traffic --------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["latency-based-routing", "usage-based-routing-v2"]) +async def test_strategy_without_traffic_spreads_picks(make_core: MakeCore, routing_strategy: str) -> None: + core: Final = make_core(model_list=_two(), routing_strategy=routing_strategy) + assert await picked_ids(core, "g", times=SPREAD_PICKS, messages=MESSAGES) == {"a", "b"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [("a", "b"), ("b", "a")]) +async def test_cost_based_routing_without_known_costs_picks_the_first_configured( + make_core: MakeCore, ids: tuple[str, str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", deployment_id) for deployment_id in ids], routing_strategy="cost-based-routing" + ) + assert await picked_ids(core, "g", messages=MESSAGES) == {ids[0]} + + +def test_sync_pick_under_usage_based_routing_v1_picks_the_first_configured(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two(), routing_strategy="usage-based-routing") + for _ in range(5): + assert core.get_available_deployment("g", messages=MESSAGES)["model_info"]["id"] == "a" # pyright: ignore[reportAttributeAccessIssue] + + +def test_sync_completion_under_least_busy_calls_the_first_configured(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_two(), routing_strategy="least-busy") + for _ in range(3): + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("completion") == ["a", "a", "a"] + + +# --- per-request routing strategy override -------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("override", ["least-busy", RoutingStrategy.LEAST_BUSY], ids=["string", "enum"]) +async def test_a_request_routing_strategy_overrides_simple_shuffle(make_core: MakeCore, override: object) -> None: + core: Final = make_core(model_list=_two()) + assert await picked_ids(core, "g", request_kwargs={"routing_strategy": override}, messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_a_simple_shuffle_request_override_spreads_picks_on_a_least_busy_router(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two(), routing_strategy="least-busy") + picked: Final = await picked_ids( + core, "g", times=SPREAD_PICKS, request_kwargs={"routing_strategy": "simple-shuffle"}, messages=MESSAGES + ) + assert picked == {"a", "b"} + + +@pytest.mark.asyncio +async def test_a_cost_based_request_override_picks_the_cheapest(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "pricey", input_cost_per_token=0.00001, output_cost_per_token=0.00002), + deployment("g", "cheap", input_cost_per_token=0.000001, output_cost_per_token=0.000002), + ] + ) + picked: Final = await picked_ids( + core, "g", request_kwargs={"routing_strategy": "cost-based-routing"}, messages=MESSAGES + ) + assert picked == {"cheap"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("override", ["bogus", "lar1", "provider-budget-routing", 5]) +async def test_an_unsupported_request_routing_strategy_is_ignored(make_core: MakeCore, override: object) -> None: + core: Final = make_core(model_list=_two()) + picked: Final = await picked_ids( + core, "g", times=SPREAD_PICKS, request_kwargs={"routing_strategy": override}, messages=MESSAGES + ) + assert picked == {"a", "b"} + + +@pytest.mark.asyncio +async def test_a_request_routing_strategy_overrides_the_routing_group_strategy(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=_two(), routing_groups=[{"group_name": "grp", "models": ["g"], "routing_strategy": "least-busy"}] + ) + assert await picked_ids(core, "g", messages=MESSAGES) == {"a"} + picked: Final = await picked_ids( + core, "g", times=SPREAD_PICKS, request_kwargs={"routing_strategy": "simple-shuffle"}, messages=MESSAGES + ) + assert picked == {"a", "b"} + + +@pytest.mark.asyncio +async def test_completion_with_a_request_routing_strategy_calls_the_overridden_pick( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_two()) + for _ in range(5): + await core.acompletion(model="g", messages=MESSAGES, routing_strategy="least-busy") # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["a"] * 5 + + +# --- runtime strategy changes ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_applying_a_routing_strategy_switches_how_picks_are_made(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + core.apply(ChangeSet(settings={"routing_strategy": "least-busy"})) + assert core.settings().routing_strategy == "least-busy" + assert await picked_ids(core, "g", messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_applying_an_invalid_routing_strategy_raises_and_keeps_the_previous_one(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two(), routing_strategy="least-busy") + with pytest.raises(ValueError, match="Invalid routing_strategy") as raised: + core.apply(ChangeSet(settings={"routing_strategy": "bogus"})) + assert str(raised.value) == INVALID_STRATEGY_MESSAGE.format(name="bogus") + assert core.settings().routing_strategy == "least-busy" + assert await picked_ids(core, "g", messages=MESSAGES) == {"a"} + + +# --- custom routing strategies ----------------------------------------------------------------- + + +def _fixed(deployment_id: str) -> dict[str, object]: # mutable-ok: the strategy interface's own type + return { + "model_name": "x", + "litellm_params": {"model": f"openai/{deployment_id}"}, + "model_info": {"id": deployment_id}, + } + + +class FixedStrategy(CustomRoutingStrategyBase): + """Always answers ``custom`` from the async pick and ``sync-custom`` from the sync pick.""" + + def __init__(self) -> None: + self.asked: list[str] = [] # mutable-ok: the models each pick was asked for + + async def async_get_available_deployment( + self, + model: str, + messages: list[dict[str, str]] | None = None, # mutable-ok: the strategy interface's own type + input: str | list | None = None, # mutable-ok: the strategy interface's own type + specific_deployment: bool | None = False, + request_kwargs: dict | None = None, # mutable-ok: the strategy interface's own type + ) -> dict: # mutable-ok: the strategy interface's own type + self.asked.append(model) + return _fixed("custom") + + def get_available_deployment( + self, + model: str, + messages: list[dict[str, str]] | None = None, # mutable-ok: the strategy interface's own type + input: str | list | None = None, # mutable-ok: the strategy interface's own type + specific_deployment: bool | None = False, + request_kwargs: dict | None = None, # mutable-ok: the strategy interface's own type + ) -> dict: # mutable-ok: the strategy interface's own type + self.asked.append(model) + return _fixed("sync-custom") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["g", "unknown-model"]) +async def test_a_custom_strategy_answer_is_returned_as_is_for_any_model(make_core: MakeCore, model: str) -> None: + core: Final = make_core(model_list=_two()) + strategy: Final = FixedStrategy() + core.register_plugin(strategy) + chosen: Final = await core.select(model) + assert chosen == _fixed("custom") + assert strategy.asked == [model] + + +@pytest.mark.asyncio +async def test_a_custom_strategy_does_not_change_the_healthy_listing(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + core.register_plugin(FixedStrategy()) + healthy: Final = await core.select("g", kind="healthy") + assert sorted(d["model_info"]["id"] for d in healthy) == ["a", "b"] # pyright: ignore[reportIndexIssue, reportArgumentType] + + +@pytest.mark.asyncio +async def test_completion_calls_the_deployment_a_custom_strategy_returns( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_two()) + core.register_plugin(FixedStrategy()) + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["custom"] + assert host.calls[-1].model == "openai/custom" + + +def test_sync_completion_uses_the_custom_strategys_sync_pick(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_two()) + core.register_plugin(FixedStrategy()) + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("completion") == ["sync-custom"] + assert core.get_available_deployment("g") == _fixed("sync-custom") # pyright: ignore[reportAttributeAccessIssue] + + +@pytest.mark.asyncio +async def test_set_custom_routing_strategy_registers_like_register_plugin(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + core.set_custom_routing_strategy(FixedStrategy()) # pyright: ignore[reportAttributeAccessIssue] + assert await core.select("g") == _fixed("custom") + + +@pytest.mark.asyncio +async def test_applying_a_different_routing_strategy_drops_the_custom_strategy(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + core.register_plugin(FixedStrategy()) + core.apply(ChangeSet(settings={"routing_strategy": "least-busy"})) + assert await picked_ids(core, "g", messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_applying_the_current_routing_strategy_keeps_the_custom_strategy(make_core: MakeCore) -> None: + core: Final = make_core(model_list=_two()) + core.register_plugin(FixedStrategy()) + core.apply(ChangeSet(settings={"routing_strategy": "simple-shuffle"})) + assert await core.select("g") == _fixed("custom") + + +@pytest.mark.asyncio +async def test_a_custom_strategy_error_reaches_the_caller(make_core: MakeCore, host: ScriptedHost) -> None: + class Refusing(FixedStrategy): + async def async_get_available_deployment( + self, model: str, *args: object, **kwargs: object + ) -> dict: # mutable-ok: the strategy interface's own type # kwargs-ok: the strategy interface + raise litellm.BadRequestError(message="refused by strategy", model=model, llm_provider="") + + core: Final = make_core(model_list=_two()) + core.register_plugin(Refusing()) + with pytest.raises(litellm.BadRequestError, match="refused by strategy"): + await core.acompletion(model="g", messages=MESSAGES, num_retries=0) # pyright: ignore[reportAttributeAccessIssue] + assert host.calls == [] diff --git a/tests/unit/test_router/core/test_selection_characterization_traffic.py b/tests/unit/test_router/core/test_selection_characterization_traffic.py new file mode 100644 index 00000000000..7793b14ef6e --- /dev/null +++ b/tests/unit/test_router/core/test_selection_characterization_traffic.py @@ -0,0 +1,145 @@ +""" +Characterization of the stateful routing strategies once traffic has gone through the core: +least-busy follows in-flight requests, latency-based follows measured latency, and usage-based +routing follows the tokens and requests each deployment served. The host here is a +``TrafficHost``, which feeds the strategy loggers the way litellm's logging does around a call. +""" + +import asyncio +import re +from typing import Final + +import pytest + +import litellm +from tests.unit.test_router.core.scripted_host import bad_request, deployment +from tests.unit.test_router.core.selection_helpers import MESSAGES, MakeCore, TrafficHost, picked_ids + + +@pytest.fixture +def host() -> TrafficHost: + return TrafficHost() + + +async def _until_called(host: TrafficHost, count: int) -> None: + for _ in range(200): + if len(host.started) >= count: + return + await asyncio.sleep(0.005) + raise AssertionError(f"expected {count} started calls, saw {len(host.started)}") + + +# --- least-busy ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_least_busy_avoids_a_deployment_with_a_request_in_flight(make_core: MakeCore, host: TrafficHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="least-busy") + host.gates["openai/a"] = asyncio.Event() + in_flight: Final = asyncio.create_task(core.acompletion(model="g", messages=MESSAGES)) # pyright: ignore[reportAttributeAccessIssue] + await _until_called(host, 1) + assert host.started == ["openai/a"] + assert await picked_ids(core, "g", times=5, messages=MESSAGES) == {"b"} + host.gates["openai/a"].set() + await in_flight + assert await picked_ids(core, "g", times=5, messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_least_busy_releases_a_deployment_whose_request_failed(make_core: MakeCore, host: TrafficHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="least-busy") + host.gates["openai/a"] = asyncio.Event() + host.answer("openai/a", bad_request()) + in_flight: Final = asyncio.create_task(core.acompletion(model="g", messages=MESSAGES, num_retries=0)) # pyright: ignore[reportAttributeAccessIssue] + await _until_called(host, 1) + assert await picked_ids(core, "g", times=5, messages=MESSAGES) == {"b"} + host.gates["openai/a"].set() + with pytest.raises(litellm.BadRequestError): + await in_flight + assert await picked_ids(core, "g", times=5, messages=MESSAGES) == {"a"} + + +@pytest.mark.asyncio +async def test_least_busy_spreads_concurrent_requests_over_idle_deployments( + make_core: MakeCore, host: TrafficHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("g", "c")], routing_strategy="least-busy" + ) + gate: Final = asyncio.Event() + for model in ("openai/a", "openai/b", "openai/c"): + host.gates[model] = gate + calls: Final = [] + for count in range(1, 4): + calls.append(asyncio.create_task(core.acompletion(model="g", messages=MESSAGES))) # pyright: ignore[reportAttributeAccessIssue] + await _until_called(host, count) + assert host.started == ["openai/a", "openai/b", "openai/c"] + gate.set() + await asyncio.gather(*calls) + + +# --- latency-based ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_latency_based_routing_moves_to_the_faster_deployment(make_core: MakeCore, host: TrafficHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "slow"), deployment("g", "fast")], routing_strategy="latency-based-routing" + ) + host.delays["openai/slow"] = 0.15 + for _ in range(6): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert "fast" in host.deployments_called() + assert await picked_ids(core, "g", times=10, messages=MESSAGES) == {"fast"} + + +# --- usage-based --------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["usage-based-routing", "usage-based-routing-v2"]) +async def test_usage_based_routing_alternates_between_deployments_by_tokens_served( + make_core: MakeCore, host: TrafficHost, routing_strategy: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy=routing_strategy) + for _ in range(4): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + called: Final = host.deployments_called() + assert sorted(called[:2]) == ["a", "b"] + assert sorted(called[2:]) == ["a", "b"] + + +@pytest.mark.asyncio +async def test_usage_based_v2_stops_picking_a_deployment_once_its_tpm_is_used( + make_core: MakeCore, host: TrafficHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", tpm=15), deployment("g", "b", tpm=1000)], + routing_strategy="usage-based-routing-v2", + ) + for _ in range(4): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + called: Final = host.deployments_called() + assert called.count("a") <= 1 + assert called[-2:] == ["b", "b"] + + +@pytest.mark.asyncio +async def test_usage_based_v2_rate_limits_once_every_deployment_used_its_rpm( + make_core: MakeCore, host: TrafficHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", rpm=1), deployment("g", "b", rpm=1)], + routing_strategy="usage-based-routing-v2", + ) + await core.acompletion(model="g", messages=MESSAGES, num_retries=0) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES, num_retries=0) # pyright: ignore[reportAttributeAccessIssue] + assert sorted(host.deployments_called()) == ["a", "b"] + with pytest.raises(litellm.RateLimitError) as raised: + await core.acompletion(model="g", messages=MESSAGES, num_retries=0) # pyright: ignore[reportAttributeAccessIssue] + assert re.search( + r"No deployments available for selected model\. Passed model=g\. Deployments=\{'[ab]': \{'current_tpm': 10, " + r"'tpm_limit': inf, 'current_rpm': 1, 'rpm_limit': 1\}", + str(raised.value), + ) + assert len(host.calls) == 2 diff --git a/tests/unit/test_router/core/test_streaming_characterization_anthropic.py b/tests/unit/test_router/core/test_streaming_characterization_anthropic.py new file mode 100644 index 00000000000..6328767f96e --- /dev/null +++ b/tests/unit/test_router/core/test_streaming_characterization_anthropic.py @@ -0,0 +1,517 @@ +""" +Characterization of /v1/messages streaming through a router core: lifecycle frames held back while a +retry or fallback can still take over, provider ``event: error`` frames, raised stream errors, safeguard +refusals, same-group stream retries and their budget, and the hidden params the returned stream carries. + +Scripted streams are async iterators of raw SSE byte frames (``streaming_helpers.anthropic_stream``). +""" + +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.router import RetryPolicy +from litellm.types.utils import Usage +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + agentic_followup_failure, + bad_request, + deployment, + rate_limit, + server_error, +) +from tests.unit.test_router.core.streaming_helpers import ( + BLOCK_START, + MESSAGE_START, + MESSAGE_STOP, + MESSAGES, + PING, + ByteStream, + anthropic_stream, + collect_bytes, + error_frame, + install_orjson_stub_if_missing, + refusal_delta, + request_logging, + text_delta, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] +REFUSAL_DETAILS: Final = {"type": "refusal", "category": "cyber"} + + +def event_names(frames: list[bytes]) -> list[bytes]: + return [frame.split(b"\n", 1)[0].removeprefix(b"event: ") for frame in frames] + + +def fallback_ok(opened: list[ByteStream] | None = None) -> Callable[..., ByteStream]: + return anthropic_stream( + MESSAGE_START, + text_delta("B"), + MESSAGE_STOP, + hidden_params={"model_id": "b-stream", "additional_headers": {"x-provider-request-id": "req-b"}}, + opened=opened, + ) + + +def answered_ok(opened: list[ByteStream] | None = None) -> Callable[..., ByteStream]: + return anthropic_stream(MESSAGE_START, text_delta("A"), MESSAGE_STOP, opened=opened) + + +async def open_stream(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.aanthropic_messages(model="a", messages=MESSAGES, max_tokens=5, stream=True, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def hidden(stream: object) -> Mapping[str, object]: + return getattr(stream, "_hidden_params") + + +def headers(stream: object) -> Mapping[str, object]: + return hidden(stream)["additional_headers"] # pyright: ignore[reportReturnType] # a dict of headers + + +# --- provider error frames before content ------------------------------------------------------ + + +async def test_an_error_frame_before_content_falls_back_and_drops_the_primarys_held_frames( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, BLOCK_START, error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_an_error_frame_before_content_with_nothing_to_take_over_is_forwarded_verbatim( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + core = make_core(model_list=GROUPS_AB, num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, error_frame()] + assert host.deployments_called() == ["a"] + + +async def test_an_error_frame_split_across_byte_chunks_is_still_recognised( + make_core: MakeCore, host: ScriptedHost +) -> None: + frame = error_frame() + host.answer("openai/a", anthropic_stream(MESSAGE_START, frame[:25], frame[25:])) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_an_error_frame_after_content_is_forwarded_without_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, text_delta("x"), error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("x"), error_frame()] + assert host.deployments_called() == ["a"] + + +async def test_a_non_retriable_error_frame_is_forwarded_even_with_a_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + frame = error_frame("invalid_request_error", "bad request") + host.answer("openai/a", anthropic_stream(MESSAGE_START, frame)) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, frame] + assert host.deployments_called() == ["a"] + + +async def test_disable_fallbacks_forwards_a_retriable_error_frame(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core, disable_fallbacks=True)) + assert frames == [MESSAGE_START, error_frame()] + assert host.deployments_called() == ["a"] + + +async def test_when_the_fallback_after_an_error_frame_fails_the_frames_error_is_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame("overloaded_error", "primary overloaded"))) + host.answer("openai/b", server_error()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream = await open_stream(core) + with pytest.raises(litellm.InternalServerError, match="primary overloaded"): + await collect_bytes(stream) + assert host.deployments_called() == ["a", "b"] + + +# --- held lifecycle frames -------------------------------------------------------------------- + + +async def test_a_ping_is_forwarded_live_while_lifecycle_frames_are_held( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, PING, error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [PING, MESSAGE_START, text_delta("B"), MESSAGE_STOP] + + +async def test_with_nothing_to_take_over_every_frame_is_forwarded_live_in_order( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, PING, BLOCK_START, error_frame())) + core = make_core(model_list=GROUPS_AB, num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, PING, BLOCK_START, error_frame()] + + +async def test_held_frames_are_flushed_in_order_when_content_arrives(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, BLOCK_START, text_delta("A"), MESSAGE_STOP)) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, BLOCK_START, text_delta("A"), MESSAGE_STOP] + assert host.deployments_called() == ["a"] + + +async def test_held_frames_are_flushed_when_the_stream_ends_without_content( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, BLOCK_START, MESSAGE_STOP)) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, BLOCK_START, MESSAGE_STOP] + assert host.deployments_called() == ["a"] + + +async def test_two_hundred_held_frames_commit_the_stream_so_a_later_error_frame_is_forwarded( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(*([BLOCK_START] * 201), error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [*([BLOCK_START] * 201), error_frame()] + assert host.deployments_called() == ["a"] + + +# --- raised stream errors ---------------------------------------------------------------------- + + +async def test_a_raised_rate_limit_before_content_falls_back(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=rate_limit())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_a_raised_bad_request_before_content_is_raised_without_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=bad_request(message="raised 400"))) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream = await open_stream(core) + with pytest.raises(litellm.BadRequestError, match="raised 400"): + await collect_bytes(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_raised_error_after_content_is_raised_without_fallback(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, text_delta("x"), error=rate_limit(message="after"))) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="after"): + await collect_bytes(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_raised_server_error_before_content_is_retried_within_the_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", anthropic_stream(MESSAGE_START, error=server_error()), answered_ok()) + core = make_core(model_list=[deployment("a", "a1"), deployment("a", "a2")], num_retries=1) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert len(host.deployments_called()) == 2 + assert len(host.sleeps) == 1 + + +# --- same-group stream retries ----------------------------------------------------------------- + + +async def test_a_failed_agentic_followup_before_content_skips_the_retries_for_the_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + failed: Final = agentic_followup_failure(server_error()) + host.answer("openai/a", anthropic_stream(MESSAGE_START, error=failed), answered_ok()) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=2) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + assert host.sleeps == [] + + +async def test_a_stream_retry_carries_the_superseded_attempt_usage(make_core: MakeCore, host: ScriptedHost) -> None: + logging_obj: Final = request_logging() + usage: Final = Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2) + failing: Final = anthropic_stream(MESSAGE_START, error=server_error()) + + def fails_after_usage(**kwargs: object) -> ByteStream: # kwargs-ok: the request + logging_obj.model_call_details["combined_usage_object"] = usage + return failing(**kwargs) + + host.answer("openai/a", fails_after_usage, answered_ok()) + core = make_core(model_list=[deployment("a", "a")], num_retries=1) + await collect_bytes(await open_stream(core, litellm_logging_obj=logging_obj)) + assert host.deployments_called() == ["a", "a"] + assert logging_obj.model_call_details["failed_attempts_usage"] == ((usage, 0.0),) + + +async def test_an_error_frame_before_content_is_retried_within_the_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame()), answered_ok()) + core = make_core(model_list=[deployment("a", "a")], num_retries=2) + stream = await open_stream(core) + frames = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "a"] + assert len(host.sleeps) == 1 + assert headers(stream)["x-litellm-attempted-retries"] == 1 + assert headers(stream)["x-litellm-max-retries"] == 2 + + +async def test_a_retry_policy_grants_stream_retries_when_num_retries_is_zero( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame("rate_limit_error", "slow")), answered_ok()) + core = make_core( + model_list=[deployment("a", "a")], num_retries=0, retry_policy=RetryPolicy(RateLimitErrorRetries=2) + ) + stream = await open_stream(core) + frames = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "a"] + assert headers(stream)["x-litellm-max-retries"] == 2 + + +async def test_request_num_retries_zero_opts_out_of_the_retry_policy_for_a_stream( + make_core: MakeCore, host: ScriptedHost +) -> None: + frame = error_frame("rate_limit_error", "slow") + host.answer("openai/a", anthropic_stream(MESSAGE_START, frame), answered_ok()) + core = make_core( + model_list=[deployment("a", "a")], num_retries=0, retry_policy=RetryPolicy(RateLimitErrorRetries=2) + ) + frames = await collect_bytes(await open_stream(core, num_retries=0)) + assert frames == [MESSAGE_START, frame] + assert host.deployments_called() == ["a"] + + +async def test_the_deployment_num_retries_is_the_stream_retry_budget(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame()), answered_ok()) + core = make_core(model_list=[deployment("a", "a", num_retries=1)], num_retries=0) + stream = await open_stream(core) + frames = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("A"), MESSAGE_STOP] + assert headers(stream)["x-litellm-max-retries"] == 1 + + +async def test_stream_retries_stop_at_the_budget_and_forward_the_last_error_frame( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + core = make_core(model_list=[deployment("a", "a")], num_retries=2) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, error_frame()] + assert host.deployments_called() == ["a", "a", "a"] + assert len(host.sleeps) == 2 + + +async def test_spent_stream_retries_hand_over_to_the_fallback_chain(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=1) + stream = await open_stream(core) + frames = await collect_bytes(stream) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "a", "b"] + assert len(host.sleeps) == 1 + + +async def test_a_superseded_attempt_and_every_later_stream_are_closed(make_core: MakeCore, host: ScriptedHost) -> None: + opened: list[ByteStream] = [] + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame(), opened=opened)) + host.answer("openai/b", fallback_ok(opened)) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=1) + await collect_bytes(await open_stream(core)) + assert len(opened) == 3 + assert all(stream.closed for stream in opened) + + +# --- safeguard refusals ------------------------------------------------------------------------ + + +async def test_a_refusal_with_stop_details_before_content_falls_back(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, refusal_delta(REFUSAL_DETAILS), MESSAGE_STOP)) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_a_refusal_falls_back_through_the_content_policy_chain(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, refusal_delta(REFUSAL_DETAILS), MESSAGE_STOP)) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, content_policy_fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("B"), MESSAGE_STOP] + assert host.deployments_called() == ["a", "b"] + + +async def test_a_refusal_without_stop_details_is_forwarded(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, refusal_delta(), MESSAGE_STOP)) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, refusal_delta(), MESSAGE_STOP] + assert host.deployments_called() == ["a"] + + +@pytest.mark.parametrize( + ("router_arguments", "request_kwargs"), + [({"fallbacks": [{"a": ["b"]}]}, {"disable_fallbacks": True}), ({}, {})], + ids=["fallbacks-disabled", "no-fallbacks"], +) +async def test_a_refusal_with_no_reachable_fallback_is_forwarded( + make_core: MakeCore, host: ScriptedHost, router_arguments: dict[str, object], request_kwargs: dict[str, object] +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, refusal_delta(REFUSAL_DETAILS), MESSAGE_STOP)) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, num_retries=0, **router_arguments) + frames = await collect_bytes(await open_stream(core, **request_kwargs)) + assert frames == [MESSAGE_START, refusal_delta(REFUSAL_DETAILS), MESSAGE_STOP] + assert host.deployments_called() == ["a"] + + +async def test_a_refusal_after_content_is_forwarded(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, text_delta("x"), refusal_delta(REFUSAL_DETAILS))) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert frames == [MESSAGE_START, text_delta("x"), refusal_delta(REFUSAL_DETAILS)] + assert host.deployments_called() == ["a"] + + +# --- what the fallback hands back -------------------------------------------------------------- + + +async def test_a_fallback_that_answers_without_streaming_is_replayed_as_an_sse_lifecycle( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer( + "openai/b", + { + "id": "msg_b", + "type": "message", + "role": "assistant", + "model": "b", + "content": [{"type": "text", "text": "B"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + frames = await collect_bytes(await open_stream(core)) + assert event_names(frames) == [ + b"message_start", + b"content_block_start", + b"content_block_delta", + b"content_block_stop", + b"message_delta", + b"message_stop", + ] + assert b'"B"' in frames[2] + + +# --- hidden params on the returned stream ------------------------------------------------------ + + +async def test_a_stream_without_recovery_reports_the_serving_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", answered_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream = await open_stream(core) + await collect_bytes(stream) + assert hidden(stream)["model_id"] == "a" + assert headers(stream) == { + "x-litellm-model-group": "a", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 0, + } + + +async def test_after_a_stream_fallback_the_stream_carries_the_fallbacks_hidden_params_and_headers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", fallback_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=0) + stream = await open_stream(core) + await collect_bytes(stream) + assert hidden(stream)["model_id"] == "b-stream" + assert headers(stream) == { + "x-litellm-model-group": "b", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 1, + "x-provider-request-id": "req-b", + } + + +async def test_after_retries_then_a_fallback_the_retry_budget_header_lingers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", anthropic_stream(MESSAGE_START, error_frame())) + host.answer("openai/b", answered_ok()) + core = make_core(model_list=GROUPS_AB, fallbacks=[{"a": ["b"]}], num_retries=1) + stream = await open_stream(core) + await collect_bytes(stream) + assert headers(stream) == { + "x-litellm-model-group": "b", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 1, + "x-litellm-max-retries": 1, + } + + +async def test_a_stream_retry_keeps_reporting_the_first_deployment_when_the_retry_has_no_hidden_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", anthropic_stream(MESSAGE_START, error=server_error()), answered_ok()) + core = make_core(model_list=[deployment("a", "a1"), deployment("a", "a2")], num_retries=1) + stream = await open_stream(core) + await collect_bytes(stream) + first, _retried = host.deployments_called() + assert hidden(stream)["model_id"] == first + assert headers(stream)["x-litellm-attempted-retries"] == 1 diff --git a/tests/unit/test_router/core/test_streaming_characterization_chat.py b/tests/unit/test_router/core/test_streaming_characterization_chat.py new file mode 100644 index 00000000000..260ad8534cf --- /dev/null +++ b/tests/unit/test_router/core/test_streaming_characterization_chat.py @@ -0,0 +1,737 @@ +""" +Characterization of chat-completion streaming through a router core: the first chunk read inside the +attempt, mid-stream errors and their fallback, mid-stream continuation, the sync twin, and the stream +timeout sent to the host. + +Each scripted stream is a ``CustomStreamWrapper`` built by ``streaming_helpers.chat_stream``; a stream +that raises before its first chunk fails inside the attempt (retries and fallbacks run as for any error), +while one that raises later fails while the caller iterates. +""" + +import asyncio +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_core import RouterCore +from litellm.types.utils import Usage +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + bad_request, + content_policy, + context_window, + deployment, + rate_limit, + server_error, +) +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGES, + chat_stream, + collect_texts, + collect_texts_sync, + drain_into, +) + +MakeCore = Callable[..., RouterCore] +PREFILL_MODEL: Final = "anthropic/claude-sonnet-4-5" +"""A model whose deployments accept an assistant prefill, so a continuation may land on them.""" +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] + + +def build(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(**{"model_list": GROUPS_AB, "fallbacks": [{"a": ["b"]}], "num_retries": 0, **arguments}) + + +async def open_stream(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.acompletion(model="a", messages=MESSAGES, stream=True, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def headers(response: object) -> Mapping[str, object]: + return getattr(response, "_hidden_params")["additional_headers"] + + +def model_id(response: object) -> object: + return getattr(response, "_hidden_params").get("model_id") + + +# --- the first chunk is read inside the attempt ----------------------------------------------- + + +async def test_a_stream_that_fails_on_its_first_chunk_is_retried_inside_the_attempt( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(), fail_after=0), chat_stream("ok")) + core = make_core(model_list=[deployment("a", "a")], num_retries=2) + stream = await open_stream(core) + assert await collect_texts(stream) == "ok" + assert host.deployments_called() == ["a", "a"] + assert len(host.sleeps) == 1 + assert headers(stream)["x-litellm-attempted-retries"] == 1 + + +async def test_a_stream_that_fails_on_its_first_chunk_falls_back_after_its_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(), fail_after=0)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, num_retries=2) + stream = await open_stream(core) + assert await collect_texts(stream) == "B" + assert host.deployments_called() == ["a", "a", "a", "b"] + assert len(host.sleeps) == 3 + assert model_id(stream) == "b" + assert headers(stream)["x-litellm-attempted-fallbacks"] == 1 + + +async def test_a_first_chunk_failure_with_nowhere_to_go_raises_the_provider_error_from_the_call( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(message="first chunk"), fail_after=0)) + core = make_core(model_list=[deployment("a", "a")], num_retries=0) + with pytest.raises(litellm.RateLimitError, match="first chunk"): + await open_stream(core) + + +async def test_a_first_chunk_bad_request_is_raised_from_the_call_without_retry( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=bad_request(message="nope"), fail_after=0)) + core = make_core(model_list=GROUPS_AB, num_retries=2) + with pytest.raises(litellm.BadRequestError, match="nope"): + await open_stream(core) + assert host.deployments_called() == ["a"] + + +async def test_the_first_chunk_already_read_is_still_delivered_to_the_caller( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("first", "second", "third")) + core = make_core(model_list=[deployment("a", "a")]) + stream = await open_stream(core) + assert await collect_texts(stream) == "firstsecondthird" + + +async def test_a_stream_that_ends_without_any_chunk_or_finish_reason_ends_quietly_without_text( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(finish=False)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + assert await collect_texts(stream) == "" + assert host.deployments_called() == ["a"] + + +async def test_a_stream_that_ends_after_content_without_a_finish_reason_raises_while_iterating( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", finish=False)) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.APIConnectionError, match="Stream ended before a finish_reason was received"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +# --- mid-stream errors ------------------------------------------------------------------------- + + +async def test_a_mid_stream_error_after_content_raises_the_provider_error_not_mid_stream_fallback_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="cut after content"))) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="cut after content") as raised: + await collect_texts(stream) + assert not isinstance(raised.value, MidStreamFallbackError) + assert host.deployments_called() == ["a"] + + +async def test_a_mid_stream_error_after_content_delivers_the_text_before_it( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit())) + core = build(make_core) + stream = await open_stream(core) + received: list[object] = [] + with pytest.raises(litellm.RateLimitError): + await drain_into(stream, received) + assert "".join(getattr(chunk, "choices")[0].delta.content or "" for chunk in received) == "hello" + + +async def test_a_mid_stream_error_before_content_falls_back_to_the_next_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + assert await collect_texts(stream) == "B" + assert host.deployments_called() == ["a", "b"] + + +async def test_a_mid_stream_error_before_content_does_not_retry_the_same_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, num_retries=2) + stream = await open_stream(core) + assert await collect_texts(stream) == "B" + assert host.deployments_called() == ["a", "b"] + assert host.sleeps == [] + + +async def test_a_mid_stream_fallback_sends_the_original_messages_to_the_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + await collect_texts(stream) + fallback_call = host.calls_to("acompletion")[1] + assert fallback_call.kwargs["messages"] == MESSAGES + assert fallback_call.model_group == "b" + assert fallback_call.kwargs["stream"] is True + + +async def test_a_mid_stream_error_before_content_without_fallbacks_raises_the_provider_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="no fallback"), leading_role_chunk=True)) + core = make_core(model_list=GROUPS_AB, num_retries=0) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="no fallback") as raised: + await collect_texts(stream) + assert not isinstance(raised.value, MidStreamFallbackError) + assert host.deployments_called() == ["a"] + + +async def test_disable_fallbacks_on_the_request_raises_a_mid_stream_error_before_content( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="disabled"), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core, disable_fallbacks=True) + with pytest.raises(litellm.RateLimitError, match="disabled"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_per_request_fallback_list_is_used_for_a_mid_stream_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/c", chat_stream("C")) + core = build(make_core, model_list=[*GROUPS_AB, deployment("c", "c")]) + stream = await open_stream(core, fallbacks=["c"]) + assert await collect_texts(stream) == "C" + assert host.deployments_called() == ["a", "c"] + + +async def test_a_mid_stream_fallback_walks_the_chain_until_a_group_answers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", server_error()) + host.answer("openai/c", chat_stream("C")) + core = build(make_core, model_list=[*GROUPS_AB, deployment("c", "c")], fallbacks=[{"a": ["b", "c"]}]) + stream = await open_stream(core) + assert await collect_texts(stream) == "C" + assert host.deployments_called() == ["a", "b", "c"] + + +async def test_when_the_mid_stream_fallback_also_fails_the_primary_provider_error_is_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="primary"), leading_role_chunk=True)) + host.answer("openai/b", server_error()) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="primary"): + await collect_texts(stream) + assert host.deployments_called() == ["a", "b"] + + +async def test_a_mid_stream_server_error_before_content_falls_through_order_tiers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a1", chat_stream(error=server_error(), leading_role_chunk=True)) + host.answer("openai/a2", chat_stream("A2")) + core = make_core(model_list=[deployment("a", "a1", order=1), deployment("a", "a2", order=2)], num_retries=0) + stream = await open_stream(core) + assert await collect_texts(stream) == "A2" + assert host.deployments_called() == ["a1", "a2"] + assert model_id(stream) == "a2" + assert headers(stream)["x-litellm-model-group"] == "a" + + +async def test_a_mid_stream_bad_request_before_content_is_raised_without_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=bad_request(message="client error"), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.BadRequestError, match="client error"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_mid_stream_content_policy_error_before_content_is_raised_without_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=content_policy(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, fallbacks=None, content_policy_fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core) + with pytest.raises(litellm.ContentPolicyViolationError): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_mid_stream_context_window_error_before_content_uses_the_context_window_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=context_window(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, fallbacks=None, context_window_fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core) + assert await collect_texts(stream) == "B" + assert host.deployments_called() == ["a", "b"] + + +async def test_a_mid_stream_context_window_error_after_content_is_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=context_window())) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, context_window_fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core) + with pytest.raises(litellm.ContextWindowExceededError): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +# --- hidden params on a streamed response ----------------------------------------------------- + + +async def test_a_stream_without_fallback_reports_the_serving_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x")) + core = build(make_core) + stream = await open_stream(core) + await collect_texts(stream) + assert model_id(stream) == "a" + assert headers(stream)["x-litellm-model-group"] == "a" + assert headers(stream)["x-litellm-attempted-fallbacks"] == 0 + + +async def test_after_a_mid_stream_fallback_the_stream_reports_the_fallback_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + await collect_texts(stream) + assert model_id(stream) == "b" + assert headers(stream) == { + "x-litellm-model-group": "b", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 1, + } + + +async def test_after_a_mid_stream_fallback_each_fallback_chunk_carries_the_fallback_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", chat_stream("B")) + core = build(make_core) + stream = await open_stream(core) + chunks = [chunk async for chunk in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + fallback_chunks = chunks[1:] + assert fallback_chunks + assert all(chunk._hidden_params.get("model_id") == "b" for chunk in fallback_chunks) + assert fallback_chunks[-1]._hidden_params["additional_headers"]["x-litellm-model-group"] == "b" + + +async def test_when_the_mid_stream_fallback_fails_the_stream_still_reports_the_primary( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer("openai/b", server_error()) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError): + await collect_texts(stream) + assert model_id(stream) == "a" + assert headers(stream)["x-litellm-model-group"] == "a" + + +# --- mid-stream continuation ------------------------------------------------------------------- + + +def continuation_core(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: constructor arguments + return build( + make_core, + model_list=[deployment("a", "a"), deployment("b", "b", model=PREFILL_MODEL)], + enable_mid_stream_fallback_continuation=True, + **arguments, + ) + + +async def test_continuation_off_by_default_raises_after_content(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="cut"))) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = build(make_core, model_list=[deployment("a", "a"), deployment("b", "b", model=PREFILL_MODEL)]) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="cut"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_continuation_continues_after_content_on_a_prefill_capable_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit())) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core) + stream = await open_stream(core) + assert await collect_texts(stream) == "hello world" + assert host.deployments_called() == ["a", "b"] + + +async def test_continuation_sends_the_partial_text_as_an_assistant_prefill( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit())) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core) + stream = await open_stream(core) + await collect_texts(stream) + assert host.calls[1].kwargs["messages"] == [*MESSAGES, {"role": "assistant", "content": "hello", "prefix": True}] + + +async def test_continuation_extends_a_trailing_prefill_turn_in_place(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("Once", " upon", error=rate_limit())) + host.answer(PREFILL_MODEL, chat_stream(" a time")) + core = continuation_core(make_core) + messages = [*MESSAGES, {"role": "assistant", "content": "Once", "prefix": True}] + stream = await core.acompletion(model="a", messages=messages, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await collect_texts(stream) + sent = host.calls[1].kwargs["messages"] + assert sent[-1] == {"role": "assistant", "content": "Once upon", "prefix": True} + assert len(sent) == 2 + + +async def test_continuation_declines_when_the_request_ends_in_a_plain_assistant_turn( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(message="cut"))) + host.answer(PREFILL_MODEL, chat_stream("y")) + core = continuation_core(make_core) + messages = [*MESSAGES, {"role": "assistant", "content": "earlier"}] + stream = await core.acompletion(model="a", messages=messages, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.RateLimitError, match="cut"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_continuation_reduces_the_output_ceiling_by_the_tokens_already_emitted( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("hello", " there", error=rate_limit())) + host.answer(PREFILL_MODEL, chat_stream(" friend")) + core = continuation_core(make_core) + stream = await open_stream(core, max_tokens=50) + await collect_texts(stream) + assert [call.kwargs["max_tokens"] for call in host.calls] == [50, 48] + + +async def test_continuation_raises_the_provider_error_when_the_output_ceiling_is_spent( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("hello", " there", error=rate_limit(message="spent"))) + host.answer(PREFILL_MODEL, chat_stream(" friend")) + core = continuation_core(make_core) + stream = await open_stream(core, max_tokens=2) + with pytest.raises(litellm.RateLimitError, match="spent"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_continuation_never_lands_on_a_fallback_that_cannot_take_a_prefill( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="no prefill target"))) + host.answer("openai/b", chat_stream("B")) + core = build(make_core, enable_mid_stream_fallback_continuation=True) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="no prefill target"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +@pytest.mark.parametrize( + "request_kwargs", + [ + {"response_format": {"type": "json_object"}}, + {"tool_choice": "required", "tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}]}, + ], + ids=["json-response-format", "forced-tool-choice"], +) +async def test_continuation_declines_a_constrained_request( + make_core: MakeCore, host: ScriptedHost, request_kwargs: dict[str, object] +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="constrained"))) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core) + stream = await open_stream(core, **request_kwargs) + with pytest.raises(litellm.RateLimitError, match="constrained"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_continuation_declines_a_fallback_target_that_overrides_request_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="override"))) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core, fallbacks=[{"a": [{"model": "b", "temperature": 0.1}]}]) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="override"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + +async def test_continuation_runs_a_plain_fallback_for_an_error_before_content( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) + host.answer(PREFILL_MODEL, chat_stream("B")) + core = continuation_core(make_core) + stream = await open_stream(core) + assert await collect_texts(stream) == "B" + assert host.calls[1].kwargs["messages"] == MESSAGES + + +def recorded( + answer: Callable[..., CustomStreamWrapper], opened: list[CustomStreamWrapper] +) -> Callable[..., CustomStreamWrapper]: + """``answer``, keeping the stream it opens so a test can read that stream's logging object.""" + + def record(**request: object) -> CustomStreamWrapper: # kwargs-ok: the host passes the request kwargs + stream: Final = answer(**request) + opened.append(stream) + return stream + + return record + + +def owe_a_served_round(logging_obj: LiteLLMLogging) -> None: + """What an agentic follow-up that failed leaves on the attempt: a stash its success log must also bill.""" + logging_obj.record_served_rounds_for_failure(((Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), 0.5),)) + + +async def test_continuation_bills_the_priced_partial_on_the_fallbacks_log( + make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(LiteLLMLogging, "response_cost_calculator", lambda *_args, **_kwargs: 0.25) + opened: Final[list[CustomStreamWrapper]] = [] + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit())) + host.answer(PREFILL_MODEL, recorded(chat_stream(" world"), opened)) + core = continuation_core(make_core) + assert await collect_texts(await open_stream(core)) == "hello world" + details: Final = opened[0].logging_obj.model_call_details + partial_usage: Final = details["continued_partial_usage"] + assert isinstance(partial_usage, Usage) and partial_usage.completion_tokens > 0 + assert details["continued_partial_cost"] == 0.25 + assert details["continued_response_id"] + + +async def test_continuation_leaves_the_cut_streams_stash_for_the_failure_log_only( + make_core: MakeCore, host: ScriptedHost +) -> None: + opened: Final[list[CustomStreamWrapper]] = [] + host.answer("openai/a", recorded(chat_stream("he", "llo", error=rate_limit()), opened)) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core) + stream = await open_stream(core) + cut_logging: Final = opened[0].logging_obj + owe_a_served_round(cut_logging) + assert await collect_texts(stream) == "hello world" + cut_logging.carry_failed_attempt_usage() + assert cut_logging._failed_attempts_owed_on_success == () # pyright: ignore[reportPrivateUsage] # the continuation bills it, so the success log must not + assert len(cut_logging.model_call_details["failed_attempts_usage"]) == 1 + + +async def test_a_plain_fallback_before_content_leaves_the_stash_owed_on_success( + make_core: MakeCore, host: ScriptedHost +) -> None: + opened: Final[list[CustomStreamWrapper]] = [] + host.answer("openai/a", recorded(chat_stream(error=rate_limit(), leading_role_chunk=True), opened)) + host.answer(PREFILL_MODEL, chat_stream("B")) + core = continuation_core(make_core) + stream = await open_stream(core) + cut_logging: Final = opened[0].logging_obj + owe_a_served_round(cut_logging) + assert await collect_texts(stream) == "B" + cut_logging.carry_failed_attempt_usage() + assert len(cut_logging._failed_attempts_owed_on_success) == 1 # pyright: ignore[reportPrivateUsage] # nothing else bills it + + +# --- the deployment slot held by an open stream ----------------------------------------------- + + +async def test_an_open_stream_holds_its_max_parallel_requests_slot_until_consumed( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("x")) + core = make_core(model_list=[deployment("a", "a", max_parallel_requests=1)], num_retries=0) + first = await open_stream(core) + with pytest.raises(litellm.RateLimitError): + await asyncio.wait_for(open_stream(core), 5) + await collect_texts(first) + second = await asyncio.wait_for(open_stream(core), 5) + assert await collect_texts(second) == "x" + + +# --- the sync twin ----------------------------------------------------------------------------- + + +def test_sync_stream_falls_back_on_a_mid_stream_error_before_content(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True)) + host.answer("openai/b", chat_stream("B", sync=True)) + core = build(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert collect_texts_sync(stream) == "B" + assert [(call.op, call.deployment_id) for call in host.calls] == [("completion", "a"), ("completion", "b")] + assert model_id(stream) == "b" + + +def test_sync_stream_raises_the_provider_error_after_content(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(message="sync cut"), sync=True)) + host.answer("openai/b", chat_stream("B", sync=True)) + core = build(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.RateLimitError, match="sync cut"): + collect_texts_sync(stream) + assert host.deployments_called() == ["a"] + + +def test_sync_stream_never_continues_after_content_even_with_continuation_on( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="sync no continuation"), sync=True)) + host.answer(PREFILL_MODEL, chat_stream(" world", sync=True)) + core = continuation_core(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.RateLimitError, match="sync no continuation"): + collect_texts_sync(stream) + assert host.deployments_called() == ["a"] + + +def test_sync_stream_disable_fallbacks_raises_a_mid_stream_error_before_content( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="sync disabled"), leading_role_chunk=True, sync=True)) + host.answer("openai/b", chat_stream("B", sync=True)) + core = build(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True, disable_fallbacks=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.RateLimitError, match="sync disabled"): + collect_texts_sync(stream) + assert host.deployments_called() == ["a"] + + +def test_sync_stream_first_chunk_failure_falls_back_inside_the_call(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("x", error=rate_limit(), fail_after=0, sync=True)) + host.answer("openai/b", chat_stream("B", sync=True)) + core = build(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert collect_texts_sync(stream) == "B" + assert host.deployments_called() == ["a", "b"] + + +def test_sync_stream_when_the_fallback_fails_raises_the_primary_error(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream(error=rate_limit(message="sync primary"), leading_role_chunk=True, sync=True)) + host.answer("openai/b", server_error()) + core = build(make_core) + stream = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + with pytest.raises(litellm.RateLimitError, match="sync primary"): + collect_texts_sync(stream) + assert host.deployments_called() == ["a", "b"] + + +# --- the stream timeout sent to the host ------------------------------------------------------- + + +@pytest.mark.parametrize( + ("router_arguments", "deployment_params", "request_kwargs", "expected"), + [ + ({"stream_timeout": 4}, {}, {}, 4), + ({"stream_timeout": 4}, {"stream_timeout": 3}, {}, 3.0), + ({"stream_timeout": 4}, {"stream_timeout": 3}, {"stream_timeout": 2}, 2), + ({}, {"stream_timeout_by_service_tier": {"flex": 9}, "stream_timeout": 3}, {"service_tier": "flex"}, 9.0), + ({}, {"stream_timeout_by_service_tier": {"flex": 9}, "stream_timeout": 3}, {"service_tier": "auto"}, 3.0), + ( + {}, + {"stream_timeout_by_service_tier": {"flex": 9}}, + {"service_tier": "flex", "stream_timeout": 2}, + 2, + ), + ({"default_litellm_params": {"stream_timeout": 8}}, {}, {}, 8), + ], + ids=[ + "router", + "deployment-over-router", + "request-over-deployment", + "service-tier-over-deployment", + "unmatched-service-tier-uses-deployment", + "request-over-service-tier", + "default-litellm-params", + ], +) +async def test_the_stream_timeout_comes_from_request_then_tier_then_deployment_then_router( + make_core: MakeCore, + host: ScriptedHost, + router_arguments: dict[str, object], + deployment_params: dict[str, object], + request_kwargs: dict[str, object], + expected: float, +) -> None: + host.answer("openai/a", chat_stream("x")) + core = make_core(model_list=[deployment("a", "a", **deployment_params)], **router_arguments) + stream = await open_stream(core, **request_kwargs) + await collect_texts(stream) + assert host.calls[0].kwargs["stream_timeout"] == expected + + +@pytest.mark.parametrize("router_arguments", [{}, {"timeout": 42}], ids=["no-router-timeout", "router-timeout"]) +async def test_without_any_stream_timeout_none_is_sent_and_the_plain_timeout_applies( + make_core: MakeCore, host: ScriptedHost, router_arguments: dict[str, object] +) -> None: + host.answer("openai/a", chat_stream("x")) + core = make_core(model_list=[deployment("a", "a")], **router_arguments) + stream = await open_stream(core) + await collect_texts(stream) + sent = host.calls[0].kwargs + assert "stream_timeout" not in sent + assert sent["timeout"] == router_arguments.get("timeout", 6000.0) + + +async def test_a_non_streaming_request_sends_no_stream_timeout(make_core: MakeCore, host: ScriptedHost) -> None: + core = make_core(model_list=[deployment("a", "a")], stream_timeout=4) + await core.acompletion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert "stream_timeout" not in host.calls[0].kwargs diff --git a/tests/unit/test_router/core/test_streaming_characterization_outcomes.py b/tests/unit/test_router/core/test_streaming_characterization_outcomes.py new file mode 100644 index 00000000000..99b3add153b --- /dev/null +++ b/tests/unit/test_router/core/test_streaming_characterization_outcomes.py @@ -0,0 +1,382 @@ +""" +Characterization of what a streamed attempt records against its deployment: the success, tokens and +request a stream counts once it is fully consumed, and the failure and cooldown a stream that breaks +after its first chunk counts, for chat, Responses and /v1/messages streams. + +Every scripted stream here logs through the host when it ends (``logged=True``), as litellm's stream +logging does, so a core that learns outcomes from logging callbacks sees the stream's outcome only +then, with the assembled response's usage. +""" + +import asyncio +import time +from collections.abc import Callable +from typing import Final + +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.llms.openai import ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rate_limit +from tests.unit.test_router.core.streaming_helpers import ( + MESSAGE_STOP, + MESSAGES, + Event, + anthropic_stream, + chat_stream, + collect_bytes, + collect_event_types, + collect_texts, + collect_texts_sync, + install_orjson_stub_if_missing, + responses_stream, + sse, + text_delta, +) + +install_orjson_stub_if_missing() + +MakeCore = Callable[..., RouterCore] +USAGE: Final = {"prompt_tokens": 4, "completion_tokens": 8, "total_tokens": 12} + + +def group( + *ids: str, **params: object +) -> list[dict[str, object]]: # kwargs-ok: litellm_params # mutable-ok: a model_list + return [deployment("g", deployment_id, None, **params) for deployment_id in ids] + + +async def open_chat(core: RouterCore, model: str = "g") -> object: + stream: Final[object] = await core.acompletion(model=model, messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + return stream # pyright: ignore[reportUnknownVariableType] # see above + + +def open_chat_sync(core: RouterCore) -> object: + stream: Final[object] = core.completion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + return stream # pyright: ignore[reportUnknownVariableType] # see above + + +async def open_responses(core: RouterCore) -> object: + stream: Final[object] = await core.aresponses(model="g", input="hi", stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + return stream # pyright: ignore[reportUnknownVariableType] # see above + + +async def open_messages(core: RouterCore) -> object: + stream: Final[object] = await core.aanthropic_messages(model="g", messages=MESSAGES, max_tokens=5, stream=True) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # endpoint method + return stream # pyright: ignore[reportUnknownVariableType] # see above + + +async def settle() -> None: + """Let the logging and recording tasks a finished stream scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +def eventually(check: Callable[[], bool]) -> None: + """Wait for ``check``: litellm logs a sync stream's outcome from a worker thread.""" + deadline: Final = time.monotonic() + 5.0 + while not check() and time.monotonic() < deadline: + time.sleep(0.01) + + +def successes(core: RouterCore, deployment_id: str) -> object: + count: Final[object] = core.health().store.get_cache(f"{deployment_id}:successes", local_only=True) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped + return count # pyright: ignore[reportUnknownVariableType] # see above + + +def cooled(core: RouterCore, *ids: str) -> list[str]: # mutable-ok: a fresh list + return sorted(dict(core.health().cooldowns.get_active_cooldowns(list(ids), None))) + + +# --- a stream consumed to its end ---------------------------------------------------------------- + + +async def test_a_consumed_stream_counts_its_final_tokens_and_one_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("hello", " world", usage=USAGE, logged=True)) + core: Final = make_core(model_list=group("a", rpm=100)) + + stream: Final = await open_chat(core) + assert await collect_texts(stream) == "hello world" + await settle() + + assert await core.health().usage("g") == (12, 1) + assert successes(core, "a") == 1 + + +async def test_a_stream_counts_only_its_request_before_it_is_consumed(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("hello", usage=USAGE, logged=True)) + core: Final = make_core(model_list=group("a", rpm=100)) + + await open_chat(core) + await settle() + + # The request counts when the stream opens, with the tokens it has read so far (none). + assert await core.health().usage("g") == (0, 1) + assert successes(core, "a") is None + + +def test_a_sync_stream_counts_its_success_only_once_consumed(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", chat_stream("hello", usage=USAGE, logged=True, sync=True)) + core: Final = make_core(model_list=group("a", rpm=100)) + + stream: Final = open_chat_sync(core) + assert successes(core, "a") is None + assert collect_texts_sync(stream) == "hello" + eventually(lambda: successes(core, "a") is not None) + + assert successes(core, "a") == 1 + # A sync call's success callback counts the success alone: the request counted when the stream + # opened, and no tokens. + assert asyncio.run(core.health().usage("g")) == (0, 1) + + +def test_a_sync_stream_that_breaks_after_content_counts_one_failure_and_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True, sync=True)) + core: Final = make_core( + model_list=[deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + num_retries=0, + allowed_fails=0, + ) + + stream: Final = open_chat_sync(core) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + collect_texts_sync(stream) + eventually(lambda: bool(cooled(core, "a", "b"))) + + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert cooled(core, "a", "b") == ["a"] + assert asyncio.run(core.health().usage("g")) == (0, 1) + + +# --- a stream that breaks after its first chunk -------------------------------------------------- + + +async def test_a_stream_that_breaks_after_content_counts_one_failure_and_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True)) + core: Final = make_core( + model_list=[deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + num_retries=0, + allowed_fails=0, + ) + + stream: Final = await open_chat(core) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + await collect_texts(stream) + await settle() + + assert host.deployments_called() == ["a"] + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert cooled(core, "a", "b") == ["a"] + # One request when the stream opened, one more for the failure. + assert await core.health().usage("g") == (0, 2) + + +async def test_a_stream_that_breaks_after_content_within_the_allowed_fails_does_not_cool_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("partial", error=rate_limit(message="mid stream"), logged=True)) + core: Final = make_core(model_list=group("a"), num_retries=0, allowed_fails=3) + + stream: Final = await open_chat(core) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + await collect_texts(stream) + await settle() + + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert cooled(core, "a") == [] + + +async def test_a_stream_that_breaks_before_content_and_falls_back_counts_one_failure_and_one_success( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + chat_stream("x", error=rate_limit(message="cut"), fail_after=1, leading_role_chunk=True, logged=True), + ) + host.answer("openai/b", chat_stream("B", usage=USAGE, logged=True)) + core: Final = make_core( + model_list=[deployment("a", "a", rpm=100), deployment("b", "b", rpm=100)], + fallbacks=[{"a": ["b"]}], + num_retries=0, + allowed_fails=3, + ) + + stream: Final = await open_chat(core, model="a") + assert await collect_texts(stream) == "B" + await settle() + + assert host.deployments_called() == ["a", "b"] + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert await core.health().usage("a") == (0, 2) + assert successes(core, "b") == 1 + assert await core.health().usage("b") == (12, 1) + + +async def test_a_stream_that_fails_on_its_first_chunk_counts_its_failure_once( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + chat_stream("x", error=rate_limit(), fail_after=0, logged=True), + chat_stream("ok", usage=USAGE, logged=True), + ) + core: Final = make_core(model_list=group("a", rpm=100), num_retries=1, allowed_fails=3) + + stream: Final = await open_chat(core) + assert await collect_texts(stream) == "ok" + await settle() + + assert host.deployments_called() == ["a", "a"] + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") == 1 + assert await core.health().usage("g") == (12, 2) + + +# --- Responses streams --------------------------------------------------------------------------- + + +def completed_event(input_tokens: int, output_tokens: int) -> ResponseCompletedEvent: + response: Final = ResponsesAPIResponse.model_construct( + id="resp", + background=None, + usage=ResponseAPIUsage( + input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=input_tokens + output_tokens + ), + ) + return ResponseCompletedEvent(type="response.completed", response=response) # pyright: ignore[reportArgumentType] # the enum accepts its value + + +async def test_a_consumed_responses_stream_counts_its_final_tokens_and_one_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + responses_stream( + Event("response.created"), + Event("response.output_text.delta"), + completed_event(4, 8), + logged=True, + total_tokens=12, + ), + ) + core: Final = make_core(model_list=group("a", rpm=100)) + + stream: Final = await open_responses(core) + assert (await collect_event_types(stream))[-1] == "response.completed" + await settle() + + assert await core.health().usage("g") == (12, 1) + assert successes(core, "a") == 1 + + +async def test_a_responses_stream_that_breaks_after_output_counts_one_failure_and_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + responses_stream( + Event("response.created"), + Event("response.output_text.delta", delta="partial"), + error=rate_limit(message="mid stream"), + logged=True, + ), + ) + core: Final = make_core( + model_list=[deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + num_retries=0, + allowed_fails=0, + ) + + stream: Final = await open_responses(core) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + await collect_event_types(stream) + await settle() + + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert cooled(core, "a", "b") == ["a"] + assert await core.health().usage("g") == (0, 2) + + +# --- /v1/messages streams ------------------------------------------------------------------------ + +MESSAGE_START_WITH_USAGE: Final = sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 3, "cache_read_input_tokens": 1, "output_tokens": 1}, + }, + }, +) +MESSAGE_DELTA_WITH_USAGE: Final = sse( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 8}}, +) + + +async def test_a_consumed_messages_stream_counts_its_final_tokens_and_one_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + anthropic_stream( + MESSAGE_START_WITH_USAGE, + text_delta("hello"), + # the delta frame split across two byte chunks + MESSAGE_DELTA_WITH_USAGE[:30], + MESSAGE_DELTA_WITH_USAGE[30:], + MESSAGE_STOP, + logged=True, + total_tokens=12, + ), + ) + core: Final = make_core(model_list=group("a", rpm=100)) + + stream: Final = await open_messages(core) + assert len(await collect_bytes(stream)) == 5 + await settle() + + assert await core.health().usage("g") == (12, 1) + assert successes(core, "a") == 1 + + +async def test_a_messages_stream_that_breaks_after_content_counts_one_failure_and_cools_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer( + "openai/a", + anthropic_stream( + MESSAGE_START_WITH_USAGE, text_delta("x"), error=rate_limit(message="mid stream"), logged=True + ), + ) + core: Final = make_core( + model_list=[deployment("g", "a", rpm=100, order=1), deployment("g", "b", order=2)], + num_retries=0, + allowed_fails=0, + ) + + stream: Final = await open_messages(core) + with pytest.raises(litellm.RateLimitError, match="mid stream"): + await collect_bytes(stream) + await settle() + + assert core.health().failures_this_minute("a") == 1 + assert successes(core, "a") is None + assert cooled(core, "a", "b") == ["a"] + assert await core.health().usage("g") == (0, 2) diff --git a/tests/unit/test_router/core/test_streaming_characterization_responses.py b/tests/unit/test_router/core/test_streaming_characterization_responses.py new file mode 100644 index 00000000000..4b601705324 --- /dev/null +++ b/tests/unit/test_router/core/test_streaming_characterization_responses.py @@ -0,0 +1,458 @@ +""" +Characterization of Responses API streaming through a router core: pre-output lifecycle events held +back while a fallback can still take over, mid-stream errors and their fallback, the continuation a +fallback receives after text was streamed, usage combined across a stream fallback, and the hidden +params the returned stream and its events carry. + +Scripted streams are ``BaseResponsesAPIStreamingIterator`` subclasses +(``streaming_helpers.responses_stream``) that raise a ``MidStreamFallbackError`` the way the +completion-bridge iterator does. +""" + +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.router_core import RouterCore +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + content_policy, + context_window, + deployment, + rate_limit, + server_error, +) +from tests.unit.test_router.core.streaming_helpers import ( + Event, + ScriptedResponsesStream, + collect_event_types, + drain_into, + responses_stream, +) + +MakeCore = Callable[..., RouterCore] +GROUPS_AB: Final = [deployment("a", "a"), deployment("b", "b")] +CREATED: Final = "response.created" +IN_PROGRESS: Final = "response.in_progress" +TEXT_DELTA: Final = "response.output_text.delta" +COMPLETED: Final = "response.completed" + + +def cut(generated_content: str = "", original: Exception | None = None) -> MidStreamFallbackError: + """The error the bridge iterator raises when its stream breaks (before output when nothing was generated).""" + return MidStreamFallbackError( + message="stream cut", + model="d1", + llm_provider="anthropic", + original_exception=original if original is not None else rate_limit(message="stream cut"), + generated_content=generated_content, + is_pre_first_chunk=not generated_content, + ) + + +def fallback_ok( + *events: object, opened: list[ScriptedResponsesStream] | None = None +) -> Callable[..., ScriptedResponsesStream]: + return responses_stream( + *(events or (Event(CREATED), Event(TEXT_DELTA), Event(COMPLETED))), + hidden_params={"model_id": "b-stream", "additional_headers": {"x-provider-request-id": "req-b"}}, + opened=opened, + ) + + +def build(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(**{"model_list": GROUPS_AB, "fallbacks": [{"a": ["b"]}], "num_retries": 0, **arguments}) + + +async def open_stream(core: RouterCore, **request: object) -> object: # kwargs-ok: request kwargs + return await core.aresponses(model="a", input="hi", stream=True, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def hidden(stream: object) -> Mapping[str, object]: + return getattr(stream, "_hidden_params") + + +def completed_event(input_tokens: int, output_tokens: int) -> ResponseCompletedEvent: + response = ResponsesAPIResponse.model_construct( + id="resp", + background=None, + usage=ResponseAPIUsage( + input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=input_tokens + output_tokens + ), + ) + return ResponseCompletedEvent(type=COMPLETED, response=response) # pyright: ignore[reportArgumentType] # the enum accepts its value + + +# --- the returned stream ------------------------------------------------------------------------ + + +async def test_a_streaming_response_is_returned_as_a_responses_streaming_iterator( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), Event(COMPLETED))) + core = build(make_core) + stream = await open_stream(core) + assert isinstance(stream, BaseResponsesAPIStreamingIterator) + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + assert hidden(stream)["model_id"] == "a" + + +async def test_the_returned_stream_latches_the_terminal_event_it_forwarded( + make_core: MakeCore, host: ScriptedHost +) -> None: + completed = Event(COMPLETED) + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), completed)) + core = build(make_core) + stream = await open_stream(core) + await collect_event_types(stream) + assert getattr(stream, "completed_response") is completed + + +async def test_lifecycle_events_are_held_until_output_and_then_forwarded_in_order( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(IN_PROGRESS), Event(TEXT_DELTA), Event(COMPLETED))) + core = build(make_core) + assert await collect_event_types(await open_stream(core)) == [CREATED, IN_PROGRESS, TEXT_DELTA, COMPLETED] + + +async def test_held_lifecycle_events_are_flushed_when_the_stream_ends_without_output( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(IN_PROGRESS))) + core = build(make_core) + assert await collect_event_types(await open_stream(core)) == [CREATED, IN_PROGRESS] + + +async def test_the_source_stream_is_closed_when_the_stream_ends(make_core: MakeCore, host: ScriptedHost) -> None: + opened: list[ScriptedResponsesStream] = [] + host.answer("openai/a", responses_stream(Event(CREATED), Event(COMPLETED), opened=opened)) + core = build(make_core) + await collect_event_types(await open_stream(core)) + assert [stream.closed for stream in opened] == [True] + + +# --- mid-stream errors before output ----------------------------------------------------------- + + +async def test_a_mid_stream_error_before_output_falls_back_and_drops_the_primarys_held_events( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(IN_PROGRESS), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core) + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + assert host.deployments_called() == ["a", "b"] + assert host.calls[1].kwargs["input"] == "hi" + assert host.calls[1].model_group == "b" + + +async def test_a_mid_stream_error_before_output_without_fallbacks_flushes_held_events_then_raises( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(IN_PROGRESS), error=cut())) + core = make_core(model_list=GROUPS_AB, num_retries=0) + stream = await open_stream(core) + seen: list[object] = [] + with pytest.raises(litellm.RateLimitError, match="stream cut") as raised: + await drain_into(stream, seen) + assert not isinstance(raised.value, MidStreamFallbackError) + assert [getattr(event, "type") for event in seen] == [CREATED, IN_PROGRESS] + assert host.deployments_called() == ["a"] + + +async def test_disable_fallbacks_raises_a_mid_stream_error_before_output( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core, disable_fallbacks=True) + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_when_the_fallback_fails_the_held_events_are_replayed_before_the_primary_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", server_error()) + core = build(make_core) + stream = await open_stream(core) + seen: list[object] = [] + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await drain_into(stream, seen) + assert [getattr(event, "type") for event in seen] == [CREATED] + assert host.deployments_called() == ["a", "b"] + + +async def test_more_than_two_hundred_lifecycle_events_are_forwarded_before_a_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(*(Event(IN_PROGRESS) for _ in range(201)), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + types = await collect_event_types(await open_stream(core)) + assert types == [*([IN_PROGRESS] * 201), CREATED, TEXT_DELTA, COMPLETED] + + +async def test_a_context_window_error_before_output_uses_the_context_window_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=context_window())) + host.answer("openai/b", fallback_ok()) + core = build(make_core, fallbacks=None, context_window_fallbacks=[{"a": ["b"]}]) + assert await collect_event_types(await open_stream(core)) == [CREATED, TEXT_DELTA, COMPLETED] + assert host.deployments_called() == ["a", "b"] + + +async def test_a_context_window_error_after_output_is_raised(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=context_window())) + host.answer("openai/b", fallback_ok()) + core = build(make_core, context_window_fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core) + with pytest.raises(litellm.ContextWindowExceededError): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_a_content_policy_cut_before_output_uses_the_content_policy_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut(original=content_policy()))) + host.answer("openai/b", fallback_ok()) + core = build(make_core, fallbacks=None, content_policy_fallbacks=[{"a": ["b"]}]) + assert await collect_event_types(await open_stream(core)) == [CREATED, TEXT_DELTA, COMPLETED] + assert host.deployments_called() == ["a", "b"] + + +# --- continuation after output ----------------------------------------------------------------- + + +async def test_after_text_the_fallback_receives_the_partial_output_and_a_request_to_continue( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel"))) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + await collect_event_types(await open_stream(core)) + sent = host.calls[1].kwargs["input"] + assert isinstance(sent, list) + assert sent[0] == {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]} + assert sent[1] == {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "hel"}]} + assert sent[2]["role"] == "user" + assert 'It ended with "hel"' in sent[2]["content"][0]["text"] + + +async def test_after_text_the_output_ceiling_is_reduced_by_the_emitted_tokens( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel"))) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + await collect_event_types(await open_stream(core, max_output_tokens=100)) + assert [call.kwargs["max_output_tokens"] for call in host.calls] == [100, 99] + + +async def test_after_text_a_spent_output_ceiling_raises_the_primary_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hello there friend"))) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core, max_output_tokens=1) + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_after_text_a_content_policy_cut_raises_instead_of_continuing( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel", content_policy()))) + host.answer("openai/b", fallback_ok()) + core = build(make_core, content_policy_fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core) + with pytest.raises(litellm.ContentPolicyViolationError): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_after_a_tool_call_began_the_primary_error_is_raised(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer( + "openai/a", + responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel"), started_tool_call=True), + ) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_after_text_a_json_schema_output_format_raises_the_primary_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel"))) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core, text={"format": {"type": "json_schema", "name": "x", "schema": {}}}) + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await collect_event_types(stream) + assert host.deployments_called() == ["a"] + + +async def test_after_text_the_primarys_events_stay_and_the_fallbacks_follow( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("hel"))) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + types = await collect_event_types(await open_stream(core)) + assert types[:2] == [CREATED, TEXT_DELTA] + assert types[-1] == COMPLETED + assert host.deployments_called() == ["a", "b"] + + +# --- usage across a stream fallback -------------------------------------------------------------- + + +async def test_the_fallbacks_completed_usage_adds_the_usage_the_primary_latched( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut(), completed_response=completed_event(3, 4))) + host.answer("openai/b", fallback_ok(Event(CREATED), completed_event(10, 20))) + core = build(make_core) + stream = await open_stream(core) + events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + usage = events[-1].response.usage + assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (13, 24, 37) + + +async def test_without_primary_usage_the_fallbacks_completed_usage_is_unchanged( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", fallback_ok(Event(CREATED), completed_event(10, 20))) + core = build(make_core) + stream = await open_stream(core) + events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + usage = events[-1].response.usage + assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (10, 20, 30) + + +# --- sequence numbers across a stream fallback --------------------------------------------------- + + +def delta(text: str, sequence_number: int | None = None) -> OutputTextDeltaEvent: + event: Final = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="msg_1", output_index=0, content_index=0, delta=text + ) + return event if sequence_number is None else event.model_copy(update={"sequence_number": sequence_number}) + + +async def numbered(stream: object) -> list[tuple[str, object]]: # mutable-ok: a fresh list + return [(event.delta, event.get("sequence_number")) async for event in stream] # pyright: ignore[reportGeneralTypeIssues, reportUnknownMemberType, reportUnknownVariableType] # an async stream of deltas + + +async def test_sequence_numbers_keep_increasing_across_a_stream_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(delta("a", 0), delta("b", 1), error=cut("ab"))) + host.answer("openai/b", fallback_ok(delta("c", 0), delta("d", 1))) + core = build(make_core) + assert await numbered(await open_stream(core)) == [("a", 0), ("b", 1), ("c", 2), ("d", 3)] + + +async def test_a_stream_fallback_continues_above_the_highest_number_the_primary_sent( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(delta("a", 0), delta("b", 3), delta("c", 1), error=cut("abc"))) + host.answer("openai/b", fallback_ok(delta("d", 0))) + core = build(make_core) + assert await numbered(await open_stream(core)) == [("a", 0), ("b", 3), ("c", 1), ("d", 4)] + + +async def test_a_stream_fallback_keeps_higher_numbers_and_passes_unnumbered_events( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(delta("a", 0), delta("b", 1), error=cut("ab"))) + host.answer("openai/b", fallback_ok(delta("c", 7), delta("d"), delta("e", 8))) + core = build(make_core) + assert await numbered(await open_stream(core)) == [("a", 0), ("b", 1), ("c", 7), ("d", None), ("e", 8)] + + +async def test_sequence_numbers_pass_through_untouched_without_a_fallback( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(delta("a", 0), delta("b", 1), delta("c", 1), delta("d", 0))) + core = build(make_core) + assert await numbered(await open_stream(core)) == [("a", 0), ("b", 1), ("c", 1), ("d", 0)] + + +# --- hidden params after a stream fallback ------------------------------------------------------- + + +async def test_after_a_stream_fallback_the_stream_carries_the_fallbacks_hidden_params_and_headers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core) + await collect_event_types(stream) + assert hidden(stream)["model_id"] == "b-stream" + assert hidden(stream)["additional_headers"] == { + "x-provider-request-id": "req-b", + "x-litellm-model-group": "b", + "x-litellm-attempted-retries": 0, + "x-litellm-attempted-fallbacks": 1, + } + + +async def test_after_a_stream_fallback_each_fallback_event_carries_the_fallbacks_hidden_params( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core) + stream = await open_stream(core) + events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + assert all(event._hidden_params["model_id"] == "b-stream" for event in events) + assert all(event._hidden_params["additional_headers"]["x-litellm-model-group"] == "b" for event in events) + + +async def test_when_the_stream_fallback_fails_the_stream_still_reports_the_primary( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", server_error()) + core = build(make_core) + stream = await open_stream(core) + with pytest.raises(litellm.RateLimitError): + await collect_event_types(stream) + assert hidden(stream)["model_id"] == "a" + assert hidden(stream)["additional_headers"]["x-litellm-model-group"] == "a" + + +async def test_the_primary_and_fallback_streams_are_closed(make_core: MakeCore, host: ScriptedHost) -> None: + opened: list[ScriptedResponsesStream] = [] + host.answer("openai/a", responses_stream(Event(CREATED), error=cut(), opened=opened)) + host.answer("openai/b", fallback_ok(opened=opened)) + core = build(make_core) + await collect_event_types(await open_stream(core)) + assert [stream.closed for stream in opened] == [True, True] diff --git a/tests/unit/test_router/core/test_usage_and_failure_counting_parity.py b/tests/unit/test_router/core/test_usage_and_failure_counting_parity.py new file mode 100644 index 00000000000..674f5496b84 --- /dev/null +++ b/tests/unit/test_router/core/test_usage_and_failure_counting_parity.py @@ -0,0 +1,455 @@ +""" +Parity of the Rust core with ``router_python_core.py`` on what a call counts: which deployments track +per-minute usage, when a failure's counters are written on a logging object the attempts share, the +exits of the failed fallback hop's cooldown, the reads and writes that reach Redis, and the keys a +wildcard stream counts under. +""" + +import asyncio +from collections.abc import Callable, Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.router_core import RouterCore +from litellm.router_utils import cooldown_handlers +from litellm.router_utils.cooldown_handlers import mark_deployment_selection_failure +from tests.unit.test_router.core.endpoints_helpers import EndpointHost +from tests.unit.test_router.core.retries_health_helpers import ( + MESSAGES, + ErrorBuilder, + MakeCore, + attempt, + attempt_sync, + cooled_down, + fresh, + not_found, + plain_error, + rate_limited, +) +from tests.unit.test_router.core.scripted_host import Call, deployment, server_error, timeout +from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts, request_logging + +USAGE: Final = {"prompt_tokens": 4, "completion_tokens": 8, "total_tokens": 12} +PER_USER_COPILOT: Final = { + "litellm_credential_name": "ghc", + "github_copilot_auth_type": "per_user_oauth", +} + + +async def settle() -> None: + """Let the recording tasks a stream's end scheduled run.""" + for _ in range(20): + await asyncio.sleep(0) + + +class SharedLoggingHost(EndpointHost): + """``EndpointHost`` that logs on a shared logging object as litellm does: every attempt stamps its + marker, and only the first failure runs the failure callbacks. ``_arealtime`` answers ``None``.""" + + __slots__ = ("timeout_cause",) + + def __init__(self) -> None: + super().__init__() + self.timeout_cause: BaseException | None = None + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLogging): + logging_obj.model_call_details["current_attempt_api_call_start_time"] = object() + return await super()._answer_async(op, args, kwargs) + + def _finish(self, op: str, call: Call, outcome: object) -> tuple[BaseException | None, object]: + error, response = super()._finish(op, call, outcome) + if isinstance(error, litellm.Timeout) and self.timeout_cause is not None: + error.__cause__ = self.timeout_cause # rebind-ok: a scripted error is rebuilt, which drops its cause + return error, None if op == "_arealtime" else response + + async def log_async(self, call: Call, error: BaseException | None, response: object) -> None: + logging_obj: Final = call.kwargs.get("litellm_logging_obj") + if error is None or not isinstance(logging_obj, LiteLLMLogging): + await super().log_async(call, error, response) + return + details: Final = logging_obj.model_call_details + if details.get("has_logged_async_failure") is True: + return + details["has_logged_async_failure"] = True + details["async_failure_logged_attempt"] = details.get("current_attempt_api_call_start_time") + await super().log_async(call, error, response) + + +class RedisDouble: + """The shared tier the routing caches talk to, holding the totals every pod counted.""" + + def __init__(self) -> None: + self.totals: Final[dict[str, float]] = {} # mutable-ok: the double's own store + self.gets: Final[list[str]] = [] # mutable-ok: what was read + self.batches: Final[list[list[str]]] = [] # mutable-ok: what was read + + async def async_get_cache(self, key: str, **_: object) -> object: + self.gets.append(key) + return self.totals.get(key) + + def get_cache(self, key: str, **_: object) -> object: + self.gets.append(key) + return self.totals.get(key) + + async def async_batch_get_cache( + self, key_list: list[str], **_: object + ) -> dict[str, object]: # mutable-ok: as RedisCache + self.batches.append(list(key_list)) + return {key: self.totals.get(key) for key in key_list} + + def batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, object]: # mutable-ok: as RedisCache + self.batches.append(list(key_list)) + return {key: self.totals.get(key) for key in key_list} + + async def async_increment(self, key: str, value: float, **_: object) -> float: + self.totals[key] = self.totals.get(key, 0) + value + return self.totals[key] + + async def async_increment_pipeline( + self, increment_list: list[Mapping[str, object]], **_: object + ) -> list[float]: # mutable-ok: as RedisCache + return [ + await self.async_increment(str(row["key"]), float(str(row["increment_value"]))) for row in increment_list + ] + + async def async_set_cache(self, key: str, value: object, **_: object) -> None: + return None + + def set_cache(self, key: str, value: object, **_: object) -> None: + return None + + +@pytest.fixture +def host() -> SharedLoggingHost: + return SharedLoggingHost() + + +def minute() -> str: + return datetime.now(timezone.utc).strftime("%H-%M") + + +def chain(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: router arguments + return make_core( + model_list=[deployment("a", "a"), deployment("b", "b"), deployment("c", "c")], + fallbacks=[{"a": ["b"]}, {"b": ["c"]}], + num_retries=0, + **arguments, + ) + + +def with_redis(core: RouterCore) -> RedisDouble: + redis: Final = RedisDouble() + core.health().store.redis_cache = redis # pyright: ignore[reportAttributeAccessIssue] # a double + return redis + + +# --- PV4: which deployments track usage ---------------------------------------------------------- + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + (deployment("g", "d1"), (None, None)), + (deployment("g", "d1", rpm=100), (10, 1)), + (deployment("g", "d1", itpm=100), (10, 1)), + (deployment("g", "d1", model_info={"rpm": 100}), (10, 1)), + (deployment("g", "d1", model_info={"tpm": 100}), (10, 1)), + (deployment("g", "d1", model_info={"otpm": 100}), (10, 1)), + ({**deployment("g", "d1"), "tpm": 100}, (10, 1)), + (deployment("g", "d1", model="gemini/gemini-3.1-flash-lite-preview"), (10, 1)), + ], + ids=[ + "no-limit", + "params-rpm", + "params-itpm", + "model-info-rpm", + "model-info-tpm", + "model-info-otpm", + "top-level-tpm", + "catalog-rpm", + ], +) +async def test_usage_is_counted_for_a_limit_anywhere_the_python_core_reads_one( + make_core: MakeCore, entry: dict[str, object], expected: tuple[int | None, int | None] +) -> None: + core: Final = make_core(model_list=[entry]) + + # The scripted host logs a sync call's success apart from the usage `make_call` writes. + _ = attempt_sync(core, "g") + + assert await core.health().usage("g") == expected + + +async def test_a_stream_counts_usage_for_a_limit_in_model_info(make_core: MakeCore, host: SharedLoggingHost) -> None: + host.answer("openai/d1", chat_stream("hello", usage=USAGE, logged=True)) + core: Final = make_core(model_list=[deployment("g", "d1", model_info={"rpm": 100})]) + + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + _ = await collect_texts(stream) + await settle() + + assert await core.health().usage("g") == (12, 1) + + +# --- realtime ------------------------------------------------------------------------------------ + + +@pytest.mark.parametrize("limit", ["tpm", "rpm", "itpm", "otpm"]) +async def test_a_realtime_session_counts_no_request_against_a_limited_deployment( + make_core: MakeCore, limit: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", **{limit: 100})], num_retries=0) + + await core._arealtime(model="g") # pyright: ignore[reportAttributeAccessIssue] # every core serves it + + assert await core.health().usage("g") == (None, None) + + +# --- P3: what a shared logging object lets through ------------------------------------------------- + + +async def test_only_the_first_failure_on_a_shared_logging_object_counts_a_request( + make_core: MakeCore, host: SharedLoggingHost +) -> None: + core: Final = chain(make_core) + host.answer("*", fresh(server_error)) + + _ = await attempt(core, "a", litellm_logging_obj=request_logging()) + + assert await core.health().usage("a") == (None, 1) + assert await core.health().usage("b") == (None, None) + assert await core.health().usage("c") == (None, None) + + +async def test_every_failure_counts_a_request_when_each_call_logs_on_its_own( + make_core: MakeCore, host: SharedLoggingHost +) -> None: + core: Final = chain(make_core) + host.answer("*", fresh(server_error)) + + _ = await attempt(core, "a") + + assert await core.health().usage("a") == (None, 1) + assert await core.health().usage("b") == (None, 1) + assert await core.health().usage("c") == (None, 1) + + +async def test_a_blocked_retry_in_the_same_group_records_nothing(make_core: MakeCore, host: SharedLoggingHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=1, allowed_fails=5) + host.answer("*", fresh(server_error)) + + _ = await attempt(core, "g", litellm_logging_obj=request_logging()) + + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 1 + + +def test_a_sync_failure_counts_no_request(make_core: MakeCore, host: SharedLoggingHost) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) + host.answer("*", fresh(server_error)) + + with pytest.raises(litellm.InternalServerError): + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # see above + + assert core.health().failures_this_minute("d1") == 1 + assert core.health().store.get_cache(f"global_router:d1:openai/d1:rpm:{minute()}", local_only=True) is None + + +# --- failure counting ------------------------------------------------------------------------------ + + +async def test_a_failure_raised_before_the_litellm_call_records_nothing( + make_core: MakeCore, host: SharedLoggingHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", model="github_copilot/gpt-4o", **PER_USER_COPILOT)], num_retries=0 + ) + + raised: Final = await attempt(core, "g", litellm_credential_name="another") + + assert isinstance(raised, litellm.BadRequestError) + assert host.calls == [] + assert core.health().failures_this_minute("d1") == 0 + assert await core.health().usage("g") == (None, None) + + +async def test_an_exception_with_no_status_is_counted_and_never_cooled( + make_core: MakeCore, host: SharedLoggingHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], num_retries=0, allowed_fails=0, cooldown_time=60 + ) + host.answer("*", fresh(plain_error)) + + _ = await attempt(core, "g") + + assert cooled_down(core, ["a", "b"]) == {} + assert core.health().failures_this_minute("a") + core.health().failures_this_minute("b") == 1 + + +# --- _trigger_cooldown_for_failed_deployment --------------------------------------------------------- + + +def selection_failure() -> Exception: + error: Final = rate_limited() + mark_deployment_selection_failure(error) + return error + + +@pytest.mark.parametrize( + ("builder", "stalled", "counted"), + [(server_error, False, 1), (selection_failure, False, 1), (timeout, True, 0), (timeout, False, 1)], + ids=["server-error", "selection-tag-not-exempt", "stream-stall-exempt", "plain-timeout"], +) +async def test_a_failed_hop_past_the_first_logged_failure_takes_the_fallback_paths_exits( + make_core: MakeCore, host: SharedLoggingHost, builder: ErrorBuilder, stalled: bool, counted: int +) -> None: + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b")], fallbacks=[{"a": ["b"]}], num_retries=0 + ) + host.answer("openai/a", fresh(server_error)) + host.answer("openai/b", fresh(builder)) + if stalled: + host.timeout_cause = MidStreamFallbackError(message="stalled", model="b", llm_provider="openai") + + _ = await attempt(core, "a", litellm_logging_obj=request_logging()) + + assert core.health().failures_this_minute("b") == counted + assert await core.health().usage("b") == (None, None) + + +@pytest.mark.parametrize(("logged", "counted"), [(True, 0), (False, 1)], ids=["generic", "chat"]) +async def test_a_failed_hops_404_is_the_callers_only_on_a_generic_call( + make_core: MakeCore, host: SharedLoggingHost, logged: bool, counted: int +) -> None: + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b")], fallbacks=[{"a": ["b"]}], num_retries=0 + ) + host.answer("openai/a", fresh(server_error)) + host.answer("openai/b", fresh(not_found)) + + logging_obj: Final = request_logging() + if logged: + with pytest.raises(litellm.InternalServerError): + await core.agenerate_content(model="a", contents=[], litellm_logging_obj=logging_obj) # pyright: ignore[reportAttributeAccessIssue] # see above + else: + _ = await attempt(core, "a", litellm_logging_obj=logging_obj) + + assert core.health().failures_this_minute("b") == counted + + +@pytest.mark.parametrize( + ("params", "router_cooldown", "error", "expected"), + [ + pytest.param({}, 7, server_error, 7, id="router-int"), + pytest.param({}, 7.0, server_error, 7.0, id="router-float"), + pytest.param({"cooldown_time": 7}, 7.0, server_error, 7, id="deployment-int-over-router-float"), + pytest.param({"cooldown_time": 7.0}, 7, server_error, 7.0, id="deployment-float-over-router-int"), + pytest.param({}, 9.0, lambda: rate_limited("9"), 9, id="header-int-over-router-float"), + ], +) +async def test_a_failed_hop_fires_the_number_python_chose_as_it_was_written( + make_core: MakeCore, + host: SharedLoggingHost, + monkeypatch: pytest.MonkeyPatch, + params: Mapping[str, object], + router_cooldown: float, + error: Callable[[], Exception], + expected: float, +) -> None: + fired: Final[list[tuple[str, object]]] = [] # mutable-ok: appended as they fire + + async def record(ctx: object, deployment_id: str, exception_status: object, cooldown_time: object) -> None: + fired.append((deployment_id, cooldown_time)) + + monkeypatch.setattr(cooldown_handlers, "router_cooldown_event_callback", record) + core: Final = make_core( + model_list=[deployment("a", "a"), deployment("b", "b", **params), deployment("b", "c", **params)], + fallbacks=[{"a": ["b"]}], + num_retries=0, + allowed_fails=0, + cooldown_time=router_cooldown, + ) + host.answer("openai/a", fresh(server_error)) + host.answer("openai/b", fresh(error)) + host.answer("openai/c", fresh(error)) + + _ = await attempt(core, "a", litellm_logging_obj=request_logging()) + for _ in range(20): + await asyncio.sleep(0) + + assert [repr(seconds) for name, seconds in fired if name != "a"] == [repr(expected)] + + +# --- P6, PV7: what reaches Redis --------------------------------------------------------------------- + + +async def test_cooldowns_are_read_as_one_batch_of_every_deployment_with_no_key_of_their_own( + make_core: MakeCore, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("other", "c")], + num_retries=0, + disable_cooldowns=True, + ) + redis: Final = with_redis(core) + + _ = await attempt(core, "g") + + assert [key for key in redis.gets if key.endswith(":cooldown")] == [] + assert redis.batches[0] == ["deployment:a:cooldown", "deployment:b:cooldown", "deployment:c:cooldown"] + + +async def test_a_second_request_within_the_read_interval_reads_no_cooldown_from_redis(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0) + redis: Final = with_redis(core) + + _ = await attempt(core, "g") + _ = await attempt(core, "g") + + assert len([batch for batch in redis.batches if batch[0].endswith(":cooldown")]) == 1 + + +async def test_a_usage_write_leaves_the_shared_total_in_memory(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", rpm=1000000)], num_retries=0) + redis: Final = with_redis(core) + rpm_key: Final = f"global_router:d1:openai/d1:rpm:{minute()}" + redis.totals[rpm_key] = 100000 + + _ = attempt_sync(core, "g") + + assert core.health().store.get_cache(rpm_key, local_only=True) == 100001 + + +async def test_a_usage_read_on_a_pod_sees_what_the_other_pods_counted(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "d1", rpm=1000000)], num_retries=0) + redis: Final = with_redis(core) + redis.totals[f"global_router:d1:openai/d1:rpm:{minute()}"] = 41 + + _ = attempt_sync(core, "g") + + assert (await core.health().usage("g"))[1] == 42 + + +# --- wildcard streams -------------------------------------------------------------------------------- + + +async def test_a_wildcard_stream_counts_its_tokens_under_the_filled_model( + make_core: MakeCore, host: SharedLoggingHost +) -> None: + host.answer("openai/gpt-4o", chat_stream("hello", usage=USAGE, logged=True)) + core: Final = make_core(model_list=[deployment("openai/*", "w", model="openai/*", rpm=100)]) + + stream: Final = await core.acompletion(model="openai/gpt-4o", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # see above + _ = await collect_texts(stream) + await settle() + + store: Final = core.health().store + assert store.get_cache(f"global_router:w:openai/gpt-4o:tpm:{minute()}", local_only=True) == 12 + assert store.get_cache(f"global_router:w:openai/*:tpm:{minute()}", local_only=True) is None diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index 819c6b9d4f5..7c2658d3d0d 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -509,6 +509,36 @@ class TestModelRateLimitingCheckIOTokens: assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == model_max_output + @pytest.mark.asyncio + async def test_otpm_reservation_uses_each_deployments_output_limit(self) -> None: + backend_model: Final = "openai/gpt-4o-mini" + catalog_max_output: Final = litellm.get_model_info(model=backend_model)["max_output_tokens"] + assert isinstance(catalog_max_output, int) + otpm_limit: Final = int(max(catalog_max_output, 987) / NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) + 1 + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + + async def _reserved(deployment_id: str, model_info: dict[str, object]) -> int: + metadata: Final[dict[str, object]] = {} + request_kwargs: Final[dict[str, object]] = { + "messages": [{"role": "user", "content": "hello"}], + "metadata": metadata, + } + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") + deployment: Final = { + "litellm_params": {"model": backend_model, "otpm": otpm_limit}, + "model_info": {"id": deployment_id, **model_info}, + } + await check.async_pre_call_check(deployment) + reservation: Final = metadata.get(OTPM_RESERVED_KEY) + assert isinstance(reservation, int) + return reservation + + plain: Final = await _reserved("io-plain-gpt-4o-mini", {}) + override: Final = await _reserved("io-overridden-gpt-4o-mini", {"max_output_tokens": 987}) + + assert plain == catalog_max_output + assert override == 987 + @pytest.mark.asyncio @pytest.mark.parametrize("use_sync_check", [False, True]) async def test_base_model_does_not_change_the_default_otpm_reservation(self, use_sync_check: bool): diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4f35efdfd6e..a21777fbee6 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -5,6 +5,7 @@ import functools import gc import itertools import io +import base64 import json import linecache import logging @@ -12,7 +13,7 @@ import os import sys import threading import warnings -from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path from types import CodeType, FrameType, MappingProxyType, SimpleNamespace @@ -42,6 +43,7 @@ from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( + AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES, SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, ) from litellm.llms.bedrock.common_utils import BedrockError @@ -61,7 +63,6 @@ from litellm.router import ( _responses_stream_holds_event, _without_line_breaks, _search_tool_support, - Span, ) from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request from litellm.router_strategy import simple_shuffle @@ -81,6 +82,7 @@ from litellm.router_utils.fallback_event_handlers import ( from litellm.router_utils.prompt_caching_cache import PromptCachingCache from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.scheduler import FlowItem +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest from litellm.router_api import RoutingContext as RouterContext from litellm.types.router import ( @@ -3651,14 +3653,28 @@ class _CutSSE(httpx.AsyncByteStream): raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") -_BACKUP_SSE: Final = "".join( - f"data: {json.dumps(chunk)}\n\n" - for chunk in ( - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]}, - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}}, +_BACKUP_SSE: Final = ( + "".join( + f"data: {json.dumps(chunk)}\n\n" + for chunk in ( + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None} + ], + }, + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}, + }, + ) ) ) + "data: [DONE]\n\n" @@ -3844,6 +3860,255 @@ async def test_continued_stream_sharing_the_proxy_logging_object_bills_the_cut_t ) +_AGENTIC_FIRST_CALL_USAGE: Final = {"prompt_tokens": 50, "completion_tokens": 7, "total_tokens": 57} +_AGENTIC_FIRST_CALL_BODY: Final = { + "id": "chatcmpl-first", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_abc", "type": "function", "function": {"name": "litellm_code_execution", "arguments": "{}"}} + ], + }, + } + ], + "usage": _AGENTIC_FIRST_CALL_USAGE, +} + + +_CRASH_THE_PLAN: Final = "crash the plan" + + +class _ScriptedAgenticFollowUps(CustomLogger): + def __init__(self, follow_ups: tuple[str, ...]) -> None: + super().__init__() + self._follow_ups: Final = follow_ups + self.plans_built = 0 + self.shared_call_succeeded: Final = asyncio.Event() + + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + assert isinstance(response, litellm.ModelResponse) + return bool(response.choices[0].message.tool_calls), {"tool_calls": [{"id": "call_abc"}]} + + async def async_build_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + follow_up: Final = self._follow_ups[min(self.plans_built, len(self._follow_ups) - 1)] + self.plans_built += 1 + if follow_up == _CRASH_THE_PLAN: + raise ValueError("the agentic plan build crashed") + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "what is 6*7?"}, {"role": "assistant", "content": "42"}], + kwargs={"mock_response": follow_up}, + ), + ) + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + if kwargs.get("litellm_call_id") == "shared-agentic-call": + self.shared_call_succeeded.set() + + +_AGENTIC_BACKUP_USAGE: Final = {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23} +_AGENTIC_BACKUP_BODY: Final = { + "id": "chatcmpl-backup", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "42"}}], + "usage": _AGENTIC_BACKUP_USAGE, +} +_AGENTIC_RATE_LIMITED: Final = { + "error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"} +} + + +async def _route_agentic_request( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str = "m", + first_call_replies: tuple[httpx.Response, ...] = (), + fallbacks: tuple[str, ...] = (), +) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + hook: Final = _ScriptedAgenticFollowUps(follow_ups) + monkeypatch.setattr(litellm, "callbacks", [hook]) + first_calls: Final = ( + respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=list(first_call_replies)) + if first_call_replies + else respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + ) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_BACKUP_BODY) + ) + router: Final = Router( + model_list=[ + {"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-test", + "api_base": "https://backup.test/v1", + }, + }, + ], + num_retries=2, + retry_after=0, + fallbacks=[{"m": list(fallbacks)}] if fallbacks else [], + ) + messages: Final = [{"role": "user", "content": "what is 6*7?"}] + shared_logging, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model=model, + messages=messages, + litellm_call_id="shared-agentic-call", + ) + try: + outcome: object = await router.acompletion( + model=model, messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging + ) + except (litellm.RateLimitError, litellm.APIConnectionError) as error: + outcome = error + return outcome, shared_logging, first_calls, hook + + +def _gpt_4o_mini_cost(usage: Mapping[str, int]) -> float: + return sum( + litellm.cost_per_token( + model="gpt-4o-mini", prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"] + ) + ) + + +@pytest.mark.asyncio +async def test_agentic_request_whose_follow_up_fails_is_not_retried_and_bills_its_one_first_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + from litellm.proxy.utils import _failure_fields_to_lift + + outcome, shared_logging, first_calls, _ = await _route_agentic_request( + monkeypatch, respx_mock, ("litellm.RateLimitError",) + ) + + lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging}) + assert isinstance(outcome, litellm.RateLimitError) + assert outcome.status_code == 429 + assert first_calls.call_count == 1 + assert lifted["combined_usage_object"].prompt_tokens == _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert lifted["combined_usage_object"].completion_tokens == _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + assert lifted["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE)) + + +@pytest.mark.asyncio +async def test_agentic_request_whose_plan_build_crashes_after_the_first_call_is_not_retried( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + outcome, _, first_calls, hook = await _route_agentic_request(monkeypatch, respx_mock, (_CRASH_THE_PLAN,)) + + assert isinstance(outcome, litellm.APIConnectionError) + assert first_calls.call_count == 1 + assert hook.plans_built == 1 + + +@pytest.mark.asyncio +async def test_agentic_request_whose_first_call_fails_is_still_retried( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + outcome, _, first_calls, hook = await _route_agentic_request( + monkeypatch, + respx_mock, + ("42",), + first_call_replies=( + httpx.Response(429, json=_AGENTIC_RATE_LIMITED), + httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY), + ), + ) + + assert isinstance(outcome, litellm.ModelResponse) + assert first_calls.call_count == 2 + assert hook.plans_built == 1 + + +async def _successful_agentic_request_billing( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str, + fallbacks: tuple[str, ...] = (), +) -> tuple[Mapping[str, object], Mapping[str, str | None]]: + from litellm.proxy.common_request_processing import cost_headers + + outcome, shared_logging, _, hook = await _route_agentic_request( + monkeypatch, respx_mock, follow_ups, model=model, fallbacks=fallbacks + ) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) + assert isinstance(outcome, litellm.ModelResponse) + success: Final = shared_logging.model_call_details["standard_logging_object"] + return success, cost_headers(success["response_cost"], shared_logging) + + +@pytest.mark.asyncio +async def test_fallback_after_a_failed_agentic_follow_up_bills_the_primarys_first_call_on_the_success_row( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + first_try, first_try_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("42",), model="backup" + ) + recovered, recovered_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("litellm.RateLimitError",), model="m", fallbacks=("backup",) + ) + + first_call_cost: Final = _gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE) + assert first_try["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_BACKUP_USAGE)) + assert "failed_attempts_cost" not in first_try["cost_breakdown"].get("additional_costs", {}) + assert recovered["response_cost"] == pytest.approx(first_try["response_cost"] + first_call_cost) + assert recovered["prompt_tokens"] == first_try["prompt_tokens"] + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert ( + recovered["completion_tokens"] + == first_try["completion_tokens"] + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + ) + assert recovered["cost_breakdown"]["total_cost"] == pytest.approx(recovered["response_cost"]) + assert recovered["cost_breakdown"]["additional_costs"]["failed_attempts_cost"] == pytest.approx(first_call_cost) + assert float(recovered_headers["x-litellm-response-cost"]) == pytest.approx(recovered["response_cost"]) + assert {key: value for key, value in recovered_headers.items() if key != "x-litellm-response-cost"} == { + key: value for key, value in first_try_headers.items() if key != "x-litellm-response-cost" + } + + def _responses_sse(*events: Mapping[str, object]) -> str: return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) @@ -6397,13 +6662,14 @@ def _make_router_with_fallback(primary="gpt-4", secondary="gpt-3.5-turbo"): ) -class _InjectedFallbackRouter(Router): - def __init__(self, fallback_response: object) -> None: - super().__init__(model_list=[], fallbacks=[{"primary": ["fallback"]}]) - self._fallback_response: Final = fallback_response +# The Rust core walks fallbacks and runs pre-routing in Rust, past the members these tests patch. +_PYTHON_CORE_ONLY: Final = pytest.mark.usefixtures("python_router_core") - async def async_function_with_fallbacks_common_utils( - self, + +def _injected_fallback_router(fallback_response: object) -> Router: + router: Final = Router(model_list=[], fallbacks=[{"primary": ["fallback"]}]) + + async def inject_fallback( e: Exception, disable_fallbacks: bool | None, fallbacks: list | None, @@ -6414,7 +6680,10 @@ class _InjectedFallbackRouter(Router): kwargs: dict[str, object], include_fallback_errors: bool = False, ) -> object: - return self._fallback_response + return fallback_response + + router.async_function_with_fallbacks_common_utils = inject_fallback + return router @pytest.mark.asyncio @@ -6777,6 +7046,7 @@ async def test_aresponses_streaming_content_policy_error_event_routes_to_content @pytest.mark.asyncio +@_PYTHON_CORE_ONLY async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback(): from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents @@ -6792,7 +7062,7 @@ async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback(): from litellm.exceptions import MidStreamFallbackError - router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "c"), delta(1, "d")])) + router: Final = _injected_fallback_router(_AsyncList([delta(0, "c"), delta(1, "d")])) primary: Final = _make_responses_iterator( chunks=[delta(0, "a"), delta(1, "b")], error=MidStreamFallbackError( @@ -6816,6 +7086,7 @@ async def test_aresponses_stream_keeps_numbering_after_a_mid_stream_fallback(): @pytest.mark.asyncio +@_PYTHON_CORE_ONLY async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumbered_events(): from litellm.exceptions import MidStreamFallbackError from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents @@ -6828,7 +7099,7 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb def numbered(sequence_number: int, text: str) -> OutputTextDeltaEvent: return delta(text).model_copy(update={"sequence_number": sequence_number}) - router: Final = _InjectedFallbackRouter(_AsyncList([numbered(7, "c"), delta("d"), numbered(8, "e")])) + router: Final = _injected_fallback_router(_AsyncList([numbered(7, "c"), delta("d"), numbered(8, "e")])) primary: Final = _make_responses_iterator( chunks=[numbered(0, "a"), numbered(1, "b")], error=MidStreamFallbackError( @@ -6857,6 +7128,7 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb @pytest.mark.asyncio +@_PYTHON_CORE_ONLY async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchanged(): from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents @@ -6870,7 +7142,7 @@ async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchan sequence_number=sequence_number, ) - router: Final = _InjectedFallbackRouter(_AsyncList([])) + router: Final = _injected_fallback_router(_AsyncList([])) primary: Final = _make_responses_iterator( chunks=[delta(0, "a"), delta(1, "b"), delta(1, "c"), delta(0, "d")], model="primary" ) @@ -6885,6 +7157,7 @@ async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchan @pytest.mark.asyncio +@_PYTHON_CORE_ONLY async def test_aresponses_stream_fallback_continues_above_the_highest_number_the_primary_sent(): from litellm.exceptions import MidStreamFallbackError from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents @@ -6899,7 +7172,7 @@ async def test_aresponses_stream_fallback_continues_above_the_highest_number_the sequence_number=sequence_number, ) - router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "d")])) + router: Final = _injected_fallback_router(_AsyncList([delta(0, "d")])) primary: Final = _make_responses_iterator( chunks=[delta(0, "a"), delta(3, "b"), delta(1, "c")], error=MidStreamFallbackError( @@ -6968,6 +7241,7 @@ async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_ ], ids=["plain-fallback", "complexity-tier-fallback"], ) +@_PYTHON_CORE_ONLY async def test_aresponses_streaming_iterator_replaces_complexity_headers_before_fallback_output( fallback_headers: dict[str, str], ) -> None: @@ -6991,7 +7265,7 @@ async def test_aresponses_streaming_iterator_replaces_complexity_headers_before_ "model_id": "fallback-deployment", "additional_headers": fallback_headers, } - router: Final = _InjectedFallbackRouter(fallback) + router: Final = _injected_fallback_router(fallback) wrapped: Final = await router._aresponses_streaming_iterator( response=source, @@ -8174,6 +8448,77 @@ def test_pre_call_checks_count_a_group_mixing_in_another_vendor_by_its_own_name( ) +_EMBEDDING_CONTEXT_WINDOW: Final = 8_192 +_EMBEDDING_IMAGE_INPUT: Final = f"data:image/png;base64,{base64.b64encode(bytes(range(256)) * 1024).decode()}" + + +def _embedding_context_router(backend: str, window: int = _EMBEDDING_CONTEXT_WINDOW) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "embedder", + "litellm_params": {"model": backend, "api_key": "sk-test"}, + "model_info": {"max_input_tokens": window}, + } + ], + enable_pre_call_checks=True, + ) + + +async def test_context_window_pre_call_check_counts_an_embedding_image_as_media_on_an_image_backend(): + router: Final = _embedding_context_router("vertex_ai/gemini-embedding-2") + deployments: Final = router.registry().deployments("embedder") + + kept: Final = router._pre_call_checks(model="embedder", healthy_deployments=deployments, input=_EMBEDDING_IMAGE_INPUT) + counted: Final = await router._acount_pre_call_check_tokens( + model="embedder", healthy_deployments=deployments, messages=None, input=_EMBEDDING_IMAGE_INPUT, request_kwargs=None + ) + + assert len(kept) == 1 + assert counted is not None and 0 < counted < _EMBEDDING_CONTEXT_WINDOW + with pytest.raises(litellm.ContextWindowExceededError): + router._pre_call_checks(model="embedder", healthy_deployments=deployments, input="word " * 20_000) + + +@pytest.mark.parametrize( + ("backend", "shape"), + [ + ("vertex_ai/gemini-embedding-2", "pdf"), + ("vertex_ai/gemini-embedding-2", "pdf_file_block"), + ("vertex_ai/gemini-embedding-2", "two_pdfs"), + ("bedrock/amazon.titan-embed-image-v1", "image"), + ("bedrock/amazon.titan-embed-image-v1", "two_images"), + ], +) +def test_context_window_pre_call_check_leaves_embedding_media_to_the_backend(backend: str, shape: str): + window: Final = litellm.get_model_info(backend)["max_input_tokens"] + assert window is not None + router: Final = _embedding_context_router(backend, window) + pdf: Final = f"data:application/pdf;base64,{base64.b64encode(bytes(window * 8)).decode()}" + inputs: Final = { + "pdf": pdf, + "pdf_file_block": [{"type": "file", "file": {"file_data": pdf}}], + "two_pdfs": [pdf, pdf], + "image": _EMBEDDING_IMAGE_INPUT, + "two_images": [_EMBEDDING_IMAGE_INPUT, _EMBEDDING_IMAGE_INPUT], + } + + kept: Final = router._pre_call_checks( + model="embedder", healthy_deployments=router.registry().deployments("embedder"), input=inputs[shape] + ) + + assert len(kept) == 1 + + +def test_context_window_pre_call_check_counts_an_embedding_image_as_text_on_a_text_only_backend(): + router: Final = _embedding_context_router("text-embedding-3-small") + + with pytest.raises(litellm.ContextWindowExceededError): + router._pre_call_checks( + model="embedder", healthy_deployments=router.registry().deployments("embedder"), input=_EMBEDDING_IMAGE_INPUT + ) + + async def test_acount_pre_call_check_tokens_counts_a_group_of_one_openai_model_with_that_models_tokenizer(): router: Final = _context_limited_router("openai/gpt-6.1-sol") @@ -13826,7 +14171,11 @@ def test_get_search_tool_support_reports_each_routable_deployment_of_the_group() @pytest.mark.parametrize("provider", ["chatgpt", "github_copilot"]) @pytest.mark.parametrize("explicit_provider", [False, True]) async def test_search_tool_catalog_lookup_needs_no_credentials( - provider: str, explicit_provider: bool, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + provider: str, + explicit_provider: bool, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt")) monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "copilot")) @@ -17494,13 +17843,16 @@ def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean( def test_model_group_info_fast_mode_requires_every_deployment( local_model_cost_map: None, other_model: str, expected: bool, reverse: bool ) -> None: - models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ( - "anthropic/claude-opus-5", other_model + models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ("anthropic/claude-opus-5", other_model) + router: Final = Router( + model_list=[ + { + "model_name": "fast-group", + "litellm_params": {"model": model, "api_key": "fake-key"}, + } + for model in models + ] ) - router: Final = Router(model_list=[{ - "model_name": "fast-group", - "litellm_params": {"model": model, "api_key": "fake-key"}, - } for model in models]) result: Final = router.get_model_group_info("fast-group") @@ -17858,10 +18210,7 @@ class _AnthropicMessagesFallbackByteStream: def _anthropic_messages_overloaded_error_chunk() -> bytes: - return ( - b"event: error\n" - b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' - ) + return b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' def _anthropic_messages_invalid_request_error_chunk() -> bytes: @@ -18677,6 +19026,7 @@ def test_anthropic_stream_forwards_ping_live_direct_call(): def test_anthropic_stream_error_is_gateway_verdict_direct_call(): assert _anthropic_stream_error_is_gateway_verdict(SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES) is True + assert _anthropic_stream_error_is_gateway_verdict(AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES) is True assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_overloaded_error_chunk()) is False assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_ping_chunk()) is False @@ -18784,6 +19134,7 @@ async def test_anthropic_messages_fallback_merges_fallback_hidden_params(): ], ids=["plain-fallback", "complexity-tier-fallback"], ) +@_PYTHON_CORE_ONLY async def test_anthropic_messages_fallback_replaces_complexity_headers_before_output( fallback_headers: dict[str, str], ) -> None: @@ -18803,7 +19154,7 @@ async def test_anthropic_messages_fallback_replaces_complexity_headers_before_ou "additional_headers": fallback_headers, }, ) - router: Final = _InjectedFallbackRouter(fallback) + router: Final = _injected_fallback_router(fallback) wrapped: Final = await router._aanthropic_messages_streaming_iterator( response=source, @@ -21678,24 +22029,23 @@ class TestRequestReasoningEffortOverride: assert ("reasoning_effort" not in body) is should_drop +@_PYTHON_CORE_ONLY class TestPreRoutingTierDrivesFallbacks: """#38832: a complexity/auto router picks a tier behind the router name, but fallback lookup stayed on the router name, so the tier's configured chain never ran and a provider failure on the tier's first hop was returned to the client.""" - class _TierRouter(litellm.Router): - async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False - ): - from litellm.types.router import PreRoutingHookResponse + @staticmethod + async def _select_tier(model, request_kwargs, messages=None, input=None, specific_deployment=False): + from litellm.types.router import PreRoutingHookResponse - if model == "smart-router": - return PreRoutingHookResponse(model="tier1", messages=messages) - return None + if model == "smart-router": + return PreRoutingHookResponse(model="tier1", messages=messages) + return None @classmethod def _router(cls, fallbacks) -> "litellm.Router": - return cls._TierRouter( + router = litellm.Router( model_list=[ { "model_name": "smart-router", @@ -21745,6 +22095,8 @@ class TestPreRoutingTierDrivesFallbacks: fallbacks=fallbacks, num_retries=0, ) + router._core_async_pre_routing_hook = cls._select_tier + return router @pytest.mark.asyncio async def test_the_selected_tier_fallback_chain_runs(self): @@ -24203,50 +24555,6 @@ def test_a_failed_routing_read_prefetch_logs_the_request_model_without_its_line_ ] -@pytest.mark.asyncio -@pytest.mark.parametrize( - "routing_strategy", - ["simple-shuffle", "usage-based-routing-v2", "least-busy", "latency-based-routing"], -) -async def test_router_subclass_overriding_async_get_healthy_deployments_with_the_old_signature_still_routes( - routing_strategy: str, -) -> None: - class OldSignatureRouter(litellm.Router): - async def async_get_healthy_deployments( - self, - model: str, - request_kwargs: dict, - messages: list[dict[str, str]] | None = None, - input: str | list | None = None, - specific_deployment: bool | None = False, - parent_otel_span: Span | None = None, - health_check_probe: bool = False, - ): - return await super().async_get_healthy_deployments( - model=model, - request_kwargs=request_kwargs, - messages=messages, - input=input, - specific_deployment=specific_deployment, - parent_otel_span=parent_otel_span, - health_check_probe=health_check_probe, - ) - - router: Final = OldSignatureRouter( - model_list=[ - { - "model_name": "m", - "litellm_params": {"model": "openai/gpt-4o", "api_key": "x", "mock_response": "hi"}, - } - ], - routing_strategy=routing_strategy, - ) - - response: Final = await router.acompletion(model="m", messages=[{"role": "user", "content": "x"}]) - - assert response.choices[0].message.content == "hi" - - def test_get_deployment_credentials_with_provider_preserves_anthropic_wif_params(): """ Test that get_deployment_credentials_with_provider preserves a litellm_params-configured @@ -24611,16 +24919,25 @@ class TestAutoRouterTraceProvenance: monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()]) classifier: Final = ObservedClassifier() - router: Final = Router(model_list=[ - {**self._marker("blue"), "litellm_params": { - "model": "auto_router/complexity_router", - "complexity_router_config": { - "classifier_type": "custom", "classifier_plugin": classifier, - "tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False, + router: Final = Router( + model_list=[ + { + **self._marker("blue"), + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "custom", + "classifier_plugin": classifier, + "tiers": {"SIMPLE": "answer"}, + "session_affinity": False, + "deployment_affinity": False, + }, + }, }, - }}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ], plugins=[classifier]) + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ], + plugins=[classifier], + ) stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"} request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}} with logger.tracer.start_as_current_span("request") as root: @@ -24705,20 +25022,22 @@ class TestAutoRouterTraceProvenance: self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]" ) -> None: logger, exporter = _tracing - router: Final = Router(model_list=[ - self._marker("blue"), - { - "model_name": "traced-router", - "model_info": {"id": "definition-semantic"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "tags": ["blue"], - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", + router: Final = Router( + model_list=[ + self._marker("blue"), + { + "model_name": "traced-router", + "model_info": {"id": "definition-semantic"}, + "litellm_params": { + "model": "auto_router/semantic-router", + "tags": ["blue"], + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, }, - }, - ]) + ] + ) with logger.start_phase_span("route traced-router"): response: Final = await router.async_pre_routing_hook( model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}} @@ -24738,26 +25057,42 @@ class TestAutoRouterTraceProvenance: ) -> None: logger, exporter = _tracing monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={ - "object": "list", - "model": "text-embedding-3-small", - "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - }) - router: Final = Router(model_list=[ - { - "model_name": "internal-semantic-router", - "model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", - }, + respx_mock.post("https://embedding.example/v1/embeddings").respond( + 200, + json={ + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, }, - {"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ]) + ) + router: Final = Router( + model_list=[ + { + "model_name": "internal-semantic-router", + "model_info": { + "id": "semantic-definition", + "team_id": "trace-team", + "team_public_model_name": "public-router", + }, + "litellm_params": { + "model": "auto_router/semantic-router", + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_base": "https://embedding.example/v1", + "api_key": "test", + }, + }, + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ] + ) request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}} with logger.tracer.start_as_current_span("request"): await router.async_get_available_deployment( @@ -25467,6 +25802,441 @@ async def test_router_clientside_credential_not_reused_by_other_callers(respx_mo used_auth_headers = {call.request.headers["authorization"] for call in route.calls[1:]} assert used_auth_headers == {"Bearer configured-key"} +def _caused_by_an_entra_rejection(error: litellm.AuthenticationError) -> litellm.AuthenticationError: + from litellm.llms.azure.common_utils import EntraRejectionError + + try: + raise error from EntraRejectionError(status_code=401, message=error.message) + except litellm.AuthenticationError as raised: + return raised + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize("caller_credential", ["client_secret", "azure_password", "azure_ad_token"]) +async def test_router_caller_entra_credential_rejection_leaves_the_shared_deployment_routable(caller_credential): + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + "client_secret": "configured-secret", + }, + "model_info": {"id": "configured-deployment"}, + } + ] + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=_caused_by_an_entra_rejection( + litellm.AuthenticationError( + message="Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided.", + llm_provider="azure", + model="gpt-4.1", + ) + ), + **{caller_credential: "not-the-credential"}, + ) + + response: Final = await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert response.choices[0].message.content == "hi" + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize( + "caller_field", + [ + pytest.param({"client_secret": "environment-secret"}, id="environment_secret"), + pytest.param({"client_id": "environment-client"}, id="environment_client_id"), + pytest.param({"azure_scope": "https://cognitiveservices.azure.com/.default"}, id="default_scope"), + pytest.param({"client_secret": ""}, id="blank_secret"), + ], +) +async def test_router_caller_entra_field_that_changes_the_deployment_credential_leaves_it_routable( + caller_field, monkeypatch +): + monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret") + monkeypatch.setenv("AZURE_CLIENT_ID", "environment-client") + monkeypatch.delenv("AZURE_SCOPE", raising=False) + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + "client_secret": "configured-secret", + "azure_scope": "api://configured-app/.default", + }, + "model_info": {"id": "configured-deployment"}, + } + ] + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=_caused_by_an_entra_rejection( + litellm.AuthenticationError( + message="Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided.", + llm_provider="azure", + model="gpt-4.1", + ) + ), + **caller_field, + ) + + response: Final = await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert response.choices[0].message.content == "hi" + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize( + ("deployment_secret", "router_default_secret"), + [ + pytest.param({"client_secret": "configured-secret"}, None, id="deployment"), + pytest.param({}, {"client_secret": "configured-secret"}, id="router_default_litellm_params"), + ], +) +async def test_router_configured_entra_credential_rejection_cools_down_the_deployment( + deployment_secret, router_default_secret +): + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + **deployment_secret, + }, + "model_info": {"id": "configured-deployment"}, + } + ], + default_litellm_params=router_default_secret, + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=litellm.AuthenticationError( + message="Failed to get Azure AD token: AADSTS7000222: The provided client secret keys are expired.", + llm_provider="azure", + model="gpt-4.1", + ), + ) + + with pytest.raises(litellm.RouterRateLimitError) as excinfo: + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert excinfo.value.all_deployments_in_cooldown is True + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize( + "caller_fields", + [ + pytest.param({"azure_scope": "https://cognitiveservices.azure.com/.default"}, id="default_scope"), + pytest.param({"azure_username": ""}, id="empty_username"), + pytest.param({"client_secret": "environment-secret"}, id="environment_secret"), + ], +) +async def test_router_caller_entra_field_that_changes_nothing_still_cools_down_a_rejected_deployment( + caller_fields, monkeypatch +): + monkeypatch.delenv("AZURE_SCOPE", raising=False) + monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret") + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + }, + "model_info": {"id": "configured-deployment"}, + } + ] + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=_caused_by_an_entra_rejection( + litellm.AuthenticationError( + message="Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided.", + llm_provider="azure", + model="gpt-4.1", + ) + ), + **caller_fields, + ) + + with pytest.raises(litellm.RouterRateLimitError) as excinfo: + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert excinfo.value.all_deployments_in_cooldown is True + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +async def test_router_caller_repeating_the_secret_an_empty_router_default_selects_still_cools_down(monkeypatch): + monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret") + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "tenant_id": "configured-tenant", + "client_id": "configured-client", + "client_secret": "configured-secret", + }, + "model_info": {"id": "configured-deployment"}, + } + ], + default_litellm_params={"client_secret": ""}, + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=_caused_by_an_entra_rejection( + litellm.AuthenticationError( + message="Failed to get Azure AD token: AADSTS7000215: Invalid client secret provided.", + llm_provider="azure", + model="gpt-4.1", + ) + ), + client_secret="environment-secret", + ) + + with pytest.raises(litellm.RouterRateLimitError) as excinfo: + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert excinfo.value.all_deployments_in_cooldown is True + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +async def test_router_caller_entra_field_keeps_the_shared_deployment(): + router = Router( + model_list=[ + { + "model_name": "shared-model", + "litellm_params": {"model": "openai/gpt-4.1-mini", "api_key": "configured-key"}, + "model_info": {"id": "shared-deployment"}, + } + ] + ) + + response: Final = await router.acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + client_id="caller-client", + ) + + assert response._hidden_params["model_id"] == "shared-deployment" + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize( + ("litellm_params", "provider"), + [ + pytest.param({"model": "openai/gpt-4.1-mini", "api_key": "revoked-key"}, "openai", id="openai"), + pytest.param( + { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "api_key": "revoked-key", + }, + "azure", + id="azure_api_key", + ), + ], +) +@pytest.mark.parametrize("caller_entra_field", ["client_id", "azure_scope"]) +async def test_router_auth_failure_entra_did_not_cause_cools_down_despite_a_caller_entra_field( + litellm_params, provider, caller_entra_field +): + router = Router( + model_list=[ + { + "model_name": "shared-model", + "litellm_params": litellm_params, + "model_info": {"id": "shared-deployment"}, + } + ] + ) + + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + mock_response=litellm.AuthenticationError( + message="Incorrect API key provided.", llm_provider=provider, model=litellm_params["model"] + ), + **{caller_entra_field: "caller-value"}, + ) + + with pytest.raises(litellm.RouterRateLimitError) as excinfo: + await router.acompletion( + model="shared-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="hi", + ) + + assert excinfo.value.all_deployments_in_cooldown is True + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +async def test_router_fallback_auth_failure_after_an_entra_rejection_still_cools_down_the_fallback(monkeypatch): + from azure.core.exceptions import ClientAuthenticationError + + def rejected_token() -> str: + raise ClientAuthenticationError(message="AADSTS7000215: Invalid client secret provided.") + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "azure_ad_token_provider": rejected_token, + }, + "model_info": {"id": "entra-deployment"}, + }, + { + "model_name": "key-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa-fallback.openai.azure.com", + "api_version": "2025-04-01-preview", + "api_key": "revoked-key", + }, + "model_info": {"id": "key-deployment"}, + }, + ], + fallbacks=[{"entra-model": ["key-model"]}], + ) + + with respx.mock: + fallback_route: Final = respx.post(url__startswith="https://litellm-qa-fallback.openai.azure.com").mock( + return_value=httpx.Response( + 401, json={"error": {"code": "401", "message": "Access denied due to invalid subscription key."}} + ) + ) + with pytest.raises(litellm.AuthenticationError): + await router.acompletion( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + client_secret="not-the-credential", + ) + + with pytest.raises(litellm.RouterRateLimitError) as excinfo: + await router.acompletion( + model="key-model", + messages=[{"role": "user", "content": "hi"}], + ) + + assert fallback_route.call_count == 1 + assert excinfo.value.all_deployments_in_cooldown is True + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.parametrize( + "stream_request", + [ + pytest.param( + lambda router, **caller_fields: router.acompletion( + model="entra-model", messages=[{"role": "user", "content": "hi"}], stream=True, **caller_fields + ), + id="chat_completions", + ), + pytest.param( + lambda router, **caller_fields: router.aanthropic_messages( + model="entra-model", + messages=[{"role": "user", "content": "hi"}], + max_tokens=5, + stream=True, + **caller_fields, + ), + id="messages", + ), + ], +) +async def test_router_caller_entra_rejection_on_a_stream_leaves_the_shared_deployment_routable( + monkeypatch: pytest.MonkeyPatch, + stream_request: Callable[..., Awaitable[object]], +) -> None: + from azure.core.exceptions import ClientAuthenticationError + + def rejected_token() -> str: + raise ClientAuthenticationError(message="AADSTS7000215: Invalid client secret provided.") + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = Router( + model_list=[ + { + "model_name": "entra-model", + "litellm_params": { + "model": "azure/gpt-4.1", + "api_base": "https://litellm-qa.openai.azure.com", + "api_version": "2025-04-01-preview", + "azure_ad_token_provider": rejected_token, + }, + "model_info": {"id": "entra-deployment"}, + } + ] + ) + + with pytest.raises(litellm.AuthenticationError): + await stream_request(router, client_secret="not-the-credential") + + with pytest.raises(litellm.AuthenticationError): + await stream_request(router) + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_router_get_async_openai_model_client(): router = Router( @@ -29298,11 +30068,21 @@ def test_reset_custom_routing_strategy(): assert "get_available_deployment" not in router._core.__dict__ assert "_core_async_get_available_deployment" not in router._core.__dict__ - assert router.async_get_available_deployment.__func__ is Router.async_get_available_deployment + assert router.async_get_available_deployment.__wrapped__.__func__ is Router.async_get_available_deployment router._reset_custom_routing_strategy() +@pytest.mark.asyncio +async def test_select_picks_through_a_custom_routing_strategy() -> None: + router: Final = _create_custom_routing_router() + router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router)) + + picked: Final = await router.select("azure-model", {"metadata": {}}) + + assert picked["model_info"]["id"] == "very-special-endpoint" + + @pytest.mark.asyncio async def test_azure_router_acompletion_passes_deployment_credentials_for_plain_and_stream( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 81c1982b987..25dd8fe47cf 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -334,8 +334,10 @@ def test_settings_snapshot_is_reused_until_a_setting_is_written(router): assert router.settings().fallbacks == [{"gpt": ["team-gpt"]}] router._core_fallbacks = [{"gpt": ["other"]}] # the router's own writes invalidate it too assert router.settings().fallbacks == [{"gpt": ["other"]}] - router.settings().fallbacks.append({"x": ["y"]}) # the snapshot holds the router's own objects - assert router._core_fallbacks[-1] == {"x": ["y"]} + router.settings().fallbacks.append( + {"x": ["y"]} + ) # the snapshot's lists are copies: settings change only via apply() + assert router._core_fallbacks == [{"gpt": ["other"]}] unrelated = router.settings() router._core_model_list = list(router._core_model_list) assert router.settings() is unrelated @@ -446,84 +448,30 @@ def test_generate_model_id_is_the_module_function(): assert Router.generate_model_id("g", params) == deployment_id("g", params) -def test_subclass_override_of_an_old_member_is_still_called(): - calls = [] +def test_router_cannot_be_subclassed(): + with pytest.raises(TypeError, match=r"^litellm\.Router cannot be subclassed"): - class Sub(litellm.Router): - async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False - ): - calls.append(model) - return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment) - - r = Sub( - model_list=[ - { - "model_name": "m", - "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x", "mock_response": "ok"}, - } - ] - ) - response = asyncio.run(r.acompletion(model="m", messages=[{"role": "user", "content": "hi"}])) - assert response.choices[0].message.content == "ok" - assert calls == ["m"] + class Sub(litellm.Router): + pass -def test_direct_call_of_a_subclass_override_runs_it_once(): - calls = [] +def test_core_classes_can_still_be_subclassed(): + from litellm.router_python_core import PythonCore - class Sub(litellm.Router): - def get_model_access_groups(self, model_name=None, model_access_group=None, team_id=None): - calls.append(model_name) - return super().get_model_access_groups(model_name, model_access_group, team_id) + class PythonSub(PythonCore): + pass - r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - assert r.get_model_access_groups("m") == {} - assert calls == ["m"] - assert asyncio.run(asyncio.to_thread(r.registry().access_groups, model_name="m")) == {} - assert calls == ["m", "m"] + assert isinstance(object.__new__(PythonSub), Router) + try: + from litellm.router_rust.core import RustCore + except ImportError: + pytest.skip("the native lib does not load") -def test_subclass_staticmethod_override_is_still_consulted(): - calls = [] + class RustSub(RustCore): + pass - class Sub(litellm.Router): - @staticmethod - def is_deployment_blocked(deployment): - calls.append(deployment.model_info.id) - return True - - r = Sub( - model_list=[ - { - "model_name": "m", - "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}, - "model_info": {"id": "d1"}, - } - ] - ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - assert r.get_deployment_credentials(model_id="d1") is None - assert calls == ["d1"] - - -def test_private_alias_skips_a_subclass_override_of_the_public_name(): - calls = [] - - class Sub(litellm.Router): - def get_all_deployments(self, *args, **kwargs): - calls.append("get_all_deployments") - return super().get_all_deployments(*args, **kwargs) - - r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) - calls.clear() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - assert len(r._get_all_deployments(model_name="m")) == 1 - assert calls == [] + assert issubclass(RustSub, RustCore) def test_old_name_writes_and_deletes_warn(router): @@ -779,23 +727,6 @@ def test_old_members_refuse_objects_that_are_not_routers(): _as_router(object()) # pyright: ignore[reportArgumentType] -def test_direct_call_of_an_async_subclass_override_runs_it_once(): - calls = [] - - class Sub(litellm.Router): - async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False - ): - calls.append(model) - return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment) - - r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - asyncio.run(r.async_pre_routing_hook("m", {})) - assert calls == ["m"] - - def test_routing_context_forwards_to_the_router(router, monkeypatch): from unittest.mock import AsyncMock, MagicMock diff --git a/tests/unit/test_router/test_router_fallbacks.py b/tests/unit/test_router/test_router_fallbacks.py index bccd6290f44..b25d5c35892 100644 --- a/tests/unit/test_router/test_router_fallbacks.py +++ b/tests/unit/test_router/test_router_fallbacks.py @@ -1207,3 +1207,156 @@ def test_context_stream_fallback_preserves_other_errors_for_existing_continuatio message="provider unavailable", model="primary", llm_provider="openai", generated_content="already visible" ) assert _context_window_stream_fallback_error(error, "primary", "openai", True) is error + + +def _context_fallback_precedence_router(context_key: str | None, ordered: bool) -> Router: + deployments: Final = ( + ("primary", "primary", 1), + ("primary" if ordered else "unused", "ordered", 2), + ("generic", "generic", 1), + ("context", "context", 1), + ) + return Router( + model_list=[ + { + "model_name": group, + "litellm_params": { + "model": f"openai/{model}", + "api_key": "test-key", + "api_base": "https://fallback-migration.local/v1", + "order": order, + }, + } + for group, model, order in deployments + ], + fallbacks=[{"primary": ["generic"]}], + context_window_fallbacks=None if context_key is None else [{context_key: ["context"]}], + num_retries=0, + disable_cooldowns=True, + ) + + +@pytest.mark.parametrize("sync_mode", (False, True)) +@pytest.mark.parametrize( + ("context_key", "ordered", "flat_fallbacks", "expected_model"), + ( + (None, True, False, "ordered"), + ("unrelated", True, False, "ordered"), + ("unrelated", False, False, "generic"), + ("primary", True, False, "context"), + ("*", True, False, "context"), + ("primary", True, True, "context"), + ("primary", False, True, "context"), + ), +) +@pytest.mark.asyncio +async def test_context_fallback_precedence_is_specific_to_the_requested_group( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + context_key: str | None, + ordered: bool, + flat_fallbacks: bool, + expected_model: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router(context_key, ordered) + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=( + httpx.Response( + 400, + json={ + "error": { + "message": "This model's maximum context length is 10 tokens", + "type": "invalid_request_error", + "code": "context_length_exceeded", + } + }, + ), + _fallback_chat_response(expected_model, f"served by {expected_model}"), + ) + ) + + response: Final = ( + router.completion( + model="primary", + messages=[{"role": "user", "content": "long prompt"}], + fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}], + ) + if sync_mode + else await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "long prompt"}], + fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}], + ) + ) + + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", expected_model) + assert response.choices[0].message.content == f"served by {expected_model}" + + +@pytest.mark.parametrize("matching_context", (False, True)) +@pytest.mark.asyncio +async def test_context_fallback_exhaustion_preserves_the_selected_chain( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + matching_context: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router("primary" if matching_context else "unrelated", True) + refusal: Final = httpx.Response( + 400, + json={"error": {"message": "This model's maximum context length is 10 tokens"}}, + ) + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=(refusal, refusal, _fallback_chat_response("generic", "generic answer")) + ) + + if matching_context: + with pytest.raises(litellm.ContextWindowExceededError): + await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "context") + return + + response: Final = await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic") + assert response.choices[0].message.content == "generic answer" + + +@pytest.mark.parametrize("sync_mode", (False, True)) +@pytest.mark.parametrize("context_error", (False, True)) +@pytest.mark.parametrize("backup_status", (429, 503)) +@pytest.mark.asyncio +async def test_ordered_fallback_failures_preserve_the_original_context_error( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + context_error: bool, + backup_status: int, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router("unrelated", True) + original_message: Final = ( + "This model's maximum context length is 10 tokens" if context_error else "primary unavailable" + ) + backup_error: Final = litellm.RateLimitError if backup_status == 429 else litellm.ServiceUnavailableError + expected_error: Final = litellm.ContextWindowExceededError if context_error else backup_error + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=( + httpx.Response(400 if context_error else 503, json={"error": {"message": original_message}}), + httpx.Response(backup_status, json={"error": {"message": "ordered deployment unavailable"}}), + httpx.Response(backup_status, json={"error": {"message": "generic deployment unavailable"}}), + ) + ) + + request: Final = ( + asyncio.to_thread(router.completion, model="primary", messages=[{"role": "user", "content": "long prompt"}]) + if sync_mode + else router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + ) + with pytest.raises(expected_error) as raised: + await request + + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic") + assert raised.value.status_code == (400 if context_error else backup_status) + assert (original_message if context_error else "generic deployment unavailable") in str(raised.value) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index b46a57ad5f1..3bbb69413d9 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -3,8 +3,9 @@ import functools import gc import json import os +import uuid import weakref -from collections.abc import Awaitable, Callable, Iterator, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final @@ -13,6 +14,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx +from websockets.exceptions import ConnectionClosed +from websockets.frames import Close import litellm from litellm import Router @@ -21,8 +24,14 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( @@ -540,6 +549,242 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +class _DiscardingLoggingWorker: + def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: + async_coroutine.close() + + +async def _client_never_sends() -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +def _realtime_client_ws() -> MagicMock: + client_ws: Final = MagicMock() + client_ws.headers = {} + client_ws.receive_text = AsyncMock(side_effect=_client_never_sends) + client_ws.send_text = AsyncMock() + client_ws.close = AsyncMock() + return client_ws + + +def _realtime_session_logging() -> LiteLLMLogging: + return LiteLLMLogging( + model="gpt-realtime-2.1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="realtime-session", + ) + + +async def _relay_upstream( + litellm_logging_obj: LiteLLMLogging, + *upstream_frames: bytes | Exception, + client_ws: MagicMock | None = None, + provider_config: BaseRealtimeConfig | None = None, + upstream_stays_open: bool = False, +) -> None: + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=_frames_then_silence(*upstream_frames) if upstream_stays_open else list(upstream_frames) + ) + backend_ws.send = AsyncMock() + relay: Final = RealTimeStreaming( + client_ws or _realtime_client_ws(), + backend_ws, + litellm_logging_obj, + provider_config=provider_config, + model="gpt-realtime-2.1", + logging_worker=_DiscardingLoggingWorker(), + ) + await asyncio.wait_for(relay.bidirectional_forward(), timeout=2) + + +_REALTIME_SESSION_CREATED: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + + +async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1000, ""), None)) + + +async def _upstream_internal_error_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream( + litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1011, "upstream failed"), None) + ) + + +async def _upstream_drop_without_close_frame_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(None, None)) + + +async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None)) + + +def _frames_then_silence(*frames: bytes | str) -> Callable[[], Awaitable[bytes | str]]: + pending: Final = iter(frames) + + async def _next_frame() -> bytes | str: + frame: Final = next(pending, None) + if frame is None: + await asyncio.Event().wait() + raise AssertionError("unreachable") + return frame + + return _next_frame + + +def _client_ws_sending(*client_frames: str) -> MagicMock: + client_ws: Final = _realtime_client_ws() + client_ws.receive_text = AsyncMock(side_effect=_frames_then_silence(*client_frames)) + return client_ws + + +async def _vertex_access_token() -> str: + return "token" + + +async def _client_frame_the_provider_rejects(litellm_logging_obj: LiteLLMLogging) -> None: + deferred_setup_provider: Final = VertexChirpRealtimeConfig( + resolve_access_token=_vertex_access_token, project="proj-1", location="us" + ) + await _relay_upstream( + litellm_logging_obj, + client_ws=_client_ws_sending( + json.dumps( + { + "type": "session.update", + "session": {"type": "transcription", "audio": {"input": {"format": {"type": "audio/pcmu"}}}}, + } + ) + ), + provider_config=deferred_setup_provider, + upstream_stays_open=True, + ) + + +async def _client_hangs_up_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + client_ws: Final = _realtime_client_ws() + client_ws.receive_text = AsyncMock(side_effect=ConnectionClosed(Close(1000, ""), None)) + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, client_ws=client_ws, upstream_stays_open=True) + + +async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None: + await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("realtime_session", "counted_successes"), + [ + (_clean_realtime_session, 1), + (_upstream_close_before_any_frame, 0), + (_refused_handshake, 0), + (_upstream_internal_error_after_a_frame, 0), + (_upstream_drop_without_close_frame_after_a_frame, 0), + (_client_frame_the_provider_rejects, 0), + (_client_hangs_up_after_a_frame, 1), + ], + ids=[ + "clean-session", + "upstream-close-before-any-frame", + "refused-handshake", + "upstream-1011-after-a-frame", + "upstream-1006-after-a-frame", + "client-frame-the-provider-rejects", + "client-hangs-up-after-a-frame", + ], +) +async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure( + realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": "openai/gpt-realtime-2.1"}, + "model_info": {"id": "realtime-a"}, + } + ], + num_retries=0, + ) + + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + await realtime_session(litellm_logging_obj) + + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging() + ) + + assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes + + +@pytest.mark.asyncio +@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) +@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) +async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( + realtime_model: str, clean_sessions: int, cooled_down: bool +) -> None: + import time + + provider, _, provider_model = realtime_model.partition("/") + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": realtime_model, "order": order}, + "model_info": {"id": deployment_id}, + } + for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b")) + ], + num_retries=0, + ) + drop: Final = litellm.ServiceUnavailableError( + message="realtime provider stream ended before the session closed", + llm_provider=provider, + model=provider_model, + ) + + def realtime_session(failure: Exception | None): + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + if failure is not None: + raise failure + await _clean_realtime_session(litellm_logging_obj) + + return _arealtime + + for _ in range(clean_sessions): + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(None), + litellm_logging_obj=_realtime_session_logging(), + ) + + async def drop_session() -> bool: + with pytest.raises(litellm.ServiceUnavailableError): + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(drop), + litellm_logging_obj=_realtime_session_logging(), + ) + return router.deployment_callback_on_failure( + kwargs={ + "exception": drop, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + drop_cooldowns: Final = [await drop_session() for _ in range(5)] + + assert any(drop_cooldowns) is cooled_down + + async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]: routed: Final[dict[str, object]] = {} @@ -599,6 +844,34 @@ async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_b await admit_turn() +@pytest.mark.asyncio +async def test_responses_websocket_turns_do_not_make_their_deployment_look_busy_to_least_busy(): + router: Final = Router( + model_list=[ + { + "model_name": "ws-least-busy", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("ws-least-busy-a", "ws-least-busy-b") + ], + routing_strategy="least-busy", + ) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-least-busy", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + for _ in range(3): + await admit_turn() + + picks: Final = [ + (await router._core_async_get_available_deployment(model="ws-least-busy", request_kwargs={"input": "hi"}))[ + "model_info" + ]["id"] + for _ in range(2) + ] + + assert sorted(picks) == ["ws-least-busy-a", "ws-least-busy-b"] + + @pytest.mark.asyncio async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm(): router = Router( diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 96519a0acbe..724c07f8c31 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -35,6 +35,7 @@ from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.proxy.utils import create_model_info_response from litellm.types.router import ( Deployment, DeploymentTypedDict, @@ -173,20 +174,26 @@ async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch _invalidate_model_cost_lowercase_map() -async def test_discovery_sends_the_secret_behind_an_oidc_api_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +async def test_discovery_sends_the_secret_behind_an_oidc_api_key( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) token: Final = tmp_path / "token" token.write_text("local-secret") - router: Final = Router(model_list=[{ - "model_name": "local", - "litellm_params": { - "model": "hosted_vllm/local-model", - "api_base": "https://discovery.test/v1", - "api_key": f"oidc/file/{token}", - }, - "model_info": {"id": "oidc-deployment"}, - }]) + router: Final = Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://discovery.test/v1", + "api_key": f"oidc/file/{token}", + }, + "model_info": {"id": "oidc-deployment"}, + } + ] + ) authorizations: Final[list[str | None]] = [] def respond(request: httpx.Request) -> httpx.Response: @@ -1073,8 +1080,12 @@ def test_bedrock_deployment_keeps_custom_prices_when_profile_lookup_fails( tier_rates: Final = inherit_configured and service_tier != "default" input_rate: Final = configured_prices[f"input_cost_per_token_{service_tier}"] if tier_rates else 0.01 output_rate: Final = configured_prices[f"output_cost_per_token_{service_tier}"] if tier_rates else 0.02 - effective_input: Final = 0.0 if override == "input_cost_per_token_priority" and service_tier == "priority" else input_rate - cache_rate: Final = 0.0 if override == "cache_read_input_token_cost" else (0.001 if inherit_configured else effective_input) + effective_input: Final = ( + 0.0 if override == "input_cost_per_token_priority" and service_tier == "priority" else input_rate + ) + cache_rate: Final = ( + 0.0 if override == "cache_read_input_token_cost" else (0.001 if inherit_configured else effective_input) + ) assert completion_cost( model=model, completion_response=ModelResponse( @@ -1564,6 +1575,8 @@ def test_ptu_service_tier_pricing_is_disabled_only_when_attribution_is_enabled( response: Final = router.completion( model=f"ptu-tier-model-{ptu_enabled}", messages=[{"role": "user", "content": "ptu service tier pricing"}], + # The deployment belongs to team-alpha, and a sync call is team filtered like an async one. + metadata={"user_api_key_team_id": "team-alpha"}, service_tier="priority", mock_response=litellm.ModelResponse( model=_TIER_BACKEND_MODEL, @@ -2164,6 +2177,154 @@ def test_capability_flags_propagate_from_deployment_model_info_to_shared_key(): _restore_model_cost_entries(model_keys) +@pytest.mark.parametrize("override_first", [True, False], ids=["override-first", "override-last"]) +def test_token_limits_stay_on_the_deployment_that_set_them(override_first: bool) -> None: + backend_model: Final = "openai/gpt-4o-mini" + shared_keys: Final = ("gpt-4o-mini", backend_model) + token_fields: Final = ("max_input_tokens", "max_output_tokens", "max_tokens") + catalog_limits: Final = { + key: {field: litellm.get_model_info(model=key).get(field) for field in token_fields} for key in shared_keys + } + catalog_cost_map_limits: Final = { + key: {field: (litellm.model_cost.get(key) or {}).get(field) for field in token_fields} for key in shared_keys + } + override_id: Final = "token-limit-override-deployment" + plain_id: Final = "token-limit-plain-deployment" + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) for key in (*shared_keys, override_id, plain_id) + } + expected_cost_map_limits: Final = { + key: { + **catalog_cost_map_limits[key], + "max_tokens": catalog_cost_map_limits[key]["max_tokens"] + or catalog_cost_map_limits[key]["max_output_tokens"], + } + for key in shared_keys + } + plain_deployment: Final = { + "model_name": "plain-gpt-4o-mini", + "litellm_params": {"model": backend_model}, + "model_info": {"id": plain_id}, + } + override_deployment: Final = { + "model_name": "overridden-gpt-4o-mini", + "litellm_params": {"model": backend_model}, + "model_info": { + "id": override_id, + "max_input_tokens": 4321, + "max_output_tokens": 987, + "max_tokens": 987, + }, + } + try: + router: Final = Router( + model_list=[override_deployment, plain_deployment] + if override_first + else [plain_deployment, override_deployment] + ) + litellm.get_model_info.cache_clear() + + for key in shared_keys: + assert { + field: (litellm.model_cost.get(key) or {}).get(field) for field in token_fields + } == expected_cost_map_limits[key] + assert {field: litellm.get_model_info(model=key).get(field) for field in token_fields} == catalog_limits[ + key + ] + + assert {field: litellm.get_model_info(model=override_id).get(field) for field in token_fields} == { + "max_input_tokens": 4321, + "max_output_tokens": 987, + "max_tokens": 987, + } + + plain_listing: Final = create_model_info_response( + model_id="plain-gpt-4o-mini", provider="openai", llm_router=router + ) + override_listing: Final = create_model_info_response( + model_id="overridden-gpt-4o-mini", provider="openai", llm_router=router + ) + assert (plain_listing["max_input_tokens"], plain_listing["max_output_tokens"]) == ( + catalog_limits[backend_model]["max_input_tokens"], + catalog_limits[backend_model]["max_output_tokens"], + ) + assert (override_listing["max_input_tokens"], override_listing["max_output_tokens"]) == (4321, 987) + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_unknown_backend_token_limits_still_fill_the_shared_key() -> None: + backend_model: Final = "hosted_vllm/somelab.lit4544-unmapped-token-limits" + first_id: Final = "unknown-token-limit-first-deployment" + second_id: Final = "unknown-token-limit-second-deployment" + token_fields: Final = ("max_input_tokens", "max_output_tokens", "max_tokens") + first_limits: Final = {"max_input_tokens": 4321, "max_output_tokens": 987, "max_tokens": 987} + second_limits: Final = {"max_input_tokens": 1234, "max_output_tokens": 456, "max_tokens": 456} + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) for key in (backend_model, first_id, second_id) + } + try: + Router( + model_list=[ + { + "model_name": "unknown-token-limit-first", + "litellm_params": {"model": backend_model, "api_key": "fake-key"}, + "model_info": {"id": first_id, **first_limits}, + }, + { + "model_name": "unknown-token-limit-second", + "litellm_params": {"model": backend_model, "api_key": "fake-key"}, + "model_info": {"id": second_id, **second_limits}, + }, + ] + ) + + assert { + field: (litellm.model_cost.get(backend_model) or {}).get(field) for field in token_fields + } == second_limits + assert {field: litellm.get_model_info(first_id).get(field) for field in token_fields} == first_limits + assert {field: litellm.get_model_info(second_id).get(field) for field in token_fields} == second_limits + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_unknown_backend_token_limit_updates_when_deployment_is_upserted() -> None: + backend_model: Final = "hosted_vllm/somelab.lit4544-token-limit-update" + model_id: Final = "unknown-token-limit-updated-deployment" + model_keys: Final = {key: copy.deepcopy(litellm.model_cost.get(key)) for key in (backend_model, model_id)} + try: + deployment: Final = Deployment( + model_name="unknown-token-limit-update", + litellm_params=LiteLLM_Params( + model=backend_model, + api_base="https://unknown-model.test/v1", + api_key="fake-key", + ), + model_info=ModelInfo(id=model_id, max_output_tokens=999), + ) + router: Final = Router(model_list=[deployment.to_json()]) + assert (litellm.model_cost.get(backend_model) or {}).get("max_output_tokens") == 999 + + router.upsert_deployment( + Deployment( + model_name="unknown-token-limit-update", + litellm_params=LiteLLM_Params( + model=backend_model, + api_base="https://unknown-model.test/v1", + api_key="fake-key", + ), + model_info=ModelInfo(id=model_id, max_output_tokens=1500), + ) + ) + + assert (litellm.model_cost.get(backend_model) or {}).get("max_output_tokens") == 1500 + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + def test_wildcard_zero_cost_request_does_not_poison_named_deployment_pricing(): """LIT-3991 end to end: a proxy has a named text-embedding-3-small deployment relying on built-in pricing plus an ``openai/*`` wildcard with @@ -2849,7 +3010,8 @@ def test_a_github_copilot_chat_pin_opts_a_dual_endpoint_model_out_of_the_respons @pytest.mark.parametrize("registered_mode, pinned_mode", [("chat", "responses"), ("responses", "chat")]) @pytest.mark.parametrize( - "model, custom_llm_provider", [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")] + "model, custom_llm_provider", + [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")], ) def test_a_github_copilot_responses_pin_still_opts_the_model_into_the_responses_api( monkeypatch: pytest.MonkeyPatch, @@ -3067,7 +3229,11 @@ def test_register_deployment_in_model_cost_writes_both_key_families(): try: Router._register_deployment_in_model_cost( model_id="both-families-id", - model_info={"id": "both-families-id", "max_input_tokens": 999, "litellm_provider": "hosted_vllm"}, + model_info={ + "id": "both-families-id", + "max_input_tokens": 999, + "litellm_provider": "hosted_vllm", + }, model="hosted_vllm/both-families-backend", custom_llm_provider=None, ) @@ -4027,7 +4193,9 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info( assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price -def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(monkeypatch: pytest.MonkeyPatch) -> None: +def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") catalog: Final = litellm.get_model_cost_map(url="") monkeypatch.setattr(litellm, "model_cost", dict(catalog)) diff --git a/tests/unit/test_router_order_fallback.py b/tests/unit/test_router_order_fallback.py index 47e0be28635..6dd445e2b5d 100644 --- a/tests/unit/test_router_order_fallback.py +++ b/tests/unit/test_router_order_fallback.py @@ -95,8 +95,9 @@ class TestGetOrderFilteredDeployments: assert len(result) == 2 -def test_get_deployment_order_returns_unvalidated_order(): - assert get_deployment_order({"litellm_params": {"order": "first"}}) == "first" +def test_get_deployment_order_reads_a_non_number_as_unordered(): + assert get_deployment_order({"litellm_params": {"order": "first"}}) is None + assert get_deployment_order({"litellm_params": {"order": "2"}}) == 2 # --------------------------------------------------------------------------- @@ -646,7 +647,6 @@ async def test_text_completion_order_fallback_hop_does_not_send_target_order_ups assert all("_target_order" not in body for body in upstream_bodies) - _OPENAI_RESPONSES_URL: Final = "https://api.openai.com/v1/responses" _MANTLE_RESPONSES_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses" _OVERLOADED_UPSTREAM: Final = {"error": {"message": "overloaded", "type": "server_error", "code": "server_error"}} diff --git a/tests/unit/test_router_weighted_failover.py b/tests/unit/test_router_weighted_failover.py index c00670473af..70231b620be 100644 --- a/tests/unit/test_router_weighted_failover.py +++ b/tests/unit/test_router_weighted_failover.py @@ -1019,3 +1019,144 @@ async def test_failover_reaches_healthy_sibling_when_affinity_pins_failed_deploy assert while_a_is_down._hidden_params["model_id"] == "B" assert while_a_is_down._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 assert after_a_recovers._hidden_params["model_id"] == "A" + + +def _two_deployment_router(model: str) -> Router: + return Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": model, "api_key": "bad", "weight": 1}, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": {"model": model, "api_key": "good", "weight": 0}, + "model_info": {"id": "B"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=0, + disable_cooldowns=True, + enable_weighted_failover=True, + ) + + +_ENDPOINTS: Final = [ + ("aembedding", "openai/text-embedding-3-small", {"input": ["hi"]}), + ("aimage_generation", "openai/dall-e-3", {"prompt": "a cat"}), + ("atranscription", "openai/whisper-1", {"file": b"audio"}), + ("aspeech", "openai/tts-1", {"input": "hi", "voice": "alloy"}), + ("atext_completion", "openai/gpt-3.5-turbo-instruct", {"prompt": "hi"}), + ("arerank", "cohere/rerank-english-v3.0", {"query": "q", "documents": ["d"]}), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("endpoint", "model", "arguments"), _ENDPOINTS, ids=[e[0] for e in _ENDPOINTS]) +async def test_weighted_failover_moves_every_endpoint_to_another_deployment( + monkeypatch: pytest.MonkeyPatch, endpoint: str, model: str, arguments: dict[str, object] +) -> None: + keys_called: list[object] = [] + answer: Final = object() + + async def provider(**kwargs: object) -> object: + keys_called.append(kwargs.get("api_key")) + if kwargs.get("api_key") == "bad": + raise litellm.InternalServerError(message="A is down", llm_provider="openai", model=model) + return answer + + monkeypatch.setattr(litellm, endpoint, provider) + router: Final = _two_deployment_router(model) + + response: Final = await getattr(router, endpoint)(model="test-model", **arguments) + + assert response is answer + assert keys_called == ["bad", "good"] + + +@pytest.mark.asyncio +async def test_weighted_failover_skips_the_failed_deployment_for_clientside_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + api_bases_called: list[object] = [] + answer: Final = object() + + async def provider(**kwargs: object) -> object: + api_bases_called.append(kwargs.get("api_base")) + if kwargs.get("api_base") == "http://a": + raise litellm.InternalServerError(message="A is down", llm_provider="openai", model="m") + return answer + + monkeypatch.setattr(litellm, "aembedding", provider) + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "http://a", "weight": 1}, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "http://b", "weight": 0}, + "model_info": {"id": "B"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=0, + disable_cooldowns=True, + enable_weighted_failover=True, + ) + + response: Final = await router.aembedding(model="test-model", input=["hi"], api_key="tenant-key") + + assert response is answer + assert api_bases_called == ["http://a", "http://b"] + + +@pytest.mark.parametrize( + ("endpoint", "attempt", "arguments"), + [ + ("embedding", "_embedding", {"input": ["hi"]}), + ("image_generation", "_image_generation", {"prompt": "a cat"}), + ], +) +def test_a_sync_attempt_stamps_the_failed_deployment( + monkeypatch: pytest.MonkeyPatch, endpoint: str, attempt: str, arguments: dict[str, object] +) -> None: + def provider(**kwargs: object) -> object: + raise litellm.InternalServerError(message="down", llm_provider="openai", model="m") + + monkeypatch.setattr(litellm, endpoint, provider) + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "openai/m", "api_key": "k"}, + "model_info": {"id": "A"}, + } + ], + ) + + with pytest.raises(litellm.InternalServerError) as raised: + getattr(router, attempt)(model="test-model", **arguments) + + assert getattr(raised.value, "failed_deployment_id", None) == "A" + + +@pytest.mark.asyncio +async def test_a_failed_pick_stamps_no_deployment() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "k"}, + "model_info": {"id": "A"}, + } + ], + ) + + with pytest.raises(litellm.BadRequestError, match="no healthy deployments") as raised: + await router._aembedding(input=["hi"], model="no-such-model") + + assert getattr(raised.value, "failed_deployment_id", None) is None diff --git a/tests/unit/test_together_ai_model_metadata.py b/tests/unit/test_together_ai_model_metadata.py index 3616c6216ca..8246277d475 100644 --- a/tests/unit/test_together_ai_model_metadata.py +++ b/tests/unit/test_together_ai_model_metadata.py @@ -64,7 +64,6 @@ DEPRECATED_MODELS: Final = { "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": "2026-02-06", "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": "2026-02-06", "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": "2026-02-03", - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": "2025-11-13", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": "2025-11-13", } diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 19b3f317927..e24499db071 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -108,6 +108,7 @@ from litellm.utils import ( client, get_non_default_completion_params, get_optional_params_image_gen, + get_optional_params_transcription, get_prompt_cache_min_tokens, get_supported_openai_params, get_token_count, @@ -967,21 +968,11 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { - "type": "number" - }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, @@ -1035,11 +1026,17 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, + "successor_model": {"type": "string"}, + "deprecation_source": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, "input_cost_per_audio_token": {"type": "number"}, "input_cost_per_audio_token_flex": {"type": "number"}, + "input_cost_per_audio_token_above_200k_tokens": {"type": "number"}, + "input_cost_per_audio_token_above_200k_tokens_flex": {"type": "number"}, + "input_cost_per_audio_token_above_200k_tokens_priority": {"type": "number"}, + "input_cost_per_audio_token_above_200k_tokens_batches": {"type": "number"}, "input_cost_per_image_token": {"type": "number"}, "input_cost_per_character": {"type": "number"}, "input_cost_per_character_above_128k_tokens": {"type": "number"}, @@ -1051,6 +1048,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "input_cost_per_token_above_100k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_200k_tokens": {"type": "number"}, "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_flex": {"type": "number"}, "input_cost_per_token_above_256k_tokens": {"type": "number"}, "input_cost_per_token_above_272k_tokens": {"type": "number"}, "input_cost_per_token_above_512k_tokens": {"type": "number"}, @@ -1081,6 +1079,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, "regional_processing_uplift_multiplier_us": {"type": "number"}, + "fedramp_uplift_multiplier": {"type": "number"}, "input_cost_per_pixel": {"type": "number"}, "input_cost_per_query": {"type": "number"}, "input_cost_per_request": {"type": "number"}, @@ -1150,6 +1149,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "output_cost_per_image_4K": {"type": "number"}, "output_cost_per_image_token": {"type": "number"}, "output_cost_per_image_token_batches": {"type": "number"}, + "output_cost_per_image_token_priority": {"type": "number"}, "output_cost_per_video_token": {"type": "number"}, "output_cost_per_pixel": {"type": "number"}, "output_cost_per_second": {"type": "number"}, @@ -1166,6 +1166,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "output_cost_per_token_above_128k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_flex": {"type": "number"}, "output_cost_per_token_above_256k_tokens": {"type": "number"}, "output_cost_per_token_above_272k_tokens": {"type": "number"}, "output_cost_per_token_above_512k_tokens": {"type": "number"}, @@ -1243,6 +1244,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_anthropic_thinking_payload": {"type": "boolean"}, "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, + "thinking_on_by_default": {"type": "boolean"}, "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, @@ -1279,6 +1281,9 @@ _COST_MAP_ROW_SCHEMA: Final = { "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls", "/v1/images/variations", "/v1/images/edits", "/v1/batch", @@ -1451,6 +1456,27 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): raise AssertionError(error_message) +def test_cost_map_schema_accepts_vertex_modality_and_context_tiers() -> None: + row: Final = { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens_flex": 3e-6, + "output_cost_per_token_above_200k_tokens_flex": 4e-6, + "input_cost_per_token_above_200k_tokens_batches": 3e-6, + "output_cost_per_token_above_200k_tokens_batches": 4e-6, + "input_cost_per_audio_token_above_200k_tokens": 5e-6, + "input_cost_per_audio_token_above_200k_tokens_flex": 6e-6, + "input_cost_per_audio_token_above_200k_tokens_priority": 7e-6, + "input_cost_per_audio_token_above_200k_tokens_batches": 8e-6, + "output_cost_per_image_token_priority": 9e-6, + "output_cost_per_image_token_batches": 10e-6, + } + + assert Draft7Validator(_COST_MAP_ROW_SCHEMA).is_valid({"synthetic-vertex-tier-model": row}) + + def test_cost_map_schema_accepts_12hr_cache_write_rates_for_batch_and_ultrafast_tiers() -> None: row: Final = { "litellm_provider": "openai", @@ -1671,6 +1697,38 @@ def test_get_provider_rerank_config(): assert isinstance(config, HostedVLLMRerankConfig) +def test_get_provider_video_config_resolves_a_custom_provider_map_base_video_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A `BaseVideoConfig` registered in `custom_provider_map` answers for its provider name, a `CustomLLM` + registered there does not, and a built-in name still resolves to its built-in config even when a `CustomLLM` + shadows it in the map.""" + from litellm.llms.custom_llm import CustomLLM + from litellm.llms.openai.videos.transformation import OpenAIVideoConfig + + class QueueVideoConfig(OpenAIVideoConfig): + pass + + video_config: Final = QueueVideoConfig() + monkeypatch.setattr( + litellm, + "custom_provider_map", + [ + {"provider": "queue-video", "custom_handler": video_config}, + {"provider": "image-only", "custom_handler": CustomLLM()}, + {"provider": "openai", "custom_handler": CustomLLM()}, + ], + ) + monkeypatch.setattr(litellm, "_custom_providers", ["queue-video", "image-only", "openai"]) + + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="queue-video") is video_config + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="image-only") is None + assert isinstance( + ProviderConfigManager.get_provider_video_config(model="sora-2", provider="openai"), OpenAIVideoConfig + ) + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="not-registered") is None + + def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts(): """Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS dropped response_format before the speech_to_completion bridge could honor it.""" @@ -1855,7 +1913,7 @@ class TestProxyFunctionCalling: ("gpt-4-turbo", "litellm_proxy/gpt-4-turbo", True), ("gpt-4-1106-preview", "litellm_proxy/gpt-4-1106-preview", True), # Azure OpenAI models - ("azure/gpt-4", "litellm_proxy/azure/gpt-4", True), + ("azure/gpt-4.1", "litellm_proxy/azure/gpt-4.1", True), ("azure/gpt-3.5-turbo", "litellm_proxy/azure/gpt-3.5-turbo", True), ( "azure/gpt-4-1106-preview", @@ -1948,7 +2006,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/together-mistral", - "together_ai/mistralai/Mistral-7B-Instruct-v0.1", + "together_ai/mistralai/Mistral-7B-Instruct-v0.3", False, ), # Ollama with custom names (cannot be resolved) @@ -2365,7 +2423,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_custom_prices_price_the_call_without_writing_the_cost_map() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2408,8 +2466,9 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.001 + 5 * 0.002) + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018) @@ -7572,9 +7631,10 @@ async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_a side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)] ) chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) - with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises( - (litellm.AuthenticationError, litellm.ServiceUnavailableError) - ) as excinfo: + with ( + caplog.at_level(logging.WARNING, logger="LiteLLM"), + pytest.raises((litellm.AuthenticationError, litellm.ServiceUnavailableError)) as excinfo, + ): await _ping_vllm_through(call_path, "vllm-sa") sent_while_failing: Final = chat.call_count await _ping_vllm_through(call_path, "vllm-sa") @@ -7823,9 +7883,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( try: with respx.mock(assert_all_called=True) as upstream: upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) - await router.acompletion( - model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} - ) + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) sent: Final = upstream.calls.last.request.headers["Authorization"] finally: router.discard() @@ -7893,7 +7951,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config with respx.mock(assert_all_called=False) as upstream: upstream.route().respond(200, json=_BEDROCK_REPLY) await router.acompletion( - model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + model="claude", + messages=_PING, + aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com", ) sent: Final = tuple( (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls @@ -8102,7 +8162,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials ( lambda reference: LiteLLM_Params( **litellm.Router( - model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}} + ] ).get_model_list(model_name="openai/*")[0]["litellm_params"] ), lambda reference: "Bearer listing-token", @@ -9108,7 +9170,9 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"][ + "max_input_tokens" + ] def test_multiple_messages_trimming(): @@ -9350,6 +9414,23 @@ def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_li assert (listings, catalog.call_count) == (([], []), 1) +@respx.mock +def test_get_valid_models_does_not_ask_a_timed_out_hosted_vllm_server_again_on_the_next_listing( + reset_mock_cache: None, +) -> None: + models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( + side_effect=httpx.ReadTimeout("vLLM did not answer") + ) + deployment: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=deployment) + for _ in range(2) + ) + + assert (listings, models_route.call_count) == (([], []), 1) + + @respx.mock def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None: respx.get("https://openrouter.ai/api/v1/models/user").mock( @@ -11051,6 +11132,48 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() - assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig) +@pytest.mark.parametrize( + ("custom_llm_provider", "model"), + [ + ("groq", "whisper-large-v3"), + ("deepgram", "nova-3"), + ("mistral", "voxtral-mini-latest"), + ("elevenlabs", "scribe_v1"), + ("fireworks_ai", "whisper-v3"), + ], +) +def test_openai_sdk_transcription_params_pass_through_to_other_providers(custom_llm_provider: str, model: str) -> None: + hints: Final = {"stream": True, "languages": ["en"], "keywords": ["LiteLLM"]} + + params: Final = get_optional_params_transcription(model=model, custom_llm_provider=custom_llm_provider, **hints) + + forwarded: Final = {**params.get("extra_body", {}), **params} + assert {key: forwarded.get(key) for key in hints} == hints + + +@respx.mock +def test_drop_params_keeps_openai_sdk_transcription_params_out_of_a_deepgram_request() -> None: + route: Final = respx.post(url__startswith="https://api.deepgram.com/v1/listen").mock( + return_value=httpx.Response( + 200, + json={"metadata": {"duration": 1.0}, "results": {"channels": [{"alternatives": [{"transcript": "hi"}]}]}}, + ) + ) + + litellm.transcription( + model="deepgram/nova-3", + file=("sample.wav", b"audio"), + api_key="dg-test", + punctuate=True, + stream=True, + languages=["en"], + keywords=["LiteLLM"], + drop_params=True, + ) + + assert dict(route.calls.last.request.url.params) == {"model": "nova-3", "punctuate": "true"} + + @pytest.mark.parametrize( ("model", "optional_params", "expected_response_format"), [ @@ -11342,6 +11465,26 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): assert prefixed_info["key"] == f"vertex_ai/{model}", model +def test_the_catalog_with_register_model_entries_lists_each_model_once_and_leaves_router_registrations_out() -> None: + from litellm.utils import catalog_with_runtime_registrations + + catalog: Final = catalog_with_runtime_registrations() + catalog_size: Final = len(catalog) + catalogued_model: Final = next(iter(catalog)) + catalogued_entry: Final = dict(catalog[catalogued_model]) + litellm.register_model({catalogued_model: {"supports_prompt_cache_breakpoint": True}}) + litellm.register_model({"qa-registered-only-model": {"litellm_provider": "openai", "mode": "chat"}}) + litellm.register_model({"qa-router-deployment-id": {"mode": "chat"}}, persist_across_reloads=False) + + merged: Final = catalog_with_runtime_registrations() + listed: Final = list(merged) + + assert len(listed) == len(merged) == catalog_size + 1 + assert "qa-registered-only-model" in listed + assert "qa-router-deployment-id" not in listed + assert dict(merged[catalogued_model]) == {**catalogued_entry, "supports_prompt_cache_breakpoint": True} + + @pytest.mark.parametrize( "key", ("openai.test-converse", "bedrock/openai.test-converse", "bedrock/converse/openai.test-converse") ) @@ -11410,7 +11553,9 @@ def test_declared_mid_conversation_system_support_prefers_the_deployment_over_th ) -def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") litellm.get_model_info.cache_clear() assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None @@ -11431,7 +11576,6 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} - @pytest.mark.parametrize( ("provider", "expected_warnings"), (("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)), diff --git a/tests/unit/videos/test_main.py b/tests/unit/videos/test_main.py index 22e1e5c05eb..76e9b062476 100644 --- a/tests/unit/videos/test_main.py +++ b/tests/unit/videos/test_main.py @@ -32,7 +32,7 @@ reflect production. from contextlib import ExitStack from dataclasses import dataclass -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import MagicMock, patch import pytest @@ -205,9 +205,7 @@ def test_video_edit__dispatch_and_provider_from_id(seams): def test_video_extension__dispatch_and_provider_from_id(seams): - result = videos_main.video_extension( - video_id=AZURE_VIDEO_ID, prompt="continue", seconds="5" - ) + result = videos_main.video_extension(video_id=AZURE_VIDEO_ID, prompt="continue", seconds="5") seams.assert_only("video_extension_handler") assert result is seams.handler.video_extension_handler.return_value @@ -285,9 +283,7 @@ def test_generation__mock_response_short_circuits(seams): def test_list__mock_response_short_circuits(seams): - resp = videos_main.video_list( - mock_response=[{"id": "v1", "object": "video", "status": "completed"}] - ) + resp = videos_main.video_list(mock_response=[{"id": "v1", "object": "video", "status": "completed"}]) assert isinstance(resp, list) assert resp[0].id == "v1" @@ -318,7 +314,7 @@ def test_get_character__mock_response_short_circuits(seams): def test_unsupported_provider_raises_without_dispatch(seams): seams.get_config.return_value = None - with pytest.raises(litellm.APIConnectionError): + with pytest.raises(litellm.BadRequestError, match="video status is not supported for azure"): videos_main.video_status(video_id=AZURE_VIDEO_ID) seams.handler.video_status_handler.assert_not_called() @@ -333,9 +329,7 @@ def test_unsupported_provider_raises_without_dispatch(seams): async def test_avideo_generation__delegates_with_async_flag(): sentinel = VideoObject(id="v-async", object="video", status="queued") with ( - patch.object( - videos_main, "video_generation", MagicMock(return_value=sentinel) - ) as sync, + patch.object(videos_main, "video_generation", MagicMock(return_value=sentinel)) as sync, patch.object( litellm, "get_llm_provider", @@ -352,9 +346,7 @@ async def test_avideo_generation__delegates_with_async_flag(): @pytest.mark.asyncio async def test_avideo_status__delegates_untouched(): sentinel = VideoObject(id="v-async", object="video", status="queued") - with patch.object( - videos_main, "video_status", MagicMock(return_value=sentinel) - ) as sync: + with patch.object(videos_main, "video_status", MagicMock(return_value=sentinel)) as sync: result = await videos_main.avideo_status(video_id="video_plain") assert result is sentinel @@ -367,9 +359,7 @@ async def test_avideo_content__pre_decodes_provider_before_delegating(): """avideo_content resolves the provider from the encoded id itself before handing off, so the sync worker receives the decoded provider, not None.""" sentinel = b"mp4-bytes" - with patch.object( - videos_main, "video_content", MagicMock(return_value=sentinel) - ) as sync: + with patch.object(videos_main, "video_content", MagicMock(return_value=sentinel)) as sync: result = await videos_main.avideo_content(video_id=AZURE_VIDEO_ID) assert result is sentinel @@ -393,9 +383,7 @@ DB_YAML_CREDS = { CREDENTIAL_OPERATIONS = [ ( "video_generation_handler", - lambda: videos_main.video_generation( - prompt="p", model="sora-2", **DB_YAML_CREDS - ), + lambda: videos_main.video_generation(prompt="p", model="sora-2", **DB_YAML_CREDS), ), ( "video_status_handler", @@ -407,21 +395,15 @@ CREDENTIAL_OPERATIONS = [ ), ( "video_remix_handler", - lambda: videos_main.video_remix( - video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS - ), + lambda: videos_main.video_remix(video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS), ), ( "video_edit_handler", - lambda: videos_main.video_edit( - video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS - ), + lambda: videos_main.video_edit(video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS), ), ( "video_extension_handler", - lambda: videos_main.video_extension( - video_id=AZURE_VIDEO_ID, prompt="p", seconds="5", **DB_YAML_CREDS - ), + lambda: videos_main.video_extension(video_id=AZURE_VIDEO_ID, prompt="p", seconds="5", **DB_YAML_CREDS), ), ( "video_list_handler", @@ -429,9 +411,7 @@ CREDENTIAL_OPERATIONS = [ ), ( "video_create_character_handler", - lambda: videos_main.video_create_character( - name="hero", video=MagicMock(name="vid"), **DB_YAML_CREDS - ), + lambda: videos_main.video_create_character(name="hero", video=MagicMock(name="vid"), **DB_YAML_CREDS), ), ( "video_get_character_handler", @@ -453,3 +433,33 @@ def test_db_yaml_credentials_reach_every_handler(seams, handler_name, invoke): assert litellm_params.get("api_base") == DB_YAML_CREDS["api_base"] assert litellm_params.get("api_version") == DB_YAML_CREDS["api_version"] assert litellm_params.get("vertex_project") == DB_YAML_CREDS["vertex_project"] + + +def test_video_generation__custom_provider_map_base_video_config_is_dispatched(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.openai.videos.transformation import OpenAIVideoConfig + + class QueueVideoConfig(OpenAIVideoConfig): + pass + + video_config: Final = QueueVideoConfig() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "queue-video", "custom_handler": video_config}]) + monkeypatch.setattr(litellm, "_custom_providers", ["queue-video"]) + handler: Final = MagicMock(spec=BaseLLMHTTPHandler) + with patch.object(videos_main, "base_llm_http_handler", handler): + videos_main.video_generation(model="queue-video/seedance-1", prompt="a cat", api_base="http://backend.test") + kw: Final = handler.video_generation_handler.call_args.kwargs + assert kw["video_generation_provider_config"] is video_config + assert kw["custom_llm_provider"] == "queue-video" + assert kw["model"] == "seedance-1" + + +def test_video_generation__custom_provider_without_a_video_config_is_a_400(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.custom_llm import CustomLLM + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "image-only", "custom_handler": CustomLLM()}]) + monkeypatch.setattr(litellm, "_custom_providers", ["image-only"]) + handler: Final = MagicMock(spec=BaseLLMHTTPHandler) + with patch.object(videos_main, "base_llm_http_handler", handler): + with pytest.raises(litellm.BadRequestError, match="video generation is not supported for image-only"): + videos_main.video_generation(model="image-only/seedance-1", prompt="a cat") + handler.video_generation_handler.assert_not_called() diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 0cadc13a213..dd428b286bd 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1081,6 +1081,11 @@ "count": 1 } }, + "src/app/(dashboard)/usage/_components/components/UsagePageView.tsx": { + "max-lines": { + "count": 1 + } + }, "src/app/(dashboard)/users/_components/BulkEditUsers.tsx": { "prefer-const": { "count": 1 @@ -1153,16 +1158,6 @@ "count": 2 } }, - "src/app/model_hub/page.tsx": { - "react-hooks/set-state-in-effect": { - "count": 1 - } - }, - "src/app/model_hub_table/page.tsx": { - "react-hooks/set-state-in-effect": { - "count": 1 - } - }, "src/components/AIHub/ModelHubTable.test.tsx": { "max-params": { "count": 1 @@ -1214,6 +1209,11 @@ "count": 1 } }, + "src/components/ModelInfoEditForm.tsx": { + "max-lines": { + "count": 1 + } + }, "src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx": { "max-nested-callbacks": { "count": 10 @@ -1324,6 +1324,11 @@ "count": 1 } }, + "src/components/add_model/ComplexityRouterConfig.tsx": { + "max-lines": { + "count": 1 + } + }, "src/components/add_model/RouterConfigBuilder.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -1349,6 +1354,11 @@ "count": 1 } }, + "src/components/add_model/build_complexity_router_config.ts": { + "max-lines": { + "count": 1 + } + }, "src/components/add_model/cache_control_settings.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2010,6 +2020,9 @@ "src/components/templates/key_edit_view.tsx": { "local/filename-pascal-case": { "count": 1 + }, + "max-lines": { + "count": 1 } }, "src/components/templates/key_info_view.tsx": { diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 54cd9087c4e..d506d114d8c 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -23,6 +23,7 @@ "@types/papaparse": "5.5.2", "class-variance-authority": "0.7.1", "clsx": "^2.1.1", + "cmdk": "1.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", "i18next": "26.5.11", @@ -2656,6 +2657,323 @@ "dev": true, "license": "MIT" }, + "node_modules/@radix-ui/primitive": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.2.0.tgz", + "integrity": "sha512-KABY+VMIRPJL+Vyzv/335FEhXXD/xDv8BMb1kl48/aXJZMH0LtHCTx5ga5RnxaEP3nZHtmTLf1P9vanmFX46bw==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.3.0.tgz", + "integrity": "sha512-v5AYEocMISkbJ+1fTXzZpiFv7NRr3D2WiYbLIKf+FkmJ8VkP6d503eV2n34MFussHXY1pcoq3yfJiQH77dc4Mg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.3.1.tgz", + "integrity": "sha512-+0E4qvVTwjAWra9sEhJ+9BTsvHE00zkUuNw1VGh7S+Gc13dfYk+S1PVQdchWDj+XYHhAai7Z7IpzXVDCGsDplg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.2.0", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.3.0", + "@radix-ui/react-dismissable-layer": "1.2.1", + "@radix-ui/react-focus-guards": "1.2.0", + "@radix-ui/react-focus-scope": "1.3.1", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.2.0", + "@radix-ui/react-presence": "1.1.11", + "@radix-ui/react-primitive": "2.2.0", + "@radix-ui/react-slot": "1.5.0", + "@radix-ui/react-use-controllable-state": "1.3.0", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.2.1.tgz", + "integrity": "sha512-KIfDcAsx+oVylFZ4xryVKusJUrLLhTQmfnJW4kwk4Km5myvhHCrj04hnWWUC166+xD4DHdH8tQGylwYX1h6+Pg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.2.0", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.2.0", + "@radix-ui/react-use-callback-ref": "1.1.5", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.2.0.tgz", + "integrity": "sha512-MPm5R0iRGUDrDnPNxmLBkv+X7pN4XKzt+MbWJEuJQsH3NFH4omrbpXpucyYCCKTSt+na/ZxGhEWQrWTZ0285cA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.3.1.tgz", + "integrity": "sha512-ugeMhbXt4y5S2T1GnIqbWDKEN962PDwnX1fN0eKOOgf5f63Dh2mn8F1FUySlgW0iF6GiVsRF32DzuDIWTUyQJg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.2.0", + "@radix-ui/react-use-callback-ref": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.2.0.tgz", + "integrity": "sha512-StX7xXnaRU41VvFtMm9zlhVGz8XF6GmWWTXftkQaY+tqVkbs/R9UmAp5mLzHHapk5hM9RMZQvO/3usyIP/g42g==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.2.0", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.11.tgz", + "integrity": "sha512-Z7/atlkab36Z8hm+V8zYc79r0W+tiOB1o0tun1By94JyadkekLQmtEAnBSP9XKXBNQsStoStS4+gX10JpS5akA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.2.0.tgz", + "integrity": "sha512-HCTSPMNqv55Qk0IX4geJbpyXazmqulcimwuyjxTkKi29v+QZbG2Zho9BnbBq4YS2PF+KlNb2CXoLz66uUmblkg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.5.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.5.0.tgz", + "integrity": "sha512-tyYd9ddPXAysAbdbD9eYxgwpCSv6f3PncqVSNKQFSR5lcJVLz3cb/pANDoYrLiHBpXv7skortyUgGUWsYuPPFQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.5.tgz", + "integrity": "sha512-Y5GBf4NKeADoXI1geiMlMuMoURT3k9MizzzKhGb/m55H94aGigZ01PIpkvsO6lML13BA6XYAXSJ6gza7vTn5nw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.3.0.tgz", + "integrity": "sha512-TAcBI7bw2ljYCbv4+zuTSd6zICu8pRt6SDN2vJ/Z06K2MXh6I9WYFe/A9ovlFDtR5HVHGm5iDxDoABm/++dbKg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.2.0", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@redocly/ajv": { "version": "8.11.2", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", @@ -3939,7 +4257,7 @@ "version": "19.2.4", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", - "dev": true, + "devOptional": true, "license": "MIT", "peerDependencies": { "@types/react": "^19.2.0" @@ -4757,6 +5075,18 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -5308,6 +5638,22 @@ "node": ">=6" } }, + "node_modules/cmdk": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz", + "integrity": "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "^1.1.1", + "@radix-ui/react-dialog": "^1.1.6", + "@radix-ui/react-id": "^1.1.0", + "@radix-ui/react-primitive": "^2.0.2" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "react-dom": "^18 || ^19 || ^19.0.0-rc" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -5746,6 +6092,12 @@ "node": ">=8" } }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "license": "MIT" + }, "node_modules/devlop": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", @@ -6959,6 +7311,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -10769,6 +11130,53 @@ } } }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/react-resizable-panels": { "version": "4.14.1", "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-4.14.1.tgz", @@ -10779,6 +11187,28 @@ "react-dom": "^18.0.0 || ^19.0.0" } }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/react-syntax-highlighter": { "version": "15.6.6", "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz", @@ -12408,6 +12838,49 @@ "dev": true, "license": "MIT" }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/use-sync-external-store": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 0707496a6bc..9fc0242f040 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -40,6 +40,7 @@ "@types/papaparse": "5.5.2", "class-variance-authority": "0.7.1", "clsx": "^2.1.1", + "cmdk": "1.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", "i18next": "26.5.11", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx index a4a615fe798..f3d9b67eb4b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx @@ -1,5 +1,7 @@ import { useAccessGroupDetails } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails"; import { AccessGroupResponse } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups"; +import { useUserLookup } from "@/app/(dashboard)/hooks/users/useUsers"; +import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels"; import { screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,6 +9,7 @@ import { renderWithProviders } from "../../../../../tests/test-utils"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails"); +vi.mock("@/app/(dashboard)/hooks/users/useUsers"); vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); vi.mock("./AccessGroupsModal/AccessGroupEditModal", () => ({ AccessGroupEditModal: ({ visible, onCancel }: { visible: boolean; onCancel: () => void }) => @@ -18,6 +21,7 @@ vi.mock("./AccessGroupsModal/AccessGroupEditModal", () => ({ })); const mockUseAccessGroupDetails = vi.mocked(useAccessGroupDetails); +const mockUseUserLookup = vi.mocked(useUserLookup); const baseMockReturnValue = { data: undefined, @@ -89,6 +93,7 @@ describe("AccessGroupDetail", () => { ...baseMockReturnValue, data: createMockAccessGroup(), } as ReturnType); + mockUseUserLookup.mockReturnValue({ data: null } as ReturnType); }); it("should render the component", () => { @@ -144,6 +149,64 @@ describe("AccessGroupDetail", () => { expect(screen.getByText("A test access group")).toBeInTheDocument(); }); + describe("user references", () => { + it("shows a user's alias as a linked name and reveals the id in a tooltip", async () => { + const user = userEvent.setup(); + mockUseUserLookup.mockReturnValue({ + data: { user_alias: "Alice", user_email: "alice@example.com" }, + } as ReturnType); + renderWith({ created_by: "user-123" }); + + const link = screen.getByRole("link", { name: "Alice" }); + expect(link).toHaveAttribute("href", expect.stringContaining("/users?user=user-123")); + expect(screen.queryByText("user-123")).not.toBeInTheDocument(); + + await user.hover(link); + + expect(await screen.findByText("user-123")).toBeInTheDocument(); + }); + + it("shows the email when the user has no alias", () => { + mockUseUserLookup.mockReturnValue({ + data: { user_alias: null, user_email: "alice@example.com" }, + } as ReturnType); + renderWith({ updated_by: "user-456" }); + + expect(screen.getByRole("link", { name: "alice@example.com" })).toHaveAttribute( + "href", + expect.stringContaining("/users?user=user-456"), + ); + }); + + it("falls back to the email when the alias contains only whitespace", () => { + mockUseUserLookup.mockReturnValue({ + data: { user_alias: " ", user_email: "alice@example.com" }, + } as ReturnType); + renderWith({ created_by: "user-789" }); + + expect(screen.getByRole("link", { name: "alice@example.com" })).toHaveAttribute( + "href", + expect.stringContaining("/users?user=user-789"), + ); + }); + + it("links the user id when lookup has no user details", () => { + renderWith({ updated_by: "missing-user" }); + + expect(screen.getByRole("link", { name: "missing-user" })).toHaveAttribute( + "href", + expect.stringContaining("/users?user=missing-user"), + ); + }); + + it("shows the default admin badge without looking up its user id", () => { + renderWith({ created_by: DEFAULT_PROXY_ADMIN_USER_ID }); + + expect(screen.getByText("Default Proxy Admin")).toBeInTheDocument(); + expect(mockUseUserLookup).toHaveBeenCalledWith(null); + }); + }); + it("should display em dash when description is empty", () => { renderWith({ description: null }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx index f0529121b21..cd8a0863e92 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx @@ -12,9 +12,9 @@ import { } from "lucide-react"; import { useState } from "react"; import { useTranslation } from "react-i18next"; -import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag"; import { BadgeLink } from "@/components/shared/BadgeLink"; import CopyButton from "@/components/shared/CopyButton"; +import { UserReference } from "@/components/shared/UserReference"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardAction, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -190,7 +190,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr {accessGroup.created_by && ( <> {t("accessGroups.detail.by")} - + )} @@ -200,7 +200,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr {accessGroup.updated_by && ( <> {t("accessGroups.detail.by")} - + )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx new file mode 100644 index 00000000000..ec88a69670e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -0,0 +1,19 @@ +"use client"; + +import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; +import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; + +export default function AlertsPage() { + return ( + + + + Model deprecations + + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx index 1339300b2ba..7f6f7ea1d17 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx @@ -1,5 +1,6 @@ "use client"; import React from "react"; +import { useTranslation } from "react-i18next"; import CodeBlock from "@/components/CodeBlock"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DocLink from "./DocLink"; @@ -12,6 +13,7 @@ interface ApiRefProps { } const APIReferenceView: React.FC = ({ proxySettings }) => { + const { t } = useTranslation(); let base_url = ""; const customDocBaseUrl = proxySettings?.LITELLM_UI_API_DOC_BASE_URL; if (customDocBaseUrl && customDocBaseUrl.trim()) { @@ -25,14 +27,11 @@ const APIReferenceView: React.FC = ({ proxySettings }) => {
{/* Header row with Docs link on the right */}
-

OpenAI Compatible Proxy: API Reference

+

{t("apiReference.view.title")}

-

- LiteLLM is OpenAI Compatible. This means your API Key works with the OpenAI SDK. Just replace the base_url to - point to your litellm proxy. Example Below{" "} -

+

{t("apiReference.view.description")}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx index f5a9d5e3a15..cee174d7fdb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx @@ -1,5 +1,6 @@ import React from "react"; import { ExternalLink } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { cn } from "@/lib/cva.config"; @@ -9,21 +10,22 @@ export type DocLinkProps = { }; const DocLink = ({ href, className }: DocLinkProps) => { + const { t } = useTranslation(); return ( - API Reference Docs + {t("apiReference.docLink.label")} - (opens in a new tab) + {t("apiReference.docLink.newTab")} ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx index d7c977b0870..4d8498a1be4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx @@ -1,17 +1,19 @@ "use client"; +import { useTranslation } from "react-i18next"; import APIReferenceView from "./_components/APIReferenceView"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; const APIReferencePage = () => { + const { t } = useTranslation(); const { accessToken } = useAuthorized(); const proxySettings = useProxySettings(accessToken); return ( <> - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/aria/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/aria/page.tsx new file mode 100644 index 00000000000..45127adc65a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/aria/page.tsx @@ -0,0 +1,5 @@ +import { LiteAdminPage } from "@/components/liteadmin/LiteAdmin"; + +export default function AriaPage() { + return ; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx index 9ffd23884b4..56e2c6dbecc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx @@ -1,4 +1,5 @@ import React, { useId } from "react"; +import { useTranslation } from "react-i18next"; import { Controller } from "react-hook-form"; import { z } from "zod"; import { FormField } from "@/components/shared/form/FormField"; @@ -16,6 +17,7 @@ import { import { Input } from "@/components/ui/input"; import { InputGroup, InputGroupAddon, InputGroupInput, InputGroupText } from "@/components/ui/input-group"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import i18n from "@/i18n/i18n"; import { useZodForm } from "@/lib/forms/useZodForm"; const budgetShape = { @@ -54,19 +56,23 @@ const numberProps = ) => onChange(event.target.value === "" ? null : event.target.valueAsNumber), }); -const OptionalBadge = () => Optional; +const OptionalBadge = () => { + const { t } = useTranslation(); + return {t("budgets.form.optional")}; +}; function SpendLimitField({ form }: { form: BudgetForm }) { + const { t } = useTranslation(); const amountId = useId(); return ( - Spend limit + {t("budgets.form.spendLimit")} @@ -91,7 +97,7 @@ function SpendLimitField({ form }: { form: BudgetForm }) { value={field.value ?? NO_RESET} onValueChange={(next) => field.onChange(next === NO_RESET ? null : next)} > - + @@ -124,13 +130,14 @@ function RateLimitField({ form, name, label }: { form: BudgetForm; name: NumberF } export function BudgetFormFields({ form, isEditing }: { form: BudgetForm; isEditing: boolean }) { + const { t } = useTranslation(); return ( {({ ref, ...field }) => } @@ -139,15 +146,13 @@ export function BudgetFormFields({ form, isEditing }: { form: BudgetForm; isEdit
- Rate limits + {t("budgets.form.rateLimits")} - - - - - Batches count against the daily limit instead of the per-minute ones. Provider rate limits still apply. - + + + + {t("budgets.form.batchNote")}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx index bd10837640d..c6bf7e8121e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx @@ -1,4 +1,5 @@ import React, { useId } from "react"; +import { useTranslation } from "react-i18next"; import { useCreateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { applyBudgetPrecision } from "./budgetPrecision"; import { BudgetFormFields, type BudgetFormValues, useBudgetForm } from "./BudgetFormFields"; @@ -11,6 +12,7 @@ interface BudgetModalProps { setIsModalVisible: React.Dispatch>; } const BudgetModal: React.FC = ({ isModalVisible, setIsModalVisible }) => { + const { t } = useTranslation(); const formId = useId(); const form = useBudgetForm({ budget_id: "" }); const createBudget = useCreateBudget(); @@ -22,14 +24,14 @@ const BudgetModal: React.FC = ({ isModalVisible, setIsModalVis const handleCreate = async (formValues: BudgetFormValues) => { try { - toast.info("Making API Call"); + toast.info(t("budgets.createModal.toast.pending")); await createBudget.mutateAsync(applyBudgetPrecision(formValues)); - toast.success("Budget Created"); + toast.success(t("budgets.createModal.toast.created")); form.reset(); setIsModalVisible(false); } catch (error) { console.error("Error creating the budget:", error); - toast.fromError(`Error creating the budget: ${error}`); + toast.fromError(t("budgets.createModal.toast.error", { error: String(error) })); } }; @@ -37,15 +39,17 @@ const BudgetModal: React.FC = ({ isModalVisible, setIsModalVis !open && handleCancel()}> - Create budget + + {t("budgets.createModal.title")} +
- }>Cancel + }>{t("budgets.createModal.cancel")}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx index 785b09252af..bbc4fed94f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx @@ -6,6 +6,7 @@ import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import { Plus, Wallet } from "lucide-react"; import React, { useCallback, useState } from "react"; +import { useTranslation } from "react-i18next"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { prism } from "react-syntax-highlighter/dist/esm/styles/prism"; @@ -29,6 +30,7 @@ interface BudgetSettingsPageProps { } const BudgetPanel: React.FC = ({ accessToken }) => { + const { t } = useTranslation(); const syntaxTheme = useSyntaxTheme(prism); const [isCreateModelVisible, setIsCreateModelVisible] = useState(false); const [isEditModalVisible, setIsEditModalVisible] = useState(false); @@ -65,10 +67,10 @@ const BudgetPanel: React.FC = ({ accessToken }) => { } try { await deleteBudget.mutateAsync(selectedBudget.budget_id); - toast.success("Budget deleted."); + toast.success(t("budgets.panel.toast.deleted")); } catch (error) { console.error("Error deleting budget:", error); - toast.fromError("Failed to delete budget"); + toast.fromError(t("budgets.panel.toast.deleteFailed")); } finally { setIsDeleteModalVisible(false); setSelectedBudget(null); @@ -85,22 +87,22 @@ const BudgetPanel: React.FC = ({ accessToken }) => { - Budgets + {t("budgets.panel.title")} - Spend, TPM and RPM limits you can assign to customers. + {t("budgets.panel.description")} {canModify && ( <> )} - Budgets - Examples + {t("budgets.panel.tabs.budgets")} + {t("budgets.panel.tabs.examples")} @@ -122,15 +124,15 @@ const BudgetPanel: React.FC = ({ accessToken }) => { /> = ({ accessToken }) => {
-

How to use budget id

+

{t("budgets.panel.examples.intro")}

- Assign Budget to Customer + {t("budgets.panel.examples.tabs.assignBudget")} - Test it (Curl) + {t("budgets.panel.examples.tabs.curl")} - Test it (OpenAI SDK) + {t("budgets.panel.examples.tabs.openaiSdk")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx index 80872c58d28..9d9e32ed86e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.tsx @@ -2,6 +2,7 @@ import { Inbox, ShieldAlert } from "lucide-react"; import React, { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import { BUDGET_DURATION_FILTER_OPTIONS, @@ -21,6 +22,7 @@ import { import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import i18n from "@/i18n/i18n"; import { ApiError } from "@/lib/http/client"; import { BUDGET_TABLE_HIDDEN_COLUMNS, getBudgetTableColumns } from "./BudgetTableColumns"; @@ -49,11 +51,19 @@ const formatFilterValue = (columnId: string, value: unknown): string => { } if (columnId === "max_budget") { const { min, max, unlimitedOnly } = (value ?? {}) as MaxBudgetFilterValue; - return unlimitedOnly === true ? "Unlimited only" : `${min ? `$${min}` : "any"} to ${max ? `$${max}` : "any"}`; + return unlimitedOnly === true + ? i18n.t("budgets.table.filterChips.unlimitedOnly") + : i18n.t("budgets.table.filterChips.maxBudgetRange", { + min: min ? `$${min}` : i18n.t("budgets.table.filterChips.any"), + max: max ? `$${max}` : i18n.t("budgets.table.filterChips.any"), + }); } if (columnId === "created_at") { const { from, to } = (value ?? {}) as CreatedAtFilterValue; - return `${from || "any"} to ${to || "any"}`; + return i18n.t("budgets.table.filterChips.createdRange", { + from: from || i18n.t("budgets.table.filterChips.any"), + to: to || i18n.t("budgets.table.filterChips.any"), + }); } return String(value); }; @@ -81,22 +91,24 @@ const normalizeCreatedAt = (draft: CreatedAtFilterValue): CreatedAtFilterValue | }; function EmptyState({ hasQuery }: { hasQuery: boolean }) { + const { t } = useTranslation(); return (
-
{hasQuery ? "No matching budgets" : "No budgets yet"}
+
+ {hasQuery ? t("budgets.table.emptyState.noMatchTitle") : t("budgets.table.emptyState.title")} +
- {hasQuery - ? "No budget matches your search or filters." - : "Create a budget to set spend, TPM and RPM limits for customers."} + {hasQuery ? t("budgets.table.emptyState.noMatchDescription") : t("budgets.table.emptyState.description")}
); } function ErrorState({ error }: { error: Error }) { + const { t } = useTranslation(); const forbidden = error instanceof ApiError && error.status === 403; return (
@@ -104,10 +116,10 @@ function ErrorState({ error }: { error: Error }) {
- {forbidden ? "You do not have access to budgets" : "Could not load budgets"} + {forbidden ? t("budgets.table.errorState.forbiddenTitle") : t("budgets.table.errorState.title")}
- {forbidden ? "Ask a proxy admin to grant you the admin viewer role." : error.message} + {forbidden ? t("budgets.table.errorState.forbiddenDescription") : error.message}
); @@ -141,19 +153,20 @@ function DurationFilter({ selected, onChange }: { selected: string[]; onChange: } function BudgetFilterFields({ get, set }: FilterDraft) { + const { t } = useTranslation(); const maxBudget = (get("max_budget") as MaxBudgetFilterValue | undefined) ?? {}; const created = (get("created_at") as CreatedAtFilterValue | undefined) ?? {}; const unlimitedOnly = maxBudget.unlimitedOnly === true; return ( <> - + set("budget_duration", selected)} /> - +
set("max_budget", normalizeMaxBudget({ ...maxBudget, min: event.target.value }))} - placeholder="Min" - aria-label="Minimum max budget" + placeholder={t("budgets.table.filters.min")} + aria-label={t("budgets.table.filters.minAriaLabel")} data-testid="budget-filter-max-budget-min" /> set("max_budget", normalizeMaxBudget({ ...maxBudget, max: event.target.value }))} - placeholder="Max" - aria-label="Maximum max budget" + placeholder={t("budgets.table.filters.max")} + aria-label={t("budgets.table.filters.maxAriaLabel")} data-testid="budget-filter-max-budget-max" />
@@ -184,23 +197,23 @@ function BudgetFilterFields({ get, set }: FilterDraft) { onCheckedChange={(checked) => set("max_budget", normalizeMaxBudget({ unlimitedOnly: checked === true }))} data-testid="budget-filter-max-budget-unlimited" /> - Unlimited only + {t("budgets.table.filters.unlimitedOnly")}
- +
set("created_at", normalizeCreatedAt({ ...created, from: event.target.value }))} - aria-label="Created from" + aria-label={t("budgets.table.filters.createdFrom")} data-testid="budget-filter-created-from" /> set("created_at", normalizeCreatedAt({ ...created, to: event.target.value }))} - aria-label="Created to" + aria-label={t("budgets.table.filters.createdTo")} data-testid="budget-filter-created-to" />
@@ -210,11 +223,12 @@ function BudgetFilterFields({ get, set }: FilterDraft) { } const BudgetTable: React.FC = ({ list, canModify, onEditClick, onDeleteClick }) => { + const { t } = useTranslation(); const [filtersOpen, setFiltersOpen] = useState(false); const columns = useMemo( - () => getBudgetTableColumns({ canModify, onEditClick, onDeleteClick }), - [canModify, onEditClick, onDeleteClick], + () => getBudgetTableColumns({ canModify, onEditClick, onDeleteClick }, t), + [canModify, onEditClick, onDeleteClick, t], ); const hasQuery = list.searchValue.trim() !== "" || list.columnFilters.length > 0; @@ -239,7 +253,7 @@ const BudgetTable: React.FC = ({ list, canModify, onEditClick, columnFilters={list.columnFilters} onColumnFiltersChange={list.onColumnFiltersChange} isLoading={list.isLoading} - loadingMessage="Loading budgets…" + loadingMessage={t("budgets.table.loading")} noDataMessage={emptyMessage} size="compact" toolbar={(table) => ( @@ -248,7 +262,7 @@ const BudgetTable: React.FC = ({ list, canModify, onEditClick, table={table} searchValue={list.searchValue} onSearchChange={list.onSearchChange} - searchPlaceholder="Search by budget ID…" + searchPlaceholder={t("budgets.table.searchPlaceholder")} onOpenFilters={() => setFiltersOpen(true)} onRefresh={list.refetch} isRefreshing={list.isFetching} @@ -259,8 +273,8 @@ const BudgetTable: React.FC = ({ list, canModify, onEditClick, table={table} open={filtersOpen} onOpenChange={setFiltersOpen} - title="Filters" - description="Narrow down your budgets" + title={t("budgets.table.filters.title")} + description={t("budgets.table.filters.description")} > {(draft) => } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index 3ec291c1158..d0538e7ed1c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -1,7 +1,9 @@ "use client"; import { ColumnDef, FilterFn } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; @@ -27,15 +29,17 @@ const serverFilter: FilterFn = () => true; serverFilter.autoRemove = () => false; function RateLimitCell({ value }: { value: number | null | undefined }) { + const { t } = useTranslation(); if (value == null) { - return n/a; + return {t("budgets.table.cells.notApplicable")}; } return {value}; } function BudgetDurationCell({ value }: { value: string | null | undefined }) { + const { t } = useTranslation(); if (!value) { - return Not set; + return {t("budgets.table.cells.notSet")}; } return {getBudgetDurationLabel(value)}; } @@ -47,10 +51,11 @@ interface BudgetRowActionsProps { } function BudgetRowActions({ budget, onEditClick, onDeleteClick }: BudgetRowActionsProps) { + const { t } = useTranslation(); return ( @@ -59,7 +64,7 @@ function BudgetRowActions({ budget, onEditClick, onDeleteClick }: BudgetRowActio onEditClick(budget)}> - Edit budget + {t("budgets.table.rowActions.edit")} onDeleteClick(budget)} > - Delete budget + {t("budgets.table.rowActions.delete")} @@ -87,16 +92,15 @@ interface BudgetTableColumnsDeps { onDeleteClick: (budget: budgetItem) => void; } -export const getBudgetTableColumns = ({ - canModify, - onEditClick, - onDeleteClick, -}: BudgetTableColumnsDeps): ColumnDef[] => [ +export const getBudgetTableColumns = ( + { canModify, onEditClick, onDeleteClick }: BudgetTableColumnsDeps, + t: TFunction, +): ColumnDef[] => [ { id: "budget_id", accessorKey: "budget_id", - meta: { title: "Budget ID" }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.budgetId") }, + header: ({ column }) => , cell: ({ row }) => ( ), @@ -105,32 +109,34 @@ export const getBudgetTableColumns = ({ id: "max_budget", accessorKey: "max_budget", filterFn: serverFilter, - meta: { title: "Max Budget", numeric: true }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.maxBudget"), numeric: true }, + header: ({ column }) => , size: 120, - cell: ({ row }) => , + cell: ({ row }) => ( + + ), }, { id: "tpm_limit", accessorKey: "tpm_limit", - meta: { title: "TPM", numeric: true }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.tpm"), numeric: true }, + header: ({ column }) => , size: 100, cell: ({ row }) => , }, { id: "rpm_limit", accessorKey: "rpm_limit", - meta: { title: "RPM", numeric: true }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.rpm"), numeric: true }, + header: ({ column }) => , size: 100, cell: ({ row }) => , }, { id: "tpd_limit", accessorKey: "tpd_limit", - meta: { title: "TPD (batch)", numeric: true }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.tpd"), numeric: true }, + header: ({ column }) => , size: 110, cell: ({ row }) => , }, @@ -138,9 +144,9 @@ export const getBudgetTableColumns = ({ id: "budget_duration", accessorKey: "budget_duration", filterFn: serverFilter, - meta: { title: "Reset" }, + meta: { title: t("budgets.table.columns.reset") }, sortDescFirst: false, - header: ({ column }) => , + header: ({ column }) => , size: 110, cell: ({ row }) => , }, @@ -148,8 +154,8 @@ export const getBudgetTableColumns = ({ id: "created_at", accessorKey: "created_at", filterFn: serverFilter, - meta: { title: "Created" }, - header: ({ column }) => , + meta: { title: t("budgets.table.columns.created") }, + header: ({ column }) => , size: 160, cell: ({ row }) => , }, @@ -158,7 +164,7 @@ export const getBudgetTableColumns = ({ { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {t("budgets.table.columns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx index 4aa69c5c15e..81c1f652eab 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx @@ -1,4 +1,5 @@ import React, { useEffect, useId } from "react"; +import { useTranslation } from "react-i18next"; import { useUpdateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { applyBudgetPrecision } from "./budgetPrecision"; @@ -23,6 +24,7 @@ interface EditBudgetModalProps { existingBudget: budgetItem; } const EditBudgetModal: React.FC = ({ isModalVisible, setIsModalVisible, existingBudget }) => { + const { t } = useTranslation(); const formId = useId(); const form = useBudgetForm(toFormValues(existingBudget)); const { dirtyFields } = form.formState; @@ -39,15 +41,15 @@ const EditBudgetModal: React.FC = ({ isModalVisible, setIs const handleUpdate = async (formValues: BudgetFormValues) => { try { - toast.info("Making API Call"); + toast.info(t("budgets.editModal.toast.pending")); const changedValues = pickDirty(formValues, dirtyFields); await updateBudget.mutateAsync(applyBudgetPrecision({ budget_id: formValues.budget_id, ...changedValues })); - toast.success("Budget Updated"); + toast.success(t("budgets.editModal.toast.updated")); form.reset(); setIsModalVisible(false); } catch (error) { console.error("Error updating the budget:", error); - toast.fromError(`Error updating the budget: ${error}`); + toast.fromError(t("budgets.editModal.toast.error", { error: String(error) })); } }; @@ -55,15 +57,15 @@ const EditBudgetModal: React.FC = ({ isModalVisible, setIs !open && handleCancel()}> - Edit budget + {t("budgets.editModal.title")}
- }>Cancel + }>{t("budgets.editModal.cancel")}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx index 9e9c2ad7fc8..6aee2bf7dfd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useState } from "react"; +import { useTranslation } from "react-i18next"; import { CircleAlert } from "lucide-react"; import { z } from "zod"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; @@ -17,21 +18,23 @@ import { useZodForm } from "@/lib/forms/useZodForm"; import { toast } from "@/lib/toast"; import { revokeSessionAndClearClientState } from "@/app/(dashboard)/hooks/useLogout"; import { getLoginUrl } from "@/utils/returnUrlUtils"; +import i18n from "@/i18n/i18n"; const changePasswordSchema = z .object({ - currentPassword: z.string().min(1, "Current password is required"), - newPassword: z.string().min(1, "New password is required"), - confirmNewPassword: z.string().min(1, "Confirm your new password"), + currentPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.currentRequired") }), + newPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.newRequired") }), + confirmNewPassword: z.string().min(1, { error: () => i18n.t("changePassword.schema.confirmRequired") }), }) .refine((values) => values.newPassword === values.confirmNewPassword, { - message: "New passwords do not match", + error: () => i18n.t("changePassword.schema.mismatch"), path: ["confirmNewPassword"], }); type ChangePasswordValues = z.infer; export function ChangePasswordForm() { + const { t } = useTranslation(); const { accessToken, passwordResetRequired } = useAuthorized(); const form = useZodForm(changePasswordSchema, { defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" }, @@ -49,12 +52,12 @@ export function ChangePasswordForm() { // The session key was minted restricted; only a fresh login lifts it. // Revoke it server-side too (best-effort) so it doesn't sit valid // until the expiry reaper gets to it. - toast.success("Password updated. Please log in with your new password."); + toast.success(t("changePassword.form.toasts.updatedSignIn")); await revokeSessionAndClearClientState(accessToken); window.location.replace(getUiPageUrl(getLoginUrl(getProxyBaseUrl()))); return; } - toast.success("Password updated"); + toast.success(t("changePassword.form.toasts.updated")); form.reset(); } catch (error) { setSubmitError(extractProxyErrorMessage(error)); @@ -67,34 +70,31 @@ export function ChangePasswordForm() {
-

Change Password

-

- Enter your current password and choose a new one. The new password must meet this proxy's password - policy. -

+

{t("changePassword.form.title")}

+

{t("changePassword.form.description")}

{passwordResetRequired && ( - - Your password must be changed before you can use the dashboard: it was either found in a known data - breach or set by an administrator as a temporary password. After updating it, you will be signed out to - log in again. - + {t("changePassword.form.resetRequired")} )}
- + {({ ref, ...field }) => } - + {({ ref, ...field }) => } - + {({ ref, ...field }) => } @@ -108,8 +108,10 @@ export function ChangePasswordForm() {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index b3acb8f9168..a6823d50b21 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels"; @@ -76,20 +77,21 @@ const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued? ); const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { + const { t } = useTranslation(); const stats = view.stats; const cheaper = stats.saved_pct != null && stats.saved_pct >= 0; const classifierCost = stats.baseline_spend == null ? null : stats.savings_estimated_classifier_cost ?? null; const comparedAll = stats.savings_estimated_turns === stats.turns; return ( - +

- Total estimated savings + {t("costOptimization.autoRouter.hero.totalSavings")}

- {stats.saved_spend == null ? "Unavailable" : usd(stats.saved_spend)} + {stats.saved_spend == null ? t("costOptimization.autoRouter.unavailable") : usd(stats.saved_spend)}

{stats.saved_pct != null && ( = ({ view }) => {
{stats.baseline_spend != null && !comparedAll && (

- Compared on {stats.savings_estimated_turns.toLocaleString()} of {stats.turns.toLocaleString()} requests; - adaptive and quality routers are excluded + {t("costOptimization.autoRouter.hero.comparedOn", { + compared: stats.savings_estimated_turns.toLocaleString(), + total: stats.turns.toLocaleString(), + })}

)} {stats.unattributed_saved_spend != null && (

- Per-router records differ from recorded savings by {usd(Math.abs(stats.unattributed_saved_spend))}, for - example history from before per-router tracking, so the baseline comparison is unavailable + {t("costOptimization.autoRouter.hero.unattributed", { + amount: usd(Math.abs(stats.unattributed_saved_spend)), + })}

)}
{stats.baseline_spend != null && classifierCost == null && (

- Breakdown unavailable because some usage predates classification-cost tracking. + {t("costOptimization.autoRouter.hero.breakdownUnavailable")}

)}
@@ -157,20 +170,24 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { }; const StackedTurnBar: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { + const { t } = useTranslation(); const segments = buckets.filter((b) => b.turns > 0); return (
{segments.map((b) => (
))}
@@ -185,47 +202,57 @@ const StackedTurnBar: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { ); }; -const BucketTable: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => ( - - - - Bucket - Turns - - Hit rate - - - - {buckets.map((b) => ( - - - - - - {b.label} - {b.sublabel} - - - - - {b.turns.toLocaleString()} - - -
-
-
- - - {pctLabel(b.hitRatePct)} - +const BucketTable: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { + const { t } = useTranslation(); + return ( +
+ + + + {t("costOptimization.autoRouter.caching.columns.bucket")} + + + {t("costOptimization.autoRouter.caching.columns.turns")} + + + + {t("costOptimization.autoRouter.caching.columns.hitRate")} + - ))} - -
-); + + + {buckets.map((b) => ( + + + + + + {b.label} + {b.sublabel} + + + + + {b.turns.toLocaleString()} + + +
+
+
+ + + {pctLabel(b.hitRatePct)} + + + ))} + + + ); +}; const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => { - const buckets = bucketRows(cache); + const { t } = useTranslation(); + const buckets = bucketRows(cache, t); const total = bucketTurnsTotal(cache); const expiredMissPct = expiredMissShare(cache); return ( @@ -233,7 +260,7 @@ const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => {
-

Cache hit rate

+

{t("costOptimization.autoRouter.caching.hitRate")}

{pctLabel(cache.hit_rate_pct)}

{expiredMissPct === null ? null : ( @@ -248,13 +275,12 @@ const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => { } > - Expired-miss + {t("costOptimization.autoRouter.caching.expiredMiss.label")} {pctLabel(expiredMissPct)} - share of all measured turns that missed cache because a return to an earlier tier came after its TTL - lapsed + {t("costOptimization.autoRouter.caching.expiredMiss.tooltip")} @@ -263,17 +289,27 @@ const CachingCard: React.FC<{ cache: AutoRouterCacheStats }> = ({ cache }) => {
-

Share of turns

+

+ {t("costOptimization.autoRouter.caching.shareOfTurns")} +

- {total.toLocaleString()} turns - measured + {total.toLocaleString()} + ), + }} + />

{cache.unordered_turns > 0 && (

- {cache.unordered_turns.toLocaleString()} turns arrived out of order across pods and are not bucketed + {t("costOptimization.autoRouter.caching.unorderedTurns", { + turns: cache.unordered_turns.toLocaleString(), + })}

)}
@@ -291,13 +327,14 @@ interface BenchmarksBodyProps { } const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey, autoRouters }) => { - if (isPending) return Loading auto-router usage...; + const { t } = useTranslation(); + if (isPending) return {t("costOptimization.autoRouter.body.loading")}; if (error instanceof ApiError && error.status === 403) { - return Auto-router usage is visible to proxy admin roles only; + return {t("costOptimization.autoRouter.body.forbidden")}; } - if (error || !data) return Auto-router usage is unavailable right now; + if (error || !data) return {t("costOptimization.autoRouter.body.unavailable")}; - const view = viewFor(data, selectedKey); + const view = viewFor(data, selectedKey, t); const stats = view.stats; return ( <> @@ -305,30 +342,32 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, -

- Savings and spend count requests on the selected UTC days. Actual spend covers every request on complexity - routers, including LLM classification cost. Baseline is actual spend plus recorded savings, so savings can be - zero or negative. -

+

{t("costOptimization.autoRouter.body.savingsNote")}

-

- Session metrics cover every session that overlaps the range, including its turns outside the range. -

+

{t("costOptimization.autoRouter.body.sessionNote")}

@@ -338,10 +377,8 @@ const BenchmarksBody: React.FC = ({ isPending, error, data,
-

Auto-router prompt caching

-

- every turn falls in exactly one bucket, by what the router did -

+

{t("costOptimization.autoRouter.caching.title")}

+

{t("costOptimization.autoRouter.caching.subtitle")}

@@ -362,21 +399,26 @@ export const AutoRouterUsageView: React.FC = ({ apiKey, userId, }) => { + const { t } = useTranslation(); const { dateValue, onDateChange } = activity; const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey, userId); const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); const { data: autoRouters } = useAutoRouters(); const groups = data?.groups ?? []; - const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers"; + const selectedLabel = data ? viewFor(data, selectedKey, t).label : t("costOptimization.autoRouter.allRouters"); const rangeLabel = formatRangeLabel(dateValue.from, dateValue.to); return (
-

Auto-router usage

- {rangeLabel &&

{rangeLabel} (UTC)

} +

{t("costOptimization.autoRouter.usageTitle")}

+ {rangeLabel && ( +

+ {t("costOptimization.autoRouter.rangeUtc", { range: rangeLabel })} +

+ )}
@@ -386,7 +428,7 @@ export const AutoRouterUsageView: React.FC = ({ {selectedLabel} - All auto-routers + {t("costOptimization.autoRouter.allRouters")} {groups.map((g) => ( {groupLabel(g, groups)} @@ -398,12 +440,7 @@ export const AutoRouterUsageView: React.FC = ({
- {userId && ( -

- Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID - are not included. -

- )} + {userId &&

{t("costOptimization.autoRouter.userScopeNote")}

} = ({ }; const AutoRouterBenchmarksTab: React.FC = ({ accessToken, activity }) => { + const { t } = useTranslation(); const [visitedTabs, setVisitedTabs] = useState(["usage"]); const handleTabChange = (value: unknown) => { @@ -430,10 +468,10 @@ const AutoRouterBenchmarksTab: React.FC = ({ acces - Usage + {t("costOptimization.autoRouter.tabs.usage")} - Shadow Evals + {t("costOptimization.autoRouter.tabs.shadowEvals")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterSummaryTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterSummaryTable.tsx index 0273fae9ec1..2a5cf572008 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterSummaryTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterSummaryTable.tsx @@ -1,3 +1,5 @@ +import { useTranslation } from "react-i18next"; + import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; @@ -10,33 +12,40 @@ interface AutoRouterSummaryTableProps { } export default function AutoRouterSummaryTable({ groups, selectedGroup }: AutoRouterSummaryTableProps) { + const { t } = useTranslation(); const visibleGroups = selectedGroup ? [selectedGroup] : groups; return ( - Router usage and savings - - Selected UTC days. Cost includes LLM calls and classification. Tokens count routed LLM input and output. - + {t("costOptimization.routerSummary.title")} + {t("costOptimization.routerSummary.description")} - +
- Auto-router - Tokens through router - Cost via router - Cost per 1M tokens - Saved vs. premium model - % saved + {t("costOptimization.routerSummary.columns.router")} + + {t("costOptimization.routerSummary.columns.tokens")} + + + {t("costOptimization.routerSummary.columns.cost")} + + + {t("costOptimization.routerSummary.columns.costPerMillion")} + + + {t("costOptimization.routerSummary.columns.saved")} + + {t("costOptimization.routerSummary.columns.savedPct")} {visibleGroups.length === 0 ? ( - No auto-routers in this range + {t("costOptimization.routerSummary.empty")} ) : ( @@ -44,29 +53,32 @@ export default function AutoRouterSummaryTable({ groups, selectedGroup }: AutoRo {groupLabel(group, groups)} - {group.total_tokens == null ? "Unavailable" : group.total_tokens.toLocaleString()} + {group.total_tokens == null + ? t("costOptimization.routerSummary.unavailable") + : group.total_tokens.toLocaleString()} {usd(group.spend)} {group.total_tokens != null && group.total_tokens > 0 ? usd((group.spend * 1_000_000) / group.total_tokens) - : "Unavailable"} + : t("costOptimization.routerSummary.unavailable")} - {group.saved_spend == null ? "Unavailable" : usd(group.saved_spend)} + {group.saved_spend == null + ? t("costOptimization.routerSummary.unavailable") + : usd(group.saved_spend)} - {group.saved_pct == null ? "Unavailable" : pctLabel(group.saved_pct)} + {group.saved_pct == null + ? t("costOptimization.routerSummary.unavailable") + : pctLabel(group.saved_pct)} )) )}
-

- Savings are estimated against each router's premium baseline. Token totals and unit costs are unavailable - for usage recorded before token tracking; unit cost also requires nonzero tokens. -

+

{t("costOptimization.routerSummary.footnote")}

); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index 7a63ccd1ee4..ab34f09f0ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import { ArrowDown, ArrowUp, ArrowUpDown, Info } from "lucide-react"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -67,6 +68,7 @@ const SortableHead = ({ sort: SortState; onSort: (column: SortColumn) => void; }) => { + const { t } = useTranslation(); const active = sort.column === column; const ActiveArrow = sort.dir === "asc" ? ArrowUp : ArrowDown; const Arrow = active ? ActiveArrow : ArrowUpDown; @@ -76,7 +78,7 @@ const SortableHead = ({
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx index 1b7369c9520..0e286418355 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx @@ -1,6 +1,8 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; import { CredentialIdentity } from "@/components/shared/CredentialIdentity"; import { Copy, Info, Loader2, Pencil, RefreshCw, Trash2 } from "lucide-react"; @@ -14,6 +16,7 @@ import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/h import { Switch } from "@/components/ui/switch"; import { getDisplayModelName } from "@/components/view_model/model_name_display"; import { copyToClipboard, formatPerSecondCost } from "@/utils/dataUtils"; +import i18n from "@/i18n/i18n"; export const MODEL_ID_COLUMN_ID = "model_info_id"; export const MODEL_NAME_COLUMN_ID = "model_name"; @@ -56,6 +59,7 @@ const formatShortDate = (value: string | null | undefined): string | null => { }; function ModelInformationCell({ model, displayName }: { model: ModelData; displayName: string }) { + const { t } = useTranslation(); const litellmModelName = model.litellm_model_name || "-"; return ( @@ -85,26 +89,37 @@ function ModelInformationCell({ model, displayName }: { model: ModelData; displa
{model.provider ? : null} - {model.provider || "Unknown provider"} + + {model.provider || t("modelsAndEndpoints.modelsTableColumns.unknownProvider")} +
- Public Model Name + + {t("modelsAndEndpoints.modelsTableColumns.information.publicModelName")} + {displayName}
- LiteLLM Model Name + + {t("modelsAndEndpoints.modelsTableColumns.information.litellmModelName")} + {litellmModelName} @@ -117,15 +132,16 @@ function ModelInformationCell({ model, displayName }: { model: ModelData; displa } function CredentialsHeader() { + const { t } = useTranslation(); return ( - Credentials + {t("modelsAndEndpoints.modelsTableColumns.columns.credentials")} @@ -135,23 +151,25 @@ function CredentialsHeader() {
- Credential types + + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.title")} +
- Reusable + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.reusable")} - Credentials saved in LiteLLM that can be added to models repeatedly. + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.reusableDescription")}
- Manual + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manual")} - Credentials added directly during model creation or defined in the config file. + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manualDescription")}
@@ -162,11 +180,12 @@ function CredentialsHeader() { } function CredentialsCell({ credentialName, label }: { credentialName: string | undefined; label: string | undefined }) { + const { t } = useTranslation(); if (!credentialName) { return ( - Manual + {t("modelsAndEndpoints.modelsTableColumns.credentialTypes.manual")} ); } @@ -182,10 +201,13 @@ function CredentialsCell({ credentialName, label }: { credentialName: string | u } function CreatedByCell({ model }: { model: ModelData }) { + const { t } = useTranslation(); const isConfigModel = !model.model_info?.db_model; const createdAt = formatShortDate(model.model_info.created_at); - const primary = isConfigModel ? "Defined in config" : model.model_info.created_by || "Unknown"; - const secondaryForDbModel = createdAt ?? "Unknown date"; + const primary = isConfigModel + ? t("modelsAndEndpoints.modelsTableColumns.createdBy.definedInConfig") + : model.model_info.created_by || t("modelsAndEndpoints.modelsTableColumns.createdBy.unknown"); + const secondaryForDbModel = createdAt ?? t("modelsAndEndpoints.modelsTableColumns.createdBy.unknownDate"); return (
@@ -207,6 +229,7 @@ function CostRow({ label, value }: { label: string; value: string }) { } function CostsCell({ model }: { model: ModelData }) { + const { t } = useTranslation(); const { input_cost: inputCost, output_cost: outputCost, output_cost_per_second: perSecond } = model; const hasPerSecond = perSecond != null; const showInput = inputCost != null && (!hasPerSecond || Number(inputCost) > 0); @@ -218,12 +241,25 @@ function CostsCell({ model }: { model: ModelData }) { return ( - {showInput && } - {showOutput && } - {hasPerSecond && } + {showInput && ( + + )} + {showOutput && ( + + )} + {hasPerSecond && ( + + )}
} /> @@ -231,6 +267,7 @@ function CostsCell({ model }: { model: ModelData }) { } function AccessGroupsCell({ accessGroups }: { accessGroups: string[] | null }) { + const { t } = useTranslation(); if (!accessGroups || accessGroups.length === 0) { return -; } @@ -253,7 +290,7 @@ function AccessGroupsCell({ accessGroups }: { accessGroups: string[] | null }) { } trigger={ - +{overflow.length} more + {t("modelsAndEndpoints.modelsTableColumns.accessGroups.more", { hiddenCount: overflow.length })} } /> @@ -281,6 +318,7 @@ function ModelRowActions({ onDeleteClick, onTogglePauseClick, }: ModelRowActionsProps) { + const { t } = useTranslation(); const modelId = model.model_info?.id; const isConfigModel = !model.model_info?.db_model; const isAdmin = userRole === "Admin" && !isViewOnly; @@ -290,17 +328,19 @@ function ModelRowActions({ const resolvePauseTooltip = (): string => { if (isConfigModel) { - return "Config models cannot be paused from the dashboard. Pause is DB-backed."; + return t("modelsAndEndpoints.modelsTableColumns.actions.configPauseTooltip"); } if (!isAdmin) { - return "Only proxy admins can pause or resume a model."; + return t("modelsAndEndpoints.modelsTableColumns.actions.adminOnlyPauseTooltip"); } - return isBlocked ? "Resume model — restore normal routing." : "Pause model — stop routing requests until resumed."; + return isBlocked + ? t("modelsAndEndpoints.modelsTableColumns.actions.resumeTooltip") + : t("modelsAndEndpoints.modelsTableColumns.actions.pauseTooltip"); }; const deleteTooltip = isConfigModel - ? "Config model cannot be deleted on the dashboard. Please delete it from the config file." - : "Delete model"; + ? t("modelsAndEndpoints.modelsTableColumns.actions.configDeleteTooltip") + : t("modelsAndEndpoints.modelsTableColumns.actions.delete"); return (
@@ -319,7 +359,11 @@ function ModelRowActions({ size="sm" checked={!isBlocked} disabled={!isPauseToggleable} - aria-label={isBlocked ? "Resume model" : "Pause model"} + aria-label={ + isBlocked + ? t("modelsAndEndpoints.modelsTableColumns.actions.resume") + : t("modelsAndEndpoints.modelsTableColumns.actions.pause") + } data-testid={`model-pause-toggle-${modelId}`} onCheckedChange={(nextChecked) => { if (isPauseToggleable && onTogglePauseClick && modelId) { @@ -339,7 +383,7 @@ function ModelRowActions({
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx index 20eb7b2e64c..d632408c85a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AccessGroupBudgetsPanel.tsx @@ -3,6 +3,7 @@ import { SortingState } from "@tanstack/react-table"; import { Inbox } from "lucide-react"; import React, { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { DataTable } from "@/components/shared/DataTable"; @@ -21,20 +22,24 @@ import { getAccessGroupBudgetColumns } from "@/app/(dashboard)/models-and-endpoi const DEFAULT_SORTING: SortingState = [{ id: "access_group", desc: false }]; function EmptyState() { + const { t } = useTranslation(); return (
-
No model access groups yet
+
+ {t("modelsAndEndpoints.accessGroupBudgetsPanel.empty.title")} +
- Put a deployment in an access group from its model settings, then give the group a shared budget here. + {t("modelsAndEndpoints.accessGroupBudgetsPanel.empty.description")}
); } export default function AccessGroupBudgetsPanel() { + const { t } = useTranslation(); const { userRole } = useAuthorized(); const { data: accessGroups, isLoading } = useModelAccessGroups(); const setBudget = useSetModelAccessGroupBudget(); @@ -57,7 +62,7 @@ export default function AccessGroupBudgetsPanel() { { accessGroup, params }, { onSuccess: () => { - toast.success(`Budget saved for "${accessGroup}"`); + toast.success(t("modelsAndEndpoints.accessGroupBudgetsPanel.toasts.saved", { accessGroup })); setEditing(null); }, }, @@ -69,7 +74,7 @@ export default function AccessGroupBudgetsPanel() { const accessGroup = clearing.access_group; clearBudget.mutate(accessGroup, { onSuccess: () => { - toast.success(`Budget cleared for "${accessGroup}"`); + toast.success(t("modelsAndEndpoints.accessGroupBudgetsPanel.toasts.cleared", { accessGroup })); setClearing(null); }, }); @@ -77,10 +82,7 @@ export default function AccessGroupBudgetsPanel() { return (
-

- A model access group can carry one budget that every key granted the group by name draws from together. Keys - that reach the group's models through a wildcard or all-proxy-models are not charged against it. -

+

{t("modelsAndEndpoints.accessGroupBudgetsPanel.description")}

} size="compact" /> @@ -105,12 +107,19 @@ export default function AccessGroupBudgetsPanel() { setClearing(null)} onOk={handleConfirmClear} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx index a5c10ca8a8b..50dc4908865 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelRetrySettingsPanel.tsx @@ -1,6 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; import ModelRetrySettingsTab from "@/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab"; import { getCallbacksCall } from "@/components/networking"; import { useUpdateRetryPolicy } from "@/app/(dashboard)/hooks/routerSettings/useUpdateRetryPolicy"; @@ -23,6 +24,7 @@ interface RouterSettings { } export default function ModelRetrySettingsPanel() { + const { t } = useTranslation(); const { accessToken, userId: userID, userRole } = useAuthorized(); const { availableModelGroups } = useModelDashboardData(); const updateRetryPolicy = useUpdateRetryPolicy(accessToken); @@ -69,7 +71,7 @@ export default function ModelRetrySettingsPanel() { { retry_policy: globalRetryPolicy, model_group_retry_policy: modelGroupRetryPolicy }, { onSuccess: () => { - toast.success("Retry settings saved successfully"); + toast.success(t("modelsAndEndpoints.modelRetrySettingsPanel.toasts.saved")); void fetchRetrySettings().then((routerSettings) => { if (routerSettings) { applyRetrySettings(routerSettings); @@ -77,7 +79,7 @@ export default function ModelRetrySettingsPanel() { }); }, onError: () => { - toast.fromError("Failed to save retry settings"); + toast.fromError(t("modelsAndEndpoints.modelRetrySettingsPanel.toasts.saveFailed")); }, }, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx index 6ad2f00fdb0..cd5c796b569 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx @@ -2,6 +2,7 @@ import { FilterInput } from "@/components/common_components/Filters/FilterInput" import { FiltersButton } from "@/components/common_components/Filters/FiltersButton"; import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton"; import { Search, User } from "lucide-react"; +import { useTranslation } from "react-i18next"; interface OrganizationFiltersProps { filters: FilterState; @@ -23,6 +24,7 @@ const OrganizationFilters = ({ onChange, onReset, }: OrganizationFiltersProps) => { + const { t } = useTranslation(); const hasActiveFilters = !!(filters.org_id || filters.org_alias); return ( @@ -30,7 +32,7 @@ const OrganizationFilters = ({ {/* Search and Filter Controls */}
onChange("org_alias", value)} icon={Search} @@ -50,7 +52,7 @@ const OrganizationFilters = ({ {showFilters && (
onChange("org_id", value)} icon={User} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx index 2810902bef6..819aa6c3334 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx @@ -4,6 +4,7 @@ import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organization import { useQueryClient } from "@tanstack/react-query"; import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; import React, { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { toast } from "@/lib/toast"; import { organizationDeleteCall } from "@/components/networking"; @@ -28,6 +29,7 @@ const ORGANIZATION_DETAIL_STATE = { const ORGANIZATION_DETAIL_URL_KEYS = { tab: ORGANIZATION_TAB_URL_KEY }; const OrganizationsPanel: React.FC = ({ userRole, accessToken, premiumUser }) => { + const { t } = useTranslation(); const [{ org: selectedOrgId }, setOrganizationDetail] = useQueryStates(ORGANIZATION_DETAIL_STATE, { history: "push", urlKeys: ORGANIZATION_DETAIL_URL_KEYS, @@ -78,7 +80,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces try { setIsDeleting(true); await organizationDeleteCall(accessToken, orgToDelete); - toast.success("Organization deleted successfully"); + toast.success(t("organizations.panel.toasts.deleted")); setIsDeleteModalOpen(false); setOrgToDelete(null); @@ -99,16 +101,19 @@ const OrganizationsPanel: React.FC = ({ userRole, acces return ( ); @@ -118,7 +123,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces
{(userRole === "Admin" || userRole === "Org Admin") && ( )} @@ -133,7 +138,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces /> ) : ( <> -

Click on an organization ID to view its details.

+

{t("organizations.panel.listHint")}

= ({ userRole, acces @@ -27,12 +29,12 @@ function EmptyState({ searchActive }: { searchActive: boolean }) {
- {searchActive ? "No matching organizations" : "No organizations yet"} + {searchActive ? t("organizations.table.empty.noMatchesTitle") : t("organizations.table.empty.noneTitle")}
{searchActive - ? "No organizations match your search. Try a different name or ID." - : "Create an organization to group teams, models, and budgets."} + ? t("organizations.table.empty.noMatchesDescription") + : t("organizations.table.empty.noneDescription")}
); @@ -47,12 +49,13 @@ const OrganizationsTable: React.FC = ({ onEditClick, onDeleteClick, }) => { + const { t } = useTranslation(); const { sorting, onSortingChange, pagination, onPaginationChange } = useOrganizationsTableState(); const columns = useMemo(() => { - const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick }; + const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick, t }; return getOrganizationsTableColumns(deps); - }, [userRole, onOrganizationClick, onEditClick, onDeleteClick]); + }, [userRole, onOrganizationClick, onEditClick, onDeleteClick, t]); return ( = ({ sorting={sorting} onSortingChange={onSortingChange} isLoading={isLoading} - loadingMessage="Loading organizations…" + loadingMessage={t("organizations.table.loading")} noDataMessage={} size="compact" /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx index dcbf141314b..b4464392ea8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -1,7 +1,9 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; @@ -25,11 +27,18 @@ const getOrganizationBudget = (organization: Organization): OrganizationBudget = (organization.litellm_budget_table ?? {}) as OrganizationBudget; function OrganizationLimitsCell({ organization }: { organization: Organization }) { + const { t } = useTranslation(); const { tpm_limit, rpm_limit } = getOrganizationBudget(organization); return (
- TPM: {tpm_limit ?? "Unlimited"} - RPM: {rpm_limit ?? "Unlimited"} + + {t("organizations.table.limits.tpmLabel")} + {tpm_limit ?? t("organizations.table.limits.unlimited")} + + + {t("organizations.table.limits.rpmLabel")} + {rpm_limit ?? t("organizations.table.limits.unlimited")} +
); } @@ -41,10 +50,11 @@ interface OrganizationRowActionsProps { } function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: OrganizationRowActionsProps) { + const { t } = useTranslation(); return ( @@ -56,7 +66,7 @@ function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: Or onClick={() => onEditClick(organization.organization_id)} > - Edit + {t("organizations.table.actions.edit")} onDeleteClick(organization.organization_id)} > - Delete + {t("organizations.table.actions.delete")} @@ -76,6 +86,7 @@ export interface OrganizationsTableColumnsDeps { onOrganizationClick: (organizationId: string) => void; onEditClick: (organizationId: string) => void; onDeleteClick: (organizationId: string) => void; + t: TFunction; } export const getOrganizationsTableColumns = ({ @@ -83,12 +94,13 @@ export const getOrganizationsTableColumns = ({ onOrganizationClick, onEditClick, onDeleteClick, + t, }: OrganizationsTableColumnsDeps): ColumnDef[] => [ { id: "organization_id", accessorKey: "organization_id", - meta: { title: "Organization ID" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.id") }, + header: ({ column }) => , size: 220, enableSorting: true, cell: ({ row }) => ( @@ -103,8 +115,8 @@ export const getOrganizationsTableColumns = ({ { id: "organization_alias", accessorKey: "organization_alias", - meta: { title: "Organization Name" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.name") }, + header: ({ column }) => , size: 200, enableSorting: true, cell: ({ row }) => { @@ -120,8 +132,8 @@ export const getOrganizationsTableColumns = ({ id: "created_at", accessorKey: "created_at", sortingFn: "datetime", - meta: { title: "Created" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.created") }, + header: ({ column }) => , size: 130, enableSorting: true, cell: ({ row }) => , @@ -129,8 +141,8 @@ export const getOrganizationsTableColumns = ({ { id: "spend", accessorKey: "spend", - meta: { title: "Spend (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.spend"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => , @@ -140,12 +152,17 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Budget (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.budget"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => ( - + ), }, { @@ -156,21 +173,21 @@ export const getOrganizationsTableColumns = ({ : undefined, sortUndefined: "last", sortDescFirst: false, - meta: { title: "Models", skeleton: "chips" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.models"), skeleton: "chips" }, + header: ({ column }) => , size: 260, enableSorting: true, cell: ({ row }) => , }, { id: "limits", - meta: { title: "TPM / RPM Limits" }, + meta: { title: t("organizations.table.columns.limits") }, header: ({ table }) => ( ), @@ -183,11 +200,15 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => organization.members?.length ?? 0, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Members", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.members"), numeric: true }, + header: ({ column }) => , size: 100, enableSorting: true, - cell: ({ row }) => {row.original.members?.length ?? 0} Members, + cell: ({ row }) => ( + + {t("organizations.table.memberCount", { count: row.original.members?.length ?? 0 })} + + ), }, ...(["tpm_limit", "rpm_limit"] as const).map( (field): ColumnDef => ({ @@ -201,7 +222,7 @@ export const getOrganizationsTableColumns = ({ { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {t("organizations.table.columns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx index f94c43b720a..4e337a88add 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx @@ -2,6 +2,8 @@ import { useId, useState } from "react"; import { Check, ExternalLink, KeyRound } from "lucide-react"; +import type { TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; import { PasswordInput } from "@/components/shared/PasswordInput"; import { Button } from "@/components/ui/button"; import { InputGroupButton } from "@/components/ui/input-group"; @@ -13,13 +15,14 @@ import type { MCPServer } from "@/components/mcp_tools/types"; import { fetchClient } from "@/lib/http/api"; import { ApiError } from "@/lib/http/client"; import { toast } from "@/lib/toast"; +import i18n from "@/i18n/i18n"; -const saveErrorMessage = (error: unknown): string => { +const saveErrorMessage = (error: unknown, t: TFunction = i18n.t.bind(i18n)): string => { if (error instanceof ApiError) { const detail = (error.body as { detail?: { error?: string } } | null)?.detail?.error; if (detail) return detail; } - return error instanceof Error && error.message ? error.message : "Failed to connect"; + return error instanceof Error && error.message ? error.message : t("playground.chat.byok.failed"); }; export function ByokConnect({ @@ -33,6 +36,7 @@ export function ByokConnect({ onConnected: () => void; onCancel?: () => void; }) { + const { t } = useTranslation(); const [credential, setCredential] = useState(""); const [save, setSave] = useState(true); const [isSaving, setIsSaving] = useState(false); @@ -41,7 +45,7 @@ export function ByokConnect({ const connect = async () => { if (!credential.trim()) { - toast.error("Enter your API key"); + toast.error(t("playground.chat.byok.enterKey")); return; } setIsSaving(true); @@ -50,11 +54,11 @@ export function ByokConnect({ params: { path: { server_id: server.server_id } }, body: { credential: credential.trim(), save }, }); - toast.success(`Connected to ${name}`); + toast.success(t("playground.chat.byok.connected", { name })); setCredential(""); onConnected(); } catch (error) { - toast.error(saveErrorMessage(error)); + toast.error(saveErrorMessage(error, t)); } finally { setIsSaving(false); } @@ -65,12 +69,14 @@ export function ByokConnect({

- {server.has_user_credential ? `Replace your ${name} API key` : `${name} needs your API key`} + {server.has_user_credential + ? t("playground.chat.byok.replaceTitle", { name }) + : t("playground.chat.byok.connectTitle", { name })}

{server.has_user_credential - ? `The new key replaces the one the gateway uses for ${name}.` - : `The gateway calls ${name} with your own key. Connect it to list and use its tools.`} + ? t("playground.chat.byok.replaceDescription", { name }) + : t("playground.chat.byok.connectDescription", { name })}

{server.byok_description && server.byok_description.length > 0 && ( @@ -84,17 +90,17 @@ export function ByokConnect({ )} - {name} API key + {t("playground.chat.byok.keyLabel", { name })} setCredential(event.target.value)} - placeholder="Paste your API key" + placeholder={t("playground.chat.byok.keyPlaceholder")} /> {server.byok_api_key_help_url && ( - Where to find this key + {t("playground.chat.byok.helpLink")} @@ -102,18 +108,18 @@ export function ByokConnect({ - Save the key for next time + {t("playground.chat.byok.save")}
{onCancel && ( )}
@@ -130,12 +136,13 @@ export function ByokKeyButton({ name: string; onConnected: () => void; }) { + const { t } = useTranslation(); const [open, setOpen] = useState(false); return ( }> - {server.has_user_credential ? "Replace key" : "Connect key"} + {server.has_user_credential ? t("playground.chat.byok.replaceKey") : t("playground.chat.byok.connectKey")} void; }) { + const { t } = useTranslation(); const input = useRef(null); if (!capabilities.attach) return null; - const label = ATTACH_LABEL[capabilities.attach]; + const label = t(ATTACH_LABEL[capabilities.attach]); return ( <> @@ -69,9 +71,15 @@ function AttachButton({ } function SendButton({ draft, disabled }: { draft: DraftState; disabled: boolean }) { + const { t } = useTranslation(); if (draft.isLoading) return ( - + ); @@ -79,7 +87,7 @@ function SendButton({ draft, disabled }: { draft: DraftState; disabled: boolean void draft.send()} > @@ -115,6 +123,7 @@ export function Composer({ settingsControl: ReactNode; sendDisabled: boolean; }) { + const { t } = useTranslation(); const isDirectCall = endpoint === EndpointType.MCP; const isRealtime = endpoint === EndpointType.REALTIME; const message = useRef(null); @@ -138,7 +147,7 @@ export function Composer({ {directTool ? ( ) : ( -

Pick a server and a tool above to fill in its arguments

+

{t("playground.chat.composer.pickTool")}

)}
); @@ -148,9 +157,9 @@ export function Composer({ value={draft.text} onChange={(event) => draft.setText(event.target.value)} onKeyDown={handleKeyDown} - placeholder={capabilities.placeholder} + placeholder={t(capabilities.placeholder)} rows={2} - aria-label="Message" + aria-label={t("playground.chat.composer.message")} className="max-h-60" /> ); @@ -168,21 +177,31 @@ export function Composer({ {capabilities.mcp && mcpControl} {capabilities.vectorStores && ( - } emptyText="No vector stores yet" {...vectorStores} /> + } + emptyText={t("playground.chat.composer.noVectorStores")} + {...vectorStores} + /> )} {capabilities.guardrails && ( - } emptyText="No guardrails yet" {...guardrails} /> + } + emptyText={t("playground.chat.composer.noGuardrails")} + {...guardrails} + /> )} {capabilities.codeInterpreter && ( - Code interpreter + {t("playground.chat.composer.codeInterpreter")} )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx index e3e52be3b7b..42ef5c92f09 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx @@ -1,6 +1,7 @@ "use client"; import { ChevronDown, FileText, Music, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; import { Attachment, @@ -33,16 +34,19 @@ export function EndpointMenu({ value: EndpointType; onValueChange: (value: EndpointType) => void; }) { + const { t } = useTranslation(); const current = ENDPOINT_OPTIONS.find((option) => option.value === value); return ( - }> + } + > {shortEndpoint(current?.label ?? value)} onValueChange(next as EndpointType)}> - Endpoint + {t("playground.chat.controls.endpoint")} {ENDPOINT_OPTIONS.map((option) => ( {option.label} @@ -61,19 +65,24 @@ export function VoiceMenu({ value: OpenAIVoice; onValueChange: (value: OpenAIVoice) => void; }) { + const { t } = useTranslation(); const current = OPEN_AI_VOICE_SELECT_OPTIONS.find((option) => option.value === value); return ( - }> - Voice: {current?.label.split(" - ")[0] ?? value} + } + > + + {t("playground.chat.controls.voiceValue", { voice: current ? t(current.label).split(" - ")[0] : value })} + onValueChange(next as OpenAIVoice)}> - Voice + {t("playground.chat.controls.voice")} {OPEN_AI_VOICE_SELECT_OPTIONS.map((option) => ( - {option.label} + {t(option.label)} ))} @@ -99,6 +108,7 @@ export function DraftAttachments({ files: readonly DraftFile[]; onRemove: (index: number) => void; }) { + const { t } = useTranslation(); return ( {files.map((draft, index) => ( @@ -109,7 +119,10 @@ export function DraftAttachments({ {formatBytes(draft.file.size)} - onRemove(index)}> + onRemove(index)} + > diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx index 31eb395408a..c6b214eb081 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx @@ -16,6 +16,9 @@ import { Timer, Wrench, } from "lucide-react"; +import type { TFunction } from "i18next"; +import { Trans, useTranslation } from "react-i18next"; +import i18n from "@/i18n/i18n"; import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; import MCPEventsDisplay from "@/components/chat_ui/MCPEventsDisplay"; import ReasoningContent from "@/components/chat_ui/ReasoningContent"; @@ -68,11 +71,20 @@ const fellBack = (message: MessageType): boolean => (message.trace?.attemptedFal export const servedModel = (message: MessageType): string | undefined => message.trace?.modelGroup ?? (message.isError ? undefined : message.model ?? message.trace?.requestedModel); -const replyModel = (message: MessageType): Readonly<{ value: string; detail: string }> | null => { +const replyModel = ( + message: MessageType, + t: TFunction = i18n.t.bind(i18n), +): Readonly<{ value: string; detail: string }> | null => { const served = servedModel(message); - if (served) return { value: served, detail: fellBack(message) ? "Served by, after a fallback" : "Served by" }; + if (served) + return { + value: served, + detail: fellBack(message) + ? t("playground.chat.conversation.servedByAfterFallback") + : t("playground.chat.conversation.servedBy"), + }; const requested = message.trace?.requestedModel ?? message.model; - return requested ? { value: requested, detail: "Requested model" } : null; + return requested ? { value: requested, detail: t("playground.chat.conversation.requestedModel") } : null; }; function Metric({ icon, value, label }: { icon: ReactNode; value: string; label?: string }) { @@ -86,35 +98,60 @@ function Metric({ icon, value, label }: { icon: ReactNode; value: string; label? } function ReplyMetrics({ message }: { message: MessageType }) { + const { t } = useTranslation(); const usage = message.usage; const cost = usage?.cost ?? message.trace?.cost; const cached = usage?.servedFromResponseCache || message.trace?.cacheHit; return (
{message.timeToFirstToken !== undefined && ( - } value={formatSeconds(message.timeToFirstToken)} label="to first token" /> + } + value={formatSeconds(message.timeToFirstToken)} + label={t("playground.chat.conversation.metrics.firstToken")} + /> )} {message.totalLatency !== undefined && ( - } value={formatSeconds(message.totalLatency)} label="total" /> + } + value={formatSeconds(message.totalLatency)} + label={t("playground.chat.conversation.metrics.total")} + /> )} {usage?.promptTokens !== undefined && ( - } value={String(usage.promptTokens)} label="in" /> + } + value={String(usage.promptTokens)} + label={t("playground.chat.conversation.metrics.in")} + /> )} {usage?.completionTokens !== undefined && ( - } value={String(usage.completionTokens)} label="out" /> + } + value={String(usage.completionTokens)} + label={t("playground.chat.conversation.metrics.out")} + /> )} {usage?.reasoningTokens ? ( - } value={String(usage.reasoningTokens)} label="reasoning" /> + } + value={String(usage.reasoningTokens)} + label={t("playground.chat.conversation.metrics.reasoning")} + /> ) : null} {usage?.cacheReadTokens ? ( - } value={String(usage.cacheReadTokens)} label="cached" /> + } + value={String(usage.cacheReadTokens)} + label={t("playground.chat.conversation.metrics.cached")} + /> ) : null} {cost !== undefined && } value={formatCost(cost)} />} {message.toolName && } value={message.toolName} />} - {cached && Cached response} + {cached && {t("playground.chat.conversation.metrics.cachedResponse")}}
); } @@ -152,7 +189,10 @@ function UserTurn({ message }: { message: MessageType }) { - Called {message.mcpToolCall.tool} + {message.mcpToolCall.tool} }} + />
                 {JSON.stringify(message.mcpToolCall.arguments, null, 2)}
@@ -190,7 +230,8 @@ function AssistantReply({
   inspected: boolean;
   onInspect: () => void;
 }) {
-  const model = replyModel(message);
+  const { t } = useTranslation();
+  const model = replyModel(message, t);
   const showMcpEvents = isLast && extras.mcpEvents.length > 0 && MCP_EVENT_ENDPOINTS.includes(extras.endpoint);
   const showCode = isLast && extras.codeInterpreter && extras.endpoint === EndpointType.RESPONSES;
   return (
@@ -201,7 +242,7 @@ function AssistantReply({
             
             {fellBack(message) && }
@@ -220,7 +261,7 @@ function AssistantReply({
         )}
         {message.isError ? (
           
-            The request failed
+            {t("playground.chat.conversation.requestFailed")}
             {textOf(message)}
           
         ) : (
@@ -239,17 +280,19 @@ function AssistantReply({
             )}
           
- {!message.isImage && !message.isAudio && } + {!message.isImage && !message.isAudio && ( + + )} {message.trace && ( - Inspect + {t("playground.chat.conversation.inspect")} )} @@ -260,12 +303,17 @@ function AssistantReply({ } function WaitingTurn({ target }: { target: string | undefined }) { + const { t } = useTranslation(); return (
- {target ? `Waiting for ${target}` : "Waiting for a reply"} + + {target + ? t("playground.chat.conversation.waitingFor", { target }) + : t("playground.chat.conversation.waitingForReply")} +
@@ -273,12 +321,13 @@ function WaitingTurn({ target }: { target: string | undefined }) { } function SwitchMarker({ model }: { model: string }) { + const { t } = useTranslation(); return ( - Switched to {model} + {t("playground.chat.conversation.switchedTo", { model })} ); } @@ -290,16 +339,15 @@ function EmptyConversation({ suggestions: readonly string[]; onPick: (text: string) => void; }) { + const { t } = useTranslation(); return ( - Send a test request - - Pick a model and anything the request should go through, then inspect what the gateway did with it - + {t("playground.chat.conversation.empty.title")} + {t("playground.chat.conversation.empty.description")} {suggestions.length > 0 && ( @@ -340,13 +388,14 @@ export function Conversation({ suggestions: readonly string[]; onPickSuggestion: (text: string) => void; }) { + const { t } = useTranslation(); if (messages.length === 0) return ; const lastIndex = messages.length - 1; const waiting = isLoading && messages[lastIndex]?.role === "user"; return ( - + {messages.flatMap((message, index) => { const switched = switchedModel(messages, index); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx index 3a9919ce38f..d294c859a18 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx @@ -14,6 +14,9 @@ import { Shuffle, Timer, } from "lucide-react"; +import type { TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; +import i18n from "@/i18n/i18n"; import type { MessageType } from "@/components/chat_ui/types"; import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from "@/components/ui/accordion"; import { Badge } from "@/components/ui/badge"; @@ -80,130 +83,175 @@ const millis = (value: number | undefined): string | undefined => value === undefined ? undefined : `${Math.round(value)}ms`; function Routing({ message }: { message: MessageType }) { + const { t } = useTranslation(); const trace = message.trace; const served = servedModel(message); const fellBack = (trace?.attemptedFallbacks ?? 0) > 0; const reported = Boolean(trace?.callId || trace?.modelGroup || trace?.providerModel); return ( - + {trace?.requestedModel && ( - - + + )} {served && ( - + {fellBack && } - + )} {trace?.providerModel && ( - + )} {trace?.apiBase && ( - + )} {trace?.callId && ( - + {trace.callId} - + )} {trace?.responseId && ( - + {trace.responseId} - + )} , value: count(trace?.attemptedFallbacks) }, - { label: "Retries", icon: , value: count(trace?.attemptedRetries) }, - { label: "Overhead", icon: , value: millis(trace?.overheadMs) }, + { + label: t("playground.chat.inspector.routing.fallbacks"), + icon: , + value: count(trace?.attemptedFallbacks), + }, + { + label: t("playground.chat.inspector.routing.retries"), + icon: , + value: count(trace?.attemptedRetries), + }, + { label: t("playground.chat.inspector.routing.overhead"), icon: , value: millis(trace?.overheadMs) }, ]} /> - {!reported && ( - - The gateway did not share routing details with this page. Open the dashboard from the proxy to see them - - )} + {!reported && {t("playground.chat.inspector.routing.notReported")}} ); } -const noneRan = (policies: readonly string[] | undefined): string | null => { - if (policies === undefined) return "No guardrails ran on this request"; - return policies.length === 0 ? "No guardrails or policies ran on this request" : null; +const noneRan = (policies: readonly string[] | undefined, t: TFunction = i18n.t.bind(i18n)): string | null => { + if (policies === undefined) return t("playground.chat.inspector.guardrails.noGuardrailsRan"); + return policies.length === 0 ? t("playground.chat.inspector.guardrails.noneRan") : null; }; function Guardrails({ message }: { message: MessageType }) { + const { t } = useTranslation(); const guardrails = message.trace?.appliedGuardrails; const policies = message.trace?.appliedPolicies; if (guardrails === undefined) { return ( - - This endpoint does not report which guardrails or policies ran + + {t("playground.chat.inspector.guardrails.unsupported")} ); } - const emptyNote = guardrails.length === 0 ? noneRan(policies) : null; + const emptyNote = guardrails.length === 0 ? noneRan(policies, t) : null; return ( - + {guardrails.map((name) => ( - - Ran + + {t("playground.chat.inspector.guardrails.ran")} ))} {policies?.map((name) => ( - - Ran + + {t("playground.chat.inspector.guardrails.ran")} ))} {emptyNote && {emptyNote}} - {policies === undefined && Policies are only reported when the dashboard is opened from the proxy} + {policies === undefined && {t("playground.chat.inspector.guardrails.policiesUnreported")}} ); } function SpeedAndCost({ message }: { message: MessageType }) { + const { t } = useTranslation(); const usage = message.usage; const cost = usage?.cost ?? message.trace?.cost; return ( - + , value: message.timeToFirstToken === undefined ? undefined : formatSeconds(message.timeToFirstToken), }, { - label: "Total", + label: t("playground.chat.inspector.performance.total"), icon: , value: message.totalLatency === undefined ? undefined : formatSeconds(message.totalLatency), }, - { label: "Cost", icon: , value: cost === undefined ? undefined : formatCost(cost) }, - { label: "Tokens in", icon: , value: count(usage?.promptTokens) }, - { label: "Tokens out", icon: , value: count(usage?.completionTokens) }, - { label: "Cached", icon: , value: count(usage?.cacheReadTokens) }, - { label: "Reasoning", icon: , value: count(usage?.reasoningTokens) }, - { label: "Gateway time", icon: , value: millis(message.trace?.responseMs) }, + { + label: t("playground.chat.inspector.performance.cost"), + icon: , + value: cost === undefined ? undefined : formatCost(cost), + }, + { + label: t("playground.chat.inspector.performance.tokensIn"), + icon: , + value: count(usage?.promptTokens), + }, + { + label: t("playground.chat.inspector.performance.tokensOut"), + icon: , + value: count(usage?.completionTokens), + }, + { + label: t("playground.chat.inspector.performance.cached"), + icon: , + value: count(usage?.cacheReadTokens), + }, + { + label: t("playground.chat.inspector.performance.reasoning"), + icon: , + value: count(usage?.reasoningTokens), + }, + { + label: t("playground.chat.inspector.performance.gatewayTime"), + icon: , + value: millis(message.trace?.responseMs), + }, ]} /> @@ -211,13 +259,12 @@ function SpeedAndCost({ message }: { message: MessageType }) { } function NothingYet() { + const { t } = useTranslation(); return ( - Nothing to inspect yet - - Each reply shows here with the deployment that served it, fallbacks, guardrails, timing and cost - + {t("playground.chat.inspector.empty.title")} + {t("playground.chat.inspector.empty.description")} ); @@ -235,6 +282,7 @@ function Trace({ message }: { message: MessageType | null }) { } function CodeTab({ code }: { code: (sdk: CodeSdk) => string }) { + const { t } = useTranslation(); const [sdk, setSdk] = useState("openai"); const snippet = code(sdk); return ( @@ -246,21 +294,21 @@ function CodeTab({ code }: { code: (sdk: CodeSdk) => string }) { variant="outline" size="sm" spacing={0} - aria-label="SDK" + aria-label={t("playground.chat.inspector.code.sdk")} > - OpenAI SDK + {t("playground.chat.inspector.code.openai")} - Azure SDK + {t("playground.chat.inspector.code.azure")} - +
- Send this request from your own app. Replace YOUR_LITELLM_API_KEY with a virtual key + {t("playground.chat.inspector.code.description")}
{snippet}
@@ -271,12 +319,13 @@ function CodeTab({ code }: { code: (sdk: CodeSdk) => string }) { } export function InspectorPanel({ message, code }: { message: MessageType | null; code: (sdk: CodeSdk) => string }) { + const { t } = useTranslation(); return (
- Trace - Code + {t("playground.chat.inspector.tabs.trace")} + {t("playground.chat.inspector.tabs.code")}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx index e5e0828825d..2a72a5facc9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx @@ -2,6 +2,7 @@ import { useId, useState, type ReactNode } from "react"; import { Blocks, KeyRound, Search } from "lucide-react"; +import { Trans, useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; @@ -84,9 +85,14 @@ function MasterRow({ onSelect: () => void; summary?: string; }) { + const { t } = useTranslation(); return (
- + ) : undefined; if (!selection.includes(focused)) return ( - Check this server to choose which of its tools the model can call + {t("playground.chat.mcp.checkServer")} ); const state = toolsByServer[focused]; @@ -283,11 +299,11 @@ export function McpDialog({ if (state.failed) return ( - Couldn't load this server's tools + {t("playground.chat.mcp.toolsFailed")} ); const tools = toolNames(focused); - if (tools.length === 0) return This server has no tools; + if (tools.length === 0) return {t("playground.chat.mcp.noTools")}; const allowed = allowedTools(toolLimits, focused, tools); return (
@@ -295,7 +311,7 @@ export function McpDialog({ {replaceKey}
@@ -333,9 +349,9 @@ export function McpDialog({ return ( !open && setReplacingKey(null)}> - }> + }> - MCP + {t("playground.chat.mcp.trigger")} {selection.length > 0 && ( {selection.length} @@ -344,23 +360,28 @@ export function McpDialog({ - MCP servers - - Pick the servers the model can use, then limit their tools. A server allows every tool until you uncheck - some. - + {t("playground.chat.mcp.title")} + {t("playground.chat.mcp.description")} {nothingAvailable ? (
- - Add one on the MCP servers page, or send as a key that can reach one + + }} + />
) : (
- +
{isLoading ? ( @@ -370,11 +391,18 @@ export function McpDialog({ ) : (
+ -
)}
@@ -383,7 +411,7 @@ export function McpDialog({
)} - }>Done + }>{t("playground.chat.mcp.done")}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx index bbea909711a..3daef04b206 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx @@ -1,6 +1,9 @@ "use client"; import { Copy, Split } from "lucide-react"; +import type { TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; +import i18n from "@/i18n/i18n"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; @@ -8,24 +11,30 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { cn } from "@/lib/cva.config"; import { toast } from "@/lib/toast"; -export function copyText(value: string, label: string): void { +export function copyText(value: string, label: string, t: TFunction = i18n.t.bind(i18n)): void { void navigator.clipboard.writeText(value).then( - () => toast.success(`${label} copied`), - () => toast.error("Unable to copy to clipboard"), + () => toast.success(t("playground.chat.modelName.copied", { label })), + () => toast.error(t("playground.chat.modelName.copyFailed")), ); } export function CopyIconButton({ value, label }: { value: string; label: string }) { + const { t } = useTranslation(); return ( copyText(value, label)} /> + )} } + render={ +
{isWide ? ( -