diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 76af64c8ccd..7180d3c6b7c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -377,6 +377,7 @@ jobs: test-path: >- tests/unit/llms/openai tests/unit/llms/meta + tests/unit/llms/test_model_inventory.py python-version: "3.12" workers: 4 reruns: 0 @@ -950,7 +951,7 @@ jobs: dist: loadscope steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: 3 + timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 8 || 3 }} with: persist-credentials: false fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} diff --git a/examples/model_metadata/README.md b/examples/model_metadata/README.md new file mode 100644 index 00000000000..7103f2d260e --- /dev/null +++ b/examples/model_metadata/README.md @@ -0,0 +1,60 @@ +# Gateway model discovery + +LiteLLM advertises model limits and capabilities in its normal `/v1/models` response and detailed `/model/info` response + +The shared metadata contract distinguishes `context_window`, the total combined input and generated output limit, from `max_input_tokens` and `max_output_tokens`. Unknown values are omitted. LiteLLM does not add the two maxima to derive context, or use a context value as an output limit + +Configure the models you want to expose in `model_list`. LiteLLM retrieves metadata for those models from supported upstream catalogs at startup, then refreshes every five minutes. OpenAI-compatible servers can publish the shared fields, `context_length`, or `max_model_len`. OpenRouter and Vercel catalogs are normalized from their provider-specific fields. ChatGPT uses the active account's authenticated native model catalog without starting a device login + +Upstream metadata overrides LiteLLM's bundled model catalog. Explicit `model_info` values override upstream metadata. A server that publishes only model IDs still works, and its missing limits can be configured manually in LiteLLM + +```yaml +model_list: + - model_name: shared-model + litellm_params: + model: openai/provider-model + api_base: https://inference.example/v1 + api_key: os.environ/PROVIDER_API_KEY + model_info: + context_window: 120000 + max_input_tokens: 110000 + max_output_tokens: 20000 + supports_function_calling: true + supports_parallel_function_calling: false + supports_reasoning: true + reasoning_effort_levels: [low, high, xhigh] + default_reasoning_effort: high + supported_endpoints: [/v1/chat/completions] + supported_modalities: [text, image] + supported_output_modalities: [text] +``` + +These example limits describe independent constraints. With 20000 output tokens reserved, this example allows at most 100000 input tokens because the total context limit also applies + +Reasoning levels are the exact values the configured route accepts. The list is not restricted to a fixed set of vendor names, and a default outside a known list is omitted. Explicit `false` capabilities and empty lists are preserved + +ChatGPT's native catalog supplies account-specific reasoning choices, defaults, input modalities, and parallel tool support. Its client policy windows remain separate `codex_context_window` and `codex_max_context_window` diagnostics in detailed model information. They are not advertised as physical total context or input limits. Missing canonical limits require verified catalog values or a manual `model_info` definition + +Native Codex UI reasoning aliases are normalized to API effort values. `ultra` is not advertised as a wire effort, and `persistent` maps to `disabled` + +For a model group with an explicit total context contract, discovery advertises the smallest known input, output, and total limit across its deployments. A limit is omitted if any deployment does not declare it. Capabilities require agreement across deployments, supported lists are intersected, and a default must agree across deployments. Existing input/output aggregation is preserved for legacy groups without a total context field + +Only configured models are exposed. Fetching a supplier's catalog does not add its other models to the downstream listing. Set `general_settings.disable_model_info_refresh: true` to disable upstream catalog polling + +Configured output budgets are request defaults, independent of supplier limits. Set `model_info.request_defaults` to advertise and apply the same policy centrally + +```yaml +model_info: + context_window: 262144 + request_defaults: + output_token_budget: 8192 + output_token_budget_by_reasoning_effort: + low: 65536 + high: 65536 + xhigh: 65536 + max: 131072 +``` + +The route chooses a configured budget from the effective request or deployment reasoning effort when the caller omits all explicit output budget parameters. An unmatched effort uses `output_token_budget`. Chat Completions and Anthropic Messages receive `max_tokens`, and Responses receives `max_output_tokens`. Caller-provided `max_tokens`, `max_completion_tokens`, or `max_output_tokens` takes precedence. The policy object is not sent to the supplier + +Discovery advertises `request_defaults` only when every deployment has the same policy. These chosen budgets do not populate `max_output_tokens`, establish a supplier maximum, or change the total context window. Clients can reserve the selected budget during compaction while the supplier's independent output limit remains unknown diff --git a/examples/model_offerings/ARCHITECTURE.md b/examples/model_offerings/ARCHITECTURE.md new file mode 100644 index 00000000000..9ef9c05bd0d --- /dev/null +++ b/examples/model_offerings/ARCHITECTURE.md @@ -0,0 +1,227 @@ +# How selected offerings become a client catalog + +Automatic supplier metadata is the normal path for AUTO offerings. Selecting an OpenRouter endpoint is a routing decision, not an instruction to replace discovery with handwritten capabilities. Keep overrides for an intentional operator policy or an evidenced missing value, not merely because a model was configured + +The schema still requires an explicit `source`. Normal automatic usage selects `source: auto` without copying a capability card; it does not mean an omitted `source` field is accepted + +This walkthrough includes the upstream sync through `dddbaa803d6e9fcc862ea7a5412f771cb1ae040e`. Relative source links follow the code in the same checkout, and the PR records validation against its published head. The feature extends native LiteLLM adapters, types, authentication, listing and Router paths; referencing those paths does not mean every implementation in them originated in this PR. See the [setup guide](README.md) for configuration and lifecycle instructions + +```text +supplier connection -> inventory -> native normalization -> offering resolution + -> Router generation -> authorized gateway catalog + -> client options -> request mapping -> supplier +``` + +## 1. Connection, offering and metadata are different layers + +The [configuration types](../../litellm/types/proxy/model_offerings.py) separate these concerns + +| Layer | What it owns | What it does not imply | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Supplier connection | Native provider/protocol, trusted or configured API base, authentication and headers | That every model at the supplier becomes public | +| Selected offering | Public `model_name`, `upstream_model`, connection reference, enabled state, AUTO/MANUAL source and optional provider policy | That the operator must write a capability card or implement a provider plugin | +| Metadata and request defaults | Limits, capabilities, effort choices/default, API surfaces/modalities and optional output-budget policy | That routing selection, UI choices and physical limits are interchangeable | + +The `provider` on an offering references a connection name. That connection's `provider` selects an existing adapter, currently `chatgpt`, `openrouter`, `vercel_ai_gateway` or generic `openai` in this external-file schema. A custom connection name or public alias does not create custom provider code. A service that already speaks the supported OpenAI-compatible protocol can use the generic adapter; new provider implementation is needed only when its authentication, protocol or request/response behavior actually requires one + +## 2. Retrieve an inventory, without exposing it wholesale + +The [native inventory dispatcher](../../litellm/llms/model_inventory.py) chooses the supplier path. Discovery happens on the gateway using the selected connection's identity. A client does not need a second connection or credentials for the supplier + +| Connection | Inventory and normalization path | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Native ChatGPT account | The [ChatGPT adapter](../../litellm/llms/chatgpt/model_info.py) reads the saved server OAuth account and calls the trusted backend's `/models?client_version=...`. An explicit base must match that backend. Discovery requests existing access tokens with device login disabled | +| OpenRouter | The [OpenAI-compatible inventory](../../litellm/llms/openai_like/model_info.py) fetches the configured models catalog. Pinned AUTO offerings additionally use the [OpenRouter endpoint adapter](../../litellm/llms/openrouter/model_info.py) for `/models/{author}/{slug}/endpoints` under that API base | +| Vercel AI Gateway | Uses the compatible catalog request, then interprets Vercel's model kind, modalities, reasoning options and language-model output-budget field | +| Generic compatible service | Uses the configured `/v1/models` catalog and recognized metadata fields, without guessing a model family from its name | + +`add_path_to_api_base` handles an API base that already includes its version path. Discovery does not blindly append another `/v1` to it. Provider-specific native getters resolve OpenRouter/Vercel bases rather than maintaining a second URL registry + +Endpoint requests are deduplicated and restricted to enabled, selected AUTO models with a policy and a successful model-inventory membership result. The implementation does not request endpoint details for every model in a large supplier catalog. MANUAL metadata resolution ignores inventory, although the manager may still refresh configured connections + +Three decisions remain separate: a supplier reports a model, the operator selects and enables its public offering, and a caller is authorized to discover/use it. Downloading a model does not automatically enable it or grant it to any key + +## 3. Normalize meaning, not just field names + +[ModelMetadata and ModelRequestDefaults](../../litellm/types/router.py) provide the shared native facet used by inventory, deployments and listing + +| Field | Meaning | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `context_window` | Combined input and generated-output token capacity | +| `max_input_tokens` | Independently known input ceiling, not context minus output | +| `max_output_tokens` | Independently known generated-output ceiling, not a requested default | +| `supports_function_calling`, `supports_parallel_function_calling`, `supports_reasoning` | Known capabilities, including an explicit false when the source supports that conclusion | +| `reasoning_effort_levels` | Exact accepted effort choices, not a guessed low/medium/high ladder | +| `default_reasoning_effort` | A compatible advertised default or normalized API-facing default, subject to the provider qualifications below | +| `supported_endpoints`, `supported_modalities`, `supported_output_modalities` | Known API surfaces and input/output modalities | +| `request_defaults` | Optional requested output budgets, separate from all physical limits | + +Unknown optional fields are omitted after applicable sources are resolved. False and an empty list remain meaningful values. Numeric zero in supplier limits is treated as unknown; malformed supplier metadata is not evidence of a successful empty inventory. Validation also prevents an effort default outside a known choice list and clears choices/defaults when reasoning is explicitly unsupported + +### Provider-specific interpretations + +The generic card adapter recognizes native fields and context aliases such as `max_model_len` and `context_length`. For unpinned OpenRouter cards it also reads `top_provider`, architecture and model-level reasoning information. For Vercel, `type` can supply chat/embedding mode, `reasoning_options` supplies effort choices, and `max_tokens` is treated as output capacity only for a language model, not blindly for embeddings + +Native ChatGPT's Codex `context_window` and `max_context_window` are retained as `codex_context_window` and `codex_max_context_window` in inventory/deployment metadata. They are not unconditionally promoted to generic physical `context_window` or independent input capacity. Independently reported input/output fields can still be used. These Codex-specific extras are not part of the generic `/v1/models` projection, so their retention is not a promise that every client receives them + +ChatGPT discovery removes `ultra` from API effort choices because it is a multi-agent UI choice. It translates the catalog spelling `persistent` to advertised API spelling `disabled`. For a native default of `ultra`, the current normalization chooses a supported `multi_agent_reasoning_effort`, otherwise supported `max`, otherwise the last remaining reported API effort, otherwise omits the default. This is an explicit UI-to-API normalization policy, not proof that multi-agent Ultra is equivalent to a particular single-request effort or that the fallback is a measured backend default + +OpenRouter effort choices describe OpenRouter's gateway contract, not an independently verified native enum for a supplier such as StreamLake. Explicit `reasoning.supported_efforts: null` expands to the documented gateway effort set; omission does not declare choices; `[]` remains empty. Mandatory reasoning excludes `none`. A disabled-by-default model does not claim an active default effort, while an explicitly reported compatible `none` default can remain. Boolean reasoning support alone never creates choices + +## 4. Resolve AUTO, pinned AUTO and MANUAL + +The precedence is implemented by `_deployment` in the [offering manager](../../litellm/proxy/model_offerings.py), not by a client-side table + +| Mode | Availability | Metadata resolution, lowest to highest priority | Missing data | +| ----------------------- | ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| AUTO, unpinned | Enabled and present in the connection's latest successful model inventory | Provider-appropriate bundled metadata fallback, then downloaded normalized supplier fields, then explicit non-null `model_info` fields | Missing supplier fields can still have known bundled fallback. Remaining unknown fields are omitted | +| AUTO, pinned OpenRouter | Same model-membership rule, with endpoint policy retained | Metadata from eligible endpoints plus applicable model-level reasoning information, then explicit non-null `model_info` fields | No unrestricted aggregate or bundled capacity substitution. Unknown endpoint guarantees stay omitted | +| MANUAL | Enabled, independent of supplier inventory membership | Handwritten `model_info` only | Missing metadata remains unknown. The existing protocol adapter can still perform inference | + +For unpinned AUTO fallback, the manager looks up the provider-prefixed model in the native loaded cost map and permits an unprefixed fallback for generic `openai`. It reuses the native catalog rather than building another pricing/model registry. Consequently, a field on an AUTO GPT offering need not have come directly from the latest live JSON. Unpinned OpenRouter metadata is a model-level card; it does not certify which endpoint will serve an individual future request + +For pinned AUTO, `OpenRouterProviderPolicy.allows_endpoint` matches full slugs and base/variant prefixes, with service-tier suffixes requiring explicit selection. For every independently known limit, the common native aggregator takes the minimum across eligible endpoints. A limit unknown on any eligible endpoint stays unknown. Capability guarantees and lists are combined conservatively, and a default is retained only when compatible across sources. Status, uptime, price and performance do not change this metadata eligibility calculation or operator availability policy + +Effort choices/defaults are combined with endpoint metadata only where eligible endpoints declare `reasoning_effort` support. More restrictive endpoint declarations narrow model-level choices. An endpoint permitting reasoning without the effort control does not justify advertising an effort selector. Missing/failed endpoint discovery never borrows a larger context from another supplier + +### What an override actually does + +An explicit valid, non-null field overrides that field, not the entire downloaded card. A false capability or `[]` choice list survives the merge, subject to native consistency validation. `null` is omitted by the configured metadata dump and therefore does not clear an inherited value. A supplied `request_defaults` object replaces that field as an object; this is not a recursive merge of its budget map + +The override facility is optional. Copied endpoint values used as a temporary workaround are not automatically an intentional long-term policy. Remove unnecessary copied limit/capability fields from an AUTO definition to let discovery supply and refresh them; retain genuine operator policy and required configuration such as pinned `mode: chat`. Keeping the facility does not require keeping a workaround forever + +For a sole eligible endpoint reporting context `300000` and output `30000`, AUTO without intentional overrides advertises those bounds. If it later reports `280000` and `24000`, the next successful refresh publishes those values for subsequent requests. An unknown independent input ceiling remains omitted in both generations. These are illustrative numbers, not a claim about a named supplier + +## 5. Configure selection without copying a card + +These snippets illustrate standalone external-file definitions. Replace placeholder model IDs with exact IDs from the relevant catalog. They do not assert current capabilities for a vendor model or modify an existing deployment + +### Native GPT offering, automatically described + +```yaml +version: 1 +providers: + account: + provider: chatgpt +offerings: + - model_name: gpt-auto + provider: account + upstream_model: catalog-model-slug + source: auto + enabled: true +``` + +No capability block is needed to enable discovery. The account must already be authorized through the native server mechanism. Add a metadata field only for an intentional exception or evidenced information the automatic sources do not provide + +### Endpoint selection, still automatically described + +```yaml +version: 1 +providers: + aggregator: + provider: openrouter + api_key: os.environ/OPENROUTER_API_KEY +offerings: + - model_name: pinned-auto + provider: aggregator + upstream_model: organization/model + source: auto + enabled: true + provider_policy: + provider: openrouter + only: [supplier/fp8] + allow_fallbacks: false + model_info: + mode: chat +``` + +The current schema requires the matching OpenRouter supplier and explicit chat mode for a policy. It does not require static context, output or reasoning fields. Routing selection and automatic metadata remain separate + +### Deliberately maintained compatible offering + +```yaml +version: 1 +providers: + custom: + provider: openai + api_base: https://supplier.example/v1 + api_key: os.environ/CUSTOM_SUPPLIER_API_KEY +offerings: + - model_name: custom-flash + provider: custom + upstream_model: organization/model + source: manual + model_info: + mode: chat + context_window: 262144 + supports_function_calling: true + supports_parallel_function_calling: true + supports_reasoning: true + reasoning_effort_levels: [low, high, xhigh, max] + supported_endpoints: [/v1/chat/completions] + supported_modalities: [text, image] + supported_output_modalities: [text] + request_defaults: + output_token_budget: 65536 + output_token_budget_by_reasoning_effort: + max: 131072 +``` + +This is an explicit MANUAL use case, such as a compatible service without a usable inventory. The context and capability block are operator assertions requiring evidence, not supplier-discovered proof. There is deliberately no independent `max_output_tokens` capability. The output budgets do not establish one. Naming a connection `custom` or using this alias still invokes the generic `openai` adapter, not a custom model-family implementation + +## 6. A requested budget is not a capability + +The [budget helper](../../litellm/router_utils/model_request_defaults.py) defaults requests only for supported chat/Responses/Messages call types when a `request_defaults` policy exists and the caller has not supplied a non-null `max_tokens`, `max_completion_tokens` or `max_output_tokens` + +It chooses the budget's effort key from request effort first, then deployment request settings, then advertised `default_reasoning_effort`. It understands a direct effort string/object and `reasoning.effort`. An exact matching effort budget wins over the base output budget. If neither supplies a budget, nothing is injected. There is no guessed effort-strength conversion + +The helper returns `max_output_tokens` for Responses calls and `max_tokens` for the other supported calls. It does not inject a reasoning effort just because a default was advertised. An explicit caller budget wins over this configured default-budget policy, and conflicting deployment budget parameters are removed in that policy path. Native/provider request translation still determines the eventual wire format + +The selector reads `reasoning_effort` and `reasoning.effort`; it is not a second implementation of every provider's thinking controls. In particular, its budget choice precedes later native request-level effort rounding. Do not assume an effort-specific default budget is recomputed for a different, unsupported effort that upstream subsequently rounds, or that every `output_config`/thinking-budget shape supplies this selector's effort key + +For the manual example, an omitted effort uses the base requested budget of `65536`; explicit `max` selects `131072`. Neither value means the physical output ceiling is known. This helper is defaulting, not a provider-independent quota or proof that a chosen budget is feasible + +An advertised reasoning default, a client electing to send that default, an explicit deployment request setting and a provider adapter's own default behavior are different things. The offering manager itself does not turn every advertised default into an outgoing request parameter. With no explicit effort setting or provider-specific injection, omission leaves the supplier's behavior in charge + +## 7. Publish one resolved generation, then authorize its catalog + +`ModelOfferingsManager._apply` compiles the resolved definitions through [Router.snapshot_with_model_list](../../litellm/router.py), setting `model_metadata_authoritative=True`. Native deployment records carry private typed provider policy, not an independent public policy registry + +The [serving view and ASGI middleware](../../litellm/proxy/offering_router.py) publish one generation atomically and pin that generation for each HTTP request, including the streaming response lifetime. Subsequent requests can see a newer generation while the earlier request finishes on its original view. This is a generation-pinning contract, not a claim that the whole native Router and all shared clients are deeply immutable. The facade blocks external model-definition mutation while reusing native resources + +The [shared model checks](../../litellm/proxy/auth/model_checks.py), native key/team/alias authorization and offering listing guard determine which selected public names the caller can see. Wildcard grants match live selected public Router names, not only old supplier registry entries. This does not broaden an exact key into access to other offerings. Alias lookup reuses the native listing/deployment context + +[create_model_info_response](../../litellm/proxy/utils.py) constructs the public object through the [discovery projection](../../litellm/proxy/model_discovery.py). In authoritative external mode it does not perform a second unrelated catalog fallback after offering resolution. The intended unpinned bundled fallback has already happened in `_deployment`. Private credentials, deployment identity and routing policy are not public model metadata + +## 8. Client options and the inference path + +A client reads its authorized gateway `/v1/models` result. A generic client can present the exact effort list as choices or variants, keep context/input/output distinct, show an advertised default and use a request-default policy when constructing a request. It must keep unknown fields unknown instead of manufacturing a supplier-specific limit or effort ladder + +Variant syntax and UI conversion belong to the client, not this LiteLLM PR. A client that rejects a requested effort variant when its catalog has no variants can do so before any inference request. This document does not claim to implement or test OpenCode's local validation or interactive UI, and no OpenCode source change is needed to define the gateway catalog contract + +For inference, the public alias resolves to a selected native deployment. The Router applies request-default budget policy and dispatches through the selected provider adapter. The [OpenRouter chat adapter](../../litellm/llms/openrouter/chat/transformation.py) preserves explicit gateway `max`/`xhigh`, and the [Messages bridge normalization](../../litellm/llms/anthropic/pass_through/utils.py) does not reduce OpenRouter effort using unrelated backend flags. Native Responses and Responses-to-chat paths retain their API-specific request shapes + +Upstream's [deployment reasoning resolver](../../litellm/router_utils/reasoning_effort_capability.py) now owns deployment-declared effort handling across chat, Responses and Messages. An accepted declared value is kept. A ranked request outside the declared set can be rounded according to native up/down policy, or dropped under native `drop_params` behavior; explicit `default` resolves to omission. Cost-map-only choices do not activate deployment rounding. Supplier choices resolved into an AUTO deployment's own `model_info` do count as its declaration. The external offering schema does not add a parallel rounding framework or expose every native Router setting + +Consequently, preserving accepted OpenRouter gateway efforts is not an all-provider or all-input passthrough guarantee. The fork still avoids the provider adapter's old unconditional max-to-xhigh rename and legacy undeclared OpenRouter approximation, while reusing upstream's declared-effort resolver. Other provider adapters keep their native request semantics. The shared native [deployment-pricing resolver](../../litellm/router_utils/deployment_pricing.py) also owns base-model key precedence for listing; the former fork-only key lookup is removed + +The offering guard merges the exact configured `only`/`allow_fallbacks: false` body and checks it again at deployment dispatch. Caller connection, deployment, provider, fallback, tier and parameter-dropping overrides cannot replace it. Pinned response-cache reads/writes are disabled to prevent output from another endpoint satisfying the request. No `require_parameters` setting or fallback supplier is silently added. Unsupported pinned API surfaces and native batches are rejected rather than bypassing selection + +## 9. Refresh, failure and present limitations + +Configuration polling defaults to five seconds, configurable from one to 300. Inventory polling defaults to 300 seconds, configurable from one to 86400. Known connection/base/credential or native-account changes invalidate the relevant prior state; changed inventory intervals or selected pinned-model requirements trigger fresh discovery. This is periodic discovery and atomic publication, not an instantaneous supplier query for every client request + +Inventory HTTP requests have a five-second timeout, a two-megabyte response bound and redirects disabled. The manager bounds concurrent provider refreshes to eight; endpoint requests within a provider are sequential. Cache identity includes supplier/base/authentication, and endpoint retention additionally checks credential scope. Positive compatible/endpoint inventories use a 300-second cache TTL and compatible/endpoint failures a 60-second negative TTL; the manager's forced scheduled refresh bypasses cached results + +A successful missing-model inventory removes an AUTO model. A failed fetch is not an empty inventory and retains last-known data only where identity permits. A successful empty endpoint inventory clears derived endpoint metadata without itself disabling an otherwise selected model. An initial endpoint failure leaves fields unknown rather than using unrestricted capacity. An invalid file edit or rejected candidate Router leaves the previous complete serving configuration in place + +Important boundaries of the current implementation remain + +- Automatic metadata is limited to fields the adapters recognize and the applicable bundled fallback can justify. It is not independent benchmarking, and unknown values can legitimately remain absent +- Raw Codex UI windows are retained internally without a generic physical-window guarantee. The Ultra default fallback is a normalization rule, not evidence of backend equivalence +- The [native ChatGPT Responses adapter](../../litellm/llms/chatgpt/responses/transformation.py) filters outgoing keys and currently excludes output-token budget fields. Do not promise that a configured budget is enforced on that backend merely because the shared helper can select it +- Pinning supports the current strict OpenRouter chat policy schema. Unknown routing controls, endpoint health/account-specific eligibility not represented by that schema, or an undocumented supplier-native effort mapping are not established by this metadata calculation +- An explicit override must be internally consistent and is not a deletion mechanism when set to null. A copied workaround can freeze AUTO fields until removed; it is not a required permanent policy +- Other clients may interpret catalog extensions differently. The regression evidence is in-process inventory, ASGI, Router and captured HTTP requests, not interactive client or paid supplier proof + +The [endpoint inventory tests](../../tests/unit/llms/openrouter/test_model_info.py), [offering catalog/request tests](../../tests/unit/proxy/test_model_offerings.py), [budget tests](../../tests/unit/router_utils/test_model_request_defaults.py) and [native account tests](../../tests/unit/llms/chatgpt/test_model_info.py) provide concrete examples of these contracts without deployment-local configuration or credentials diff --git a/examples/model_offerings/README.md b/examples/model_offerings/README.md new file mode 100644 index 00000000000..e32c77b7fb5 --- /dev/null +++ b/examples/model_offerings/README.md @@ -0,0 +1,113 @@ +# External model offerings + +For the source-backed flow from supplier inventory to client options and requests, see [How selected offerings become a client catalog](ARCHITECTURE.md). It explains automatic metadata authority, intentional overrides, pinned endpoint guarantees and current adapter limitations + +Set `general_settings.model_offerings_path` in the normal proxy configuration and keep its `model_list` empty. Disable `store_model_in_db`; the database may still serve authentication and usage tracking. Existing proxy configurations behave as before when the path is absent + +```yaml +model_list: [] +general_settings: + model_offerings_path: /app/offerings/offerings.yaml + store_model_in_db: false +``` + +Mount a host directory into Docker, for example `./offerings:/app/offerings:ro`. Keep `offerings.yaml` inside it. Replace the file atomically within that directory so the container sees the new inode. A bind mount of an individual file does not reliably see atomic replacement + +```yaml +version: 1 +config_poll_seconds: 5 +inventory_poll_seconds: 300 +providers: + openrouter: + provider: openrouter + api_key: os.environ/OPENROUTER_API_KEY + vercel: + provider: vercel_ai_gateway + api_key: os.environ/VERCEL_API_KEY + local: + provider: openai + api_base: http://inference-server:8000/v1 + api_key: os.environ/INFERENCE_API_KEY + subscribed: + provider: chatgpt +offerings: + - model_name: selected + source: auto + provider: openrouter + upstream_model: organization/model + enabled: true + - model_name: handwritten + source: manual + provider: openrouter + upstream_model: organization/another-model + model_info: + mode: chat + context_window: 262144 + supports_function_calling: true + request_defaults: + output_token_budget: 8192 + output_token_budget_by_reasoning_effort: + low: 65536 + high: 65536 + xhigh: 65536 + max: 131072 +``` + +These example token values illustrate operator configuration, not a claim about the example supplier models. Request defaults express chosen budgets and do not populate independent output capability limits + +`source` belongs to each offering. AUTO entries require membership in the latest successful full inventory for their connection. Missing entries disappear from model listings and return a useful unavailable error before forwarding. A returning entry becomes available again. AUTO metadata combines the verified catalog with downloaded supplier fields and then explicit overrides, in increasing priority, and freezes the result in the serving snapshot. MANUAL entries use only handwritten definitions and are independent of supplier inventory membership. Both sources accept shared `model_info` overrides. Wildcards and duplicate public aliases are rejected + +Startup validates the complete file and awaits initial inventory requests. AUTO entries remain unavailable when no successful inventory exists; MANUAL entries remain usable. A successful empty inventory means absence. HTTP errors, authentication failures, timeouts and malformed responses retain the last successful inventory for unchanged credentials. Changing connection credentials starts with unknown availability and forces a fresh inventory. ChatGPT uses its existing native OAuth account and cannot initiate device login from discovery or request handling + +The file is checked every five seconds by default, with a configurable range of 1 to 300 seconds. Inventories refresh every 300 seconds by default, with a range of 1 to 86400 seconds. Changing that interval triggers a fresh inventory request and resets its deadline. Invalid file edits retain the whole last-valid serving configuration, including credentials, and log only the error kind. Unselected inventory entries never become public routes. Caller connection overrides and runtime model-management mutations are rejected in external mode; edit the external file instead + +Each HTTP request pins one immutable serving snapshot through the complete ASGI response lifetime, including streaming. Publishing a new snapshot changes subsequent requests while requests already running finish on their previous native Router. Snapshot creation isolates deployment indexes and metadata caches while reusing the initialized Router's bounded client, usage and routing resources. It does not register fresh callbacks or close shared HTTP clients on each reload. The polling task stops at proxy shutdown after requests drain. Inventory refreshes reacquire the dedicated shared HTTP client through the native cache, whose eviction lifecycle owns client closure + +Native subscription inventory uses the server-owned backend from `CHATGPT_API_BASE` / `OPENAI_CHATGPT_API_BASE`, or the provider default. An explicit connection URL must match that backend; per-model destinations cannot receive saved server OAuth credentials. Discovery never starts device login. Supplier-specific inventory behavior lives in the shared `litellm/llms/model_inventory.py` dispatcher. + +Connection identities use HMAC with the existing private server salt/master key. With that key configured, deployment IDs remain stable across restarts and change when supplier credentials or native account identity changes. No additional configured secret is required. Unconfigured standalone/development mode uses a private process key and does not promise ID stability across restart. Upgrading from earlier unkeyed offering IDs changes IDs once: encrypted reasoning tied to a removed ID may be discarded on replay or model switch, while readable history and model limits/defaults remain available. Native affinity callbacks use the request-pinned serving view during reloads. + +Caller `extra_body.model` overrides are rejected before supplier forwarding; harmless extra body fields remain supported. Structured `reasoning_effort` objects such as `{effort: high, summary: auto}` select the same configured output budget as the corresponding effort string. + +## Provider endpoint policy + +An offering can own a typed `provider_policy`. Currently OpenRouter chat offerings support an exact endpoint allowlist with fallbacks disabled. The policy belongs to the offering, not the connection, and never accepts credentials or arbitrary request-body options. The supplier must match the policy provider and `model_info.mode` must explicitly be `chat` + +```yaml +offerings: + - model_name: selected-endpoint + source: auto + provider: openrouter + upstream_model: organization/model + enabled: false + provider_policy: + provider: openrouter + only: [supplier/exact-endpoint] + allow_fallbacks: false + model_info: + mode: chat + context_window: 32000 + max_output_tokens: 4000 +``` + +Append this entry to the external file's existing `offerings`, using its configured OpenRouter connection name. The example limits are synthetic overrides. Omit them to discover bounds automatically. `only` contains provider slugs, not URLs. Base slugs include their variants, and variant slugs include their regions, following OpenRouter's matching contract. Service-tier endpoints require an explicit tier slug. Multiple slugs intentionally allow any matching endpoint; `allow_fallbacks` must remain false + +The gateway merges its policy into harmless caller `extra_body` fields and verifies it again at deployment dispatch. Caller `provider`, `models`, `route`, `service_tier`, fallback controls, parameter-dropping/allowlist controls, nested body overrides and mock responses are rejected for policy-controlled offerings, even through aliases. Tier overrides could change the eligible endpoint set and invalidate its advertised guarantees; they are rejected rather than silently reduced or rerouted. Unpinned tier controls remain unchanged. Caller deployment selectors are rejected throughout external mode. Policy-controlled requests disable LiteLLM model-group fallbacks as well as OpenRouter provider fallback. Unavailable endpoints fail instead of serving a different offering. Retries may still contact the same pinned endpoint + +Policy-controlled requests disable response-cache reads and writes, overriding caller cache settings. Existing response-cache keys do not reliably separate provider endpoints; cached output from another endpoint must not satisfy a pinned request + +Chat Completions, native Responses and translated Anthropic Messages preserve the policy, including streaming. Other APIs are rejected for policy-controlled offerings. Batch execution remains unavailable throughout external mode. Offerings without policy retain existing provider-routing behavior, apart from the deployment-selector restriction + +AUTO still checks model membership. Pinned AUTO offerings additionally discover only their selected models' endpoint inventories, using the configured connection, bounded native HTTP client and credential-scoped cache. Aggregate model limits and bundled metadata never substitute for pinned endpoint limits. Context, independent input and output bounds are the conservative minimum across all eligible endpoints, and a field unknown on any eligible endpoint is omitted. Input capacity is not inferred by subtracting output from context + +Reasoning choices and defaults come from OpenRouter's model-level reasoning contract and are advertised only when every eligible endpoint declares the effort control. An endpoint's more restrictive declared choices narrow that set. This describes OpenRouter's accepted gateway options, not an independently discovered backend-native effort enum. An explicit empty choice list remains empty. Explicit null expands only to OpenRouter's documented gateway efforts, while omission does not declare choices. Mandatory reasoning excludes the disable value, and a disabled-by-default model does not claim an active default effort. Boolean reasoning support alone never supplies choices. Endpoint capability guarantees are combined conservatively, without using status, uptime or performance to change routing or availability + +Every explicit `model_info` field remains authoritative. Existing handwritten context/output limits continue overriding discovery until the operator removes them. MANUAL offerings remain entirely handwritten. An advertised default does not itself inject request effort. Accepted declared OpenRouter gateway efforts, including `max` and `xhigh`, are preserved without the old provider rename or backend-flag approximation. Upstream's native deployment resolver owns unsupported-value rounding/default/drop behavior across providers and routes; this is not a promise to forward every arbitrary effort unchanged + +Successful empty endpoint inventories clear derived endpoint metadata. Failed refreshes retain last-known endpoint metadata only for the same connection identity; an initial failure leaves unknown fields omitted, never replaced with another supplier's limits. Endpoint discovery does not disable an otherwise selected model or broaden its policy. Metadata refreshes publish atomically, and requests already running retain their previous snapshot. Policy edits still change the affected offering's private deployment ID + +The endpoint and reasoning provenance follows OpenRouter's [endpoint API](https://openrouter.ai/docs/api/api-reference/endpoints/list-all-endpoints-for-a-model), [per-model reasoning options](https://openrouter.ai/docs/guides/best-practices/reasoning-tokens#discovering-per-model-reasoning-options) and [provider slug matching](https://openrouter.ai/docs/guides/routing/provider-selection#base-slug-matching), checked October 15, 2026 + +Native provider batch creation and uploads with purpose `batch` are rejected in external offering mode. Native batch records can select their own supplier models, which would bypass the configured offering catalog. Existing proxy configurations without external offerings retain native batch support. + +The optional bundled cost-map JSON-schema extension is deferred: its trusted-base guard validates PR artifacts with the base generator. The runtime metadata/configuration contract validates separate total/input/output limits and defaults independently, and unknown optional bundled catalog properties remain forward compatible. diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..0e49951a0ee 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -90,7 +90,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = False, # deprecated param, use `turn_off_message_logging` instead message_logging: bool = True, - **kwargs, + **kwargs: object, # kwargs-ok: legacy logger initialization accepts and ignores extension options ) -> None: """ Args: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 00c6ce97e78..9b9cf4da5ca 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -229,7 +229,7 @@ def is_expected_client_error(exception: BaseException | None) -> bool: return 400 <= status < 500 -def coerce_token_limit(value: object) -> int | None: +def coerce_token_limit(value: object, *, strict_positive: bool = False) -> int | None: """ Coerce a max_input_tokens / max_output_tokens value to an int, treating a malformed value as absent. @@ -241,11 +241,14 @@ def coerce_token_limit(value: object) -> int | None: Args: value: The raw configured or cost-map value + strict_positive: Require positive integers without legacy numeric coercion Returns: The value as an int, or None if it is missing or not a usable number. Bools are rejected because True/False is never a meaningful token limit. """ + if strict_positive: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None if isinstance(value, bool): return None if isinstance(value, int): diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/pass_through/utils.py index 31619a04b74..f466d1d3078 100644 --- a/litellm/llms/anthropic/pass_through/utils.py +++ b/litellm/llms/anthropic/pass_through/utils.py @@ -221,6 +221,8 @@ def normalize_reasoning_effort_value( if deployment_rounds_reasoning_effort(deployment_model_info): return effort + if custom_llm_provider == "openrouter" or custom_llm_provider is None and model.startswith("openrouter/"): + return effort from litellm.utils import get_model_info try: diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index ff9c6476cba..977a0562f63 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -2,7 +2,9 @@ import base64 import json import os import time -from collections.abc import Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from typing import Final, TypeAlias import httpx @@ -38,6 +40,16 @@ OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" JsonObject: TypeAlias = Mapping[str, JsonValue] _JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) +_DEVICE_LOGIN_ENABLED: Final[ContextVar[bool]] = ContextVar("chatgpt_device_login_enabled", default=True) + + +@contextmanager +def prevent_device_login() -> Generator[None]: + token: Final = _DEVICE_LOGIN_ENABLED.set(False) + try: + yield + finally: + _DEVICE_LOGIN_ENABLED.reset(token) def _optional_str(value: JsonValue | None) -> str | None: @@ -64,7 +76,7 @@ class Authenticator: def get_api_base(self) -> str: return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE - def get_access_token(self) -> str: + def get_access_token(self, *, allow_device_login: bool = True) -> str: auth_data: Final = self._read_auth_file() if auth_data: access_token: Final = _optional_str(auth_data.get("access_token")) @@ -78,6 +90,12 @@ class Authenticator: except RefreshAccessTokenError as exc: verbose_logger.warning("ChatGPT refresh token failed, re-login required: %s", exc) + if not allow_device_login or not _DEVICE_LOGIN_ENABLED.get(): + raise GetAccessTokenError( + status_code=401, + message="ChatGPT authorization is unavailable; model discovery cannot start device login", + ) + if not can_block_current_thread(): raise GetAccessTokenError( message=( diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index fe33219f110..9078e804114 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -229,7 +229,7 @@ def get_chatgpt_default_headers( access_token: str, account_id: str | None, session_id: str | None = None, -) -> dict: +) -> dict[str, str]: originator: Final = get_chatgpt_originator() user_agent: Final = get_chatgpt_user_agent(originator) headers: Final = { diff --git a/litellm/llms/chatgpt/model_info.py b/litellm/llms/chatgpt/model_info.py new file mode 100644 index 00000000000..5d41fd7656f --- /dev/null +++ b/litellm/llms/chatgpt/model_info.py @@ -0,0 +1,207 @@ +import asyncio +import hashlib +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # typed native cache and HTTP boundaries + +import httpx +from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import coerce_token_limit +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) +from litellm.types.router import ModelMetadata + +from .authenticator import Authenticator +from .common_utils import get_chatgpt_default_headers + +_CATALOG_CLIENT_VERSION: Final = "0.159.3" + + +class _ReasoningLevel(BaseModel): + model_config = ConfigDict(frozen=True) + + effort: str + + +class _ChatGPTModel(ModelMetadata): + slug: str + max_context_window: int | None = None + default_reasoning_level: str | None = None + supported_reasoning_levels: tuple[_ReasoningLevel, ...] | None = None + input_modalities: Sequence[str] | None = None + supports_parallel_tool_calls: bool | None = None + multi_agent_reasoning_effort: str | None = None + + @field_validator("*", mode="before") + @classmethod + def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: + field_name: Final = ( + "context_window" + if info.field_name == "max_context_window" + else "supports_parallel_function_calling" + if info.field_name == "supports_parallel_tool_calls" + else info.field_name + ) + return ModelMetadata.validate_supplier_field(field_name, value) + + @field_validator("max_context_window", mode="before") + @classmethod + def maximum_context(cls, value: object) -> int | None: + return coerce_token_limit(value, strict_positive=True) + + def metadata(self) -> Mapping[str, object]: + native_efforts: Final = ( + self.reasoning_effort_levels + if self.reasoning_effort_levels is not None + else tuple(level.effort for level in self.supported_reasoning_levels) + if self.supported_reasoning_levels is not None + else None + ) + # Codex Ultra is a multi-agent UI choice, not an API reasoning effort. + api_efforts: Final = ( + tuple("disabled" if effort == "persistent" else effort for effort in native_efforts if effort != "ultra") + if native_efforts is not None + else None + ) + native_default: Final = self.default_reasoning_effort or self.default_reasoning_level + default_effort: Final = ( + self.multi_agent_reasoning_effort + if native_default == "ultra" and self.multi_agent_reasoning_effort in (api_efforts or ()) + else "max" + if native_default == "ultra" and "max" in (api_efforts or ()) + else api_efforts[-1] + if native_default == "ultra" and api_efforts + else None + if native_default == "ultra" + else "disabled" + if native_default == "persistent" + else native_default + ) + normalized: Final = ModelMetadata( + max_input_tokens=self.max_input_tokens, + max_output_tokens=self.max_output_tokens, + supports_function_calling=self.supports_function_calling, + supports_parallel_function_calling=( + self.supports_parallel_function_calling + if self.supports_parallel_function_calling is not None + else self.supports_parallel_tool_calls + ), + supports_reasoning=( + self.supports_reasoning + if self.supports_reasoning is not None + else bool(self.supported_reasoning_levels) + if self.supported_reasoning_levels is not None + else None + ), + reasoning_effort_levels=api_efforts, + default_reasoning_effort=default_effort, + supported_endpoints=("/responses",), + supported_modalities=( + self.supported_modalities if self.supported_modalities is not None else self.input_modalities + ), + supported_output_modalities=self.supported_output_modalities, + ) + native_windows: Final = { + "codex_context_window": self.context_window, + "codex_max_context_window": self.max_context_window, + } + return MappingProxyType( + { + **normalized.model_dump(exclude_none=True), + **{key: value for key, value in native_windows.items() if value is not None}, + } + ) + + +class _ChatGPTCatalog(BaseModel): + model_config = ConfigDict(frozen=True) + + models: tuple[_ChatGPTModel, ...] + error: object | None = None + + +def chatgpt_model_inventory_api_base(api_base: str | None = None, *, authenticator: Authenticator | None = None) -> str: + """Saved server OAuth credentials may only be sent to the server-configured backend.""" + auth: Final = authenticator if authenticator is not None else Authenticator() + trusted_base: Final = auth.get_api_base().rstrip("/") + if api_base is not None and api_base.rstrip("/") != trusted_base: + raise ValueError("Native model discovery requires the server-configured API base") + return trusted_base + + +def chatgpt_model_inventory_identity() -> str | None: + try: + return Authenticator().get_account_id() + except Exception: # noqa: BLE001 # unreadable server auth is not proof of an account change + return None + + +async def get_chatgpt_model_inventory( + *, + client: AsyncHTTPHandler, + cache: InMemoryCache, + api_base: str | None = None, + authenticator: Authenticator | None = None, + force_refresh: bool = False, +) -> SupplierModelInventory | SupplierInventoryUnavailable: + inventory_cache: Final = cast(ModelInventoryCache, cache) # cast-ok: native cache implements this protocol + inventory_client: Final = cast(ModelInventoryHTTPClient, client) # cast-ok: native HTTP handler implements get + auth: Final = authenticator if authenticator is not None else Authenticator() + try: + trusted_base: Final = chatgpt_model_inventory_api_base(api_base, authenticator=auth) + account: Final = await asyncio.to_thread(auth.get_account_id) + url: Final = f"{trusted_base}/models?client_version={_CATALOG_CLIENT_VERSION}" + credential_scope: Final = hashlib.sha256(json.dumps((url, account)).encode()).hexdigest() if account else None + except Exception: # noqa: BLE001 # malformed native auth is unavailable, never an authoritative empty inventory + return SupplierInventoryUnavailable("authentication") + try: + token: Final = await asyncio.to_thread(auth.get_access_token, allow_device_login=False) + except Exception: # noqa: BLE001 # unavailable existing OAuth must never initiate a device login + return SupplierInventoryUnavailable("authentication", credential_scope) + try: + headers: Final = { + **get_chatgpt_default_headers(token, account), + "accept": "application/json", + } + cache_key: Final = ( + "chatgpt_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + ) + cached: Final[object] = inventory_cache.get_cache(cache_key) + if not force_refresh and isinstance(cached, SupplierModelInventory): + return cached + response: Final = await inventory_client.get( + url=url, + headers=headers, + timeout=httpx.Timeout(5.0), + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + catalog: Final = _ChatGPTCatalog.model_validate_json(response.content) + ids: Final = tuple(card.slug for card in catalog.models) + if ( + catalog.error is not None + or any(not model_id.strip() or model_id != model_id.strip() for model_id in ids) + or len(frozenset(ids)) != len(ids) + ): + return SupplierInventoryUnavailable("malformed", credential_scope) + inventory: Final = SupplierModelInventory( + MappingProxyType({card.slug: card.metadata() for card in catalog.models}), credential_scope + ) + inventory_cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + except httpx.HTTPStatusError: + return SupplierInventoryUnavailable("http", credential_scope) + except ValidationError: + return SupplierInventoryUnavailable("malformed", credential_scope) + except Exception: # noqa: BLE001 # optional metadata discovery must not interrupt proxy startup + return SupplierInventoryUnavailable("transport", credential_scope) + return inventory diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 34953a79c60..1965130d086 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -113,7 +113,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): } filtered: Final = {k: v for k, v in request.items() if k in allowed_keys} - service_tier: Final = _CHATGPT_SERVICE_TIERS.get(request.get("service_tier")) + requested_service_tier: Final[object] = request.get("service_tier") + service_tier: Final = ( + _CHATGPT_SERVICE_TIERS.get(requested_service_tier) if isinstance(requested_service_tier, str) else None + ) if service_tier is not None: filtered["service_tier"] = service_tier return filtered diff --git a/litellm/llms/model_inventory.py b/litellm/llms/model_inventory.py new file mode 100644 index 00000000000..366ff01372d --- /dev/null +++ b/litellm/llms/model_inventory.py @@ -0,0 +1,137 @@ +"""Supplier-specific inventory dispatch shared by router and selected-offering discovery.""" + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # native HTTP/cache protocol boundary + +from pydantic import TypeAdapter + +import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.chatgpt.model_info import ( + chatgpt_model_inventory_api_base, + chatgpt_model_inventory_identity, + get_chatgpt_model_inventory, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_DISCOVERY_PROVIDERS, get_openai_compatible_model_inventory +from litellm.llms.openrouter.model_info import get_openrouter_endpoint_inventory, openrouter_endpoint_metadata +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) +from litellm.types.router import LiteLLM_Params +from litellm.utils import resolve_api_key_secret_reference + +_EMPTY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) +_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def model_inventory_api_base(provider: str, api_base: str | None) -> str | None: + if provider == "chatgpt": + return chatgpt_model_inventory_api_base(api_base) + if provider == "openrouter": + return litellm.OpenrouterConfig.get_api_base(api_base) + if provider == "vercel_ai_gateway": + return litellm.VercelAIGatewayConfig.get_api_base(api_base) + return api_base + + +def model_inventory_identity(provider: str) -> str | None: + return chatgpt_model_inventory_identity() if provider == "chatgpt" else None + + +async def get_provider_model_inventory( + *, + provider: str, + api_base: str | None, + headers: Mapping[str, str], + client: AsyncHTTPHandler, + cache: InMemoryCache, + force_refresh: bool = False, + endpoint_models: tuple[str, ...] = (), + previous: SupplierModelInventory | None = None, +) -> SupplierModelInventory | SupplierInventoryUnavailable: + if provider == "chatgpt": + return await get_chatgpt_model_inventory( + api_base=api_base, + client=client, + cache=cache, + force_refresh=force_refresh, + ) + resolved_base: Final = model_inventory_api_base(provider, api_base) + if resolved_base is None: + return SupplierInventoryUnavailable("malformed") + inventory: Final = await get_openai_compatible_model_inventory( + provider=provider, + api_base=resolved_base, + headers=headers, + client=client, + cache=cache, + force_refresh=force_refresh, + ) + if provider != "openrouter" or not isinstance(inventory, SupplierModelInventory) or not endpoint_models: + return inventory + return await get_openrouter_endpoint_inventory( + inventory=inventory, + models=endpoint_models, + previous=previous, + api_base=resolved_base, + headers=headers, + client=cast(ModelInventoryHTTPClient, client), # cast-ok: native handler implements bounded inventory GET + cache=cast(ModelInventoryCache, cache), # cast-ok: native in-memory cache implements inventory protocol + force_refresh=force_refresh, + ) + + +def get_inventory_model_metadata( + inventory: SupplierModelInventory, model: str, policy: OpenRouterProviderPolicy | None = None +) -> Mapping[str, object]: + metadata: Final = inventory.models.get(model, _EMPTY_METADATA) + return openrouter_endpoint_metadata(metadata, inventory.endpoints.get(model, ()), policy) if policy else metadata + + +async def get_deployment_model_metadata( + *, + params: LiteLLM_Params, + client: AsyncHTTPHandler, + cache: InMemoryCache, +) -> Mapping[str, object]: + metadata: Final = TypeAdapter(Mapping[str, object]).validate_python(params.model_dump()) + if metadata.get("use_clientside_credentials") or "*" in params.model: + return _EMPTY_METADATA + connection: Final = ( + (params.model.removeprefix("chatgpt/"), "chatgpt", None, params.api_base) + if params.model.startswith("chatgpt/") + else litellm.get_llm_provider(model=params.model, litellm_params=params) + ) + model, provider, dynamic_api_key, api_base = connection + if provider != "chatgpt" and provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return _EMPTY_METADATA + api_key: Final = resolve_api_key_secret_reference( + params.api_key or dynamic_api_key, + model=model, + llm_provider=provider, + upstream=MappingProxyType({"api_base": params.api_base}), + ) + headers: Final = _HEADERS.validate_python(metadata.get("extra_headers") or metadata.get("headers") or {}) + inventory: Final = await get_provider_model_inventory( + provider=provider, + api_base=api_base, + client=client, + cache=cache, + headers=MappingProxyType( + { + **({"authorization": f"Bearer {api_key}"} if api_key else {}), + **{key.lower(): value for key, value in headers.items()}, + } + ), + ) + return ( + inventory.models.get(model, _EMPTY_METADATA) + if isinstance(inventory, SupplierModelInventory) + else _EMPTY_METADATA + ) diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index e3b2dc427c4..14a8abf2ac5 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -1,81 +1,251 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final, TypeAlias +from typing import Annotated, Final, TypeAlias, cast # noqa: TID251 # typed native cache and HTTP boundaries import httpx -from pydantic import BeforeValidator, ConfigDict +from pydantic import BeforeValidator, ConfigDict, ValidationError, ValidationInfo, field_validator -from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openrouter import OPENROUTER_REASONING_EFFORTS +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelInventory, +) +from litellm.types.router import ModelMetadata from litellm.utils import add_path_to_api_base MODEL_INFO_REFRESH_SECONDS: Final = 300 MODEL_INFO_REFRESH_CONCURRENCY: Final = 8 -MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"}) -_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({}) +MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset( + {"hosted_vllm", "openai", "text-completion-openai", "openai_like", "openrouter", "vercel_ai_gateway"} +) +_EMPTY_LIMITS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def model_inventory_cache_key(provider: str, url: str, headers: Mapping[str, str]) -> str: + return ( + "upstream_model_info:" + + hashlib.sha256( + json.dumps((provider, url, sorted((key.lower(), value) for key, value in headers.items()))).encode() + ).hexdigest() + ) def _positive_limit(value: object) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + if value is None or type(value) is int and value == 0: + return None + limit: Final = coerce_token_limit(value, strict_positive=True) + if type(value) is int and limit is not None: + return limit + raise ValueError("Supplier token limits must be nonnegative integers or null") _TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] -class _ModelCard(LiteLLMBaseModel): +class _Modalities(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) + input: Sequence[str] | None = None + output: Sequence[str] | None = None + + +class _Architecture(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + input_modalities: Sequence[str] | None = None + output_modalities: Sequence[str] | None = None + + +class _TopProvider(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + context_length: _TokenLimit = None + max_completion_tokens: _TokenLimit = None + + +class _Reasoning(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + supported_efforts: Sequence[str] | None = None + default_effort: str | None = None + default_enabled: bool | None = None + mandatory: bool | None = None + + def effort_levels(self) -> Sequence[str] | None: + """Explicit null accepts the gateway enum; omission does not declare an effort selector.""" + levels: Final = ( + OPENROUTER_REASONING_EFFORTS + if self.supported_efforts is None and "supported_efforts" in self.model_fields_set + else self.supported_efforts + ) + return ( + tuple(level for level in levels if level != "none" or self.mandatory is not True) + if levels is not None + else None + ) + + +class _ReasoningOption(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str + values: Sequence[str] | None = None + + +class _ModelCard(ModelMetadata): id: str + type: str | None = None + mode: str | None = None max_model_len: _TokenLimit = None context_length: _TokenLimit = None - max_input_tokens: _TokenLimit = None - max_output_tokens: _TokenLimit = None + max_tokens: _TokenLimit = None + supported_parameters: Sequence[str] | None = None + modalities: _Modalities | None = None + architecture: _Architecture | None = None + top_provider: _TopProvider | None = None + reasoning: _Reasoning | None = None + reasoning_options: tuple[_ReasoningOption, ...] = () - def token_limits(self) -> Mapping[str, int]: - context: Final = self.max_model_len or self.context_length - input_limit: Final = self.max_input_tokens or context - output_limit: Final = self.max_output_tokens or context - return MappingProxyType( - { - key: value - for key, value in ( - ("max_tokens", context), - ("max_input_tokens", min(input_limit, context) if input_limit and context else input_limit), - ("max_output_tokens", min(output_limit, context) if output_limit and context else output_limit), + @field_validator("*", mode="before") + @classmethod + def strict_supplier_metadata(cls, value: object, info: ValidationInfo) -> object: + return ModelMetadata.validate_supplier_field(info.field_name, value) + + def token_limits(self, provider: str) -> Mapping[str, object]: + context: Final = self.context_window or self.max_model_len or self.context_length + parameters: Final = self.supported_parameters + supported_tools: Final = "tools" in parameters if parameters is not None else None + supported_reasoning: Final = ( + "reasoning" in parameters or "reasoning_effort" in parameters if parameters is not None else None + ) + metadata: Final = ModelMetadata( + context_window=( + self.top_provider.context_length or context + if provider == "openrouter" and self.top_provider is not None + else context + ), + max_input_tokens=self.max_input_tokens, + max_output_tokens=( + self.max_output_tokens + or (self.max_tokens if provider == "vercel_ai_gateway" and self.type in (None, "language") else None) + or ( + self.top_provider.max_completion_tokens + if provider == "openrouter" and self.top_provider is not None + else None ) - if value is not None - } + ), + supports_function_calling=self.supports_function_calling + if self.supports_function_calling is not None + else supported_tools, + supports_parallel_function_calling=self.supports_parallel_function_calling, + supports_reasoning=self.supports_reasoning if self.supports_reasoning is not None else supported_reasoning, + reasoning_effort_levels=( + self.reasoning_effort_levels + if self.reasoning_effort_levels is not None + else ( + self.reasoning.effort_levels() + if provider == "openrouter" and self.reasoning is not None + else next((option.values for option in self.reasoning_options if option.type == "effort"), None) + if provider == "vercel_ai_gateway" + else None + ) + ), + default_reasoning_effort=self.default_reasoning_effort + or ( + self.reasoning.default_effort + if provider == "openrouter" + and self.reasoning is not None + and (self.reasoning.default_enabled is not False or self.reasoning.default_effort == "none") + else None + ), + supported_endpoints=self.supported_endpoints, + request_defaults=self.request_defaults, + supported_modalities=self.supported_modalities + if self.supported_modalities is not None + else ( + self.architecture.input_modalities + if provider == "openrouter" and self.architecture is not None + else self.modalities.input + if provider == "vercel_ai_gateway" and self.modalities is not None + else None + ), + supported_output_modalities=self.supported_output_modalities + if self.supported_output_modalities is not None + else ( + self.architecture.output_modalities + if provider == "openrouter" and self.architecture is not None + else self.modalities.output + if provider == "vercel_ai_gateway" and self.modalities is not None + else None + ), + ) + mode: Final = ( + self.mode + if self.mode in ("chat", "completion", "embedding") + else ( + "embedding" + if provider == "vercel_ai_gateway" and self.type == "embedding" + else "chat" + if provider == "vercel_ai_gateway" and self.type == "language" + else None + ) + ) + return MappingProxyType( + {**metadata.model_dump(mode="json", exclude_none=True), **({"mode": mode} if mode is not None else {})} ) class _ModelList(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - data: tuple[_ModelCard, ...] = () + data: tuple[_ModelCard, ...] + error: object | None = None async def get_openai_compatible_model_info( *, model: str, api_base: str, + provider: str = "openai", headers: Mapping[str, str], client: AsyncHTTPHandler, cache: InMemoryCache, -) -> Mapping[str, int]: - url: Final = add_path_to_api_base(api_base, "/v1/models") - cache_key: Final = ( - "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() +) -> Mapping[str, object]: + inventory: Final = await get_openai_compatible_model_inventory( + api_base=api_base, provider=provider, headers=headers, client=client, cache=cache ) - cached: Final[object] = cache.get_cache(cache_key) - if isinstance(cached, _ModelList): - return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS) + return ( + inventory.models.get(model, _EMPTY_LIMITS) if isinstance(inventory, SupplierModelInventory) else _EMPTY_LIMITS + ) + + +async def get_openai_compatible_model_inventory( + *, + api_base: str, + provider: str = "openai", + headers: Mapping[str, str], + client: AsyncHTTPHandler, + cache: InMemoryCache, + force_refresh: bool = False, +) -> SupplierModelInventory | SupplierInventoryUnavailable: + inventory_cache: Final = cast(ModelInventoryCache, cache) # cast-ok: native cache implements this protocol + inventory_client: Final = cast(ModelInventoryHTTPClient, client) # cast-ok: native HTTP handler implements get + url: Final = add_path_to_api_base(api_base, "/v1/models") + cache_key: Final = model_inventory_cache_key(provider, url, headers) + cached: Final[object] = inventory_cache.get_cache(cache_key) + if not force_refresh and isinstance(cached, (SupplierModelInventory, SupplierInventoryUnavailable)): + return cached try: - response: Final = await client.get( + response: Final = await inventory_client.get( url=url, headers=dict(headers), timeout=httpx.Timeout(5.0), @@ -84,10 +254,30 @@ async def get_openai_compatible_model_info( ) response.raise_for_status() models: Final = _ModelList.model_validate_json(response.content) - except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh - verbose_logger.debug("Could not discover upstream model token limits") - cache.set_cache(cache_key, _ModelList(), ttl=60) - return _EMPTY_LIMITS + ids: Final = tuple(card.id for card in models.data) + if ( + models.error is not None + or any(not model_id.strip() or model_id != model_id.strip() for model_id in ids) + or len(frozenset(ids)) != len(ids) + ): + unavailable: Final = SupplierInventoryUnavailable("malformed") + inventory_cache.set_cache(cache_key, unavailable, ttl=60) + return unavailable + except httpx.HTTPStatusError: + http_failure: Final = SupplierInventoryUnavailable("http") + inventory_cache.set_cache(cache_key, http_failure, ttl=60) + return http_failure + except ValidationError: + malformed: Final = SupplierInventoryUnavailable("malformed") + inventory_cache.set_cache(cache_key, malformed, ttl=60) + return malformed + except Exception: # noqa: BLE001 # optional upstream discovery must not interrupt proxy refresh + transport: Final = SupplierInventoryUnavailable("transport") + inventory_cache.set_cache(cache_key, transport, ttl=60) + return transport - cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) - return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS) + inventory: Final = SupplierModelInventory( + MappingProxyType({card.id: card.token_limits(provider) for card in models.data}), credential_scope=cache_key + ) + inventory_cache.set_cache(cache_key, inventory, ttl=MODEL_INFO_REFRESH_SECONDS) + return inventory diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 9aaf552e79a..1a6c2ce42af 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -13,9 +13,11 @@ from typing import TYPE_CHECKING, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +import litellm from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.llms.openrouter import OpenRouterErrorMessage from litellm.types.utils import ModelResponse, ModelResponseStream @@ -46,6 +48,10 @@ def openrouter_model_supports_cache_control(model: str) -> bool: class OpenrouterConfig(OpenAIGPTConfig): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + def get_supported_openai_params(self, model: str) -> list: """ Include reasoning parameters regardless of model-map capabilities. @@ -60,10 +66,6 @@ class OpenrouterConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - # OpenRouter expects "xhigh" instead of "max" for reasoning_effort. - if non_default_params.get("reasoning_effort") == "max": - non_default_params = {**non_default_params, "reasoning_effort": "xhigh"} - mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) # OpenRouter-only parameters diff --git a/litellm/llms/openrouter/model_info.py b/litellm/llms/openrouter/model_info.py new file mode 100644 index 00000000000..53879006f1c --- /dev/null +++ b/litellm/llms/openrouter/model_info.py @@ -0,0 +1,206 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final +from urllib.parse import quote + +import httpx +from pydantic import ConfigDict, ValidationError, ValidationInfo, field_validator + +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS, model_inventory_cache_key +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import ( + ModelInventoryCache, + ModelInventoryHTTPClient, + SupplierInventoryUnavailable, + SupplierModelEndpoint, + SupplierModelInventory, +) +from litellm.types.router import ModelMetadata, common_model_metadata +from litellm.utils import add_path_to_api_base + + +class _Endpoint(ModelMetadata): + tag: str + context_length: int | None = None + max_prompt_tokens: int | None = None + max_completion_tokens: int | None = None + supported_parameters: tuple[str, ...] | None = None + + @field_validator("context_length", "max_prompt_tokens", "max_completion_tokens", mode="before") + @classmethod + def known_limit(cls, value: object) -> object: + if value is None or type(value) is int and value >= 0: + return value + raise ValueError("Endpoint limits must be nonnegative integers or null") + + @field_validator("*", mode="before") + @classmethod + def strict_metadata(cls, value: object, info: ValidationInfo) -> object: + return ModelMetadata.validate_supplier_field(info.field_name, value) + + def normalized(self) -> SupplierModelEndpoint: + parameters: Final = self.supported_parameters + return SupplierModelEndpoint( + self.tag, + ModelMetadata.model_validate( + { + **self.model_dump(exclude_none=True), + "context_window": self.context_length, + "max_input_tokens": self.max_prompt_tokens, + "max_output_tokens": self.max_completion_tokens, + "supports_function_calling": self.supports_function_calling + if self.supports_function_calling is not None + else "tools" in parameters + if parameters is not None + else None, + "supports_parallel_function_calling": self.supports_parallel_function_calling + if self.supports_parallel_function_calling is not None + else True + if parameters is not None and "parallel_tool_calls" in parameters + else None, + "supports_reasoning": self.supports_reasoning + if self.supports_reasoning is not None + else bool({"reasoning", "reasoning_effort"}.intersection(parameters)) + if parameters is not None + else None, + } + ), + parameters, + ) + + +class _EndpointData(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + id: str + endpoints: tuple[_Endpoint, ...] + + +class _EndpointResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + data: _EndpointData + error: object | None = None + + +async def _fetch_endpoints( + model: str, + *, + api_base: str, + headers: Mapping[str, str], + client: ModelInventoryHTTPClient, + cache: ModelInventoryCache, + force_refresh: bool, +) -> tuple[SupplierModelEndpoint, ...] | SupplierInventoryUnavailable: + if len(model.split("/")) != 2 or any(part in ("", ".", "..") for part in model.split("/")): + return SupplierInventoryUnavailable("malformed") + url: Final = add_path_to_api_base(api_base, f"/v1/models/{quote(model, safe='/')}/endpoints") + cache_key: Final = model_inventory_cache_key("openrouter-endpoints", url, headers) + cached: Final[object] = cache.get_cache(cache_key) + if not force_refresh and isinstance(cached, (_EndpointData, SupplierInventoryUnavailable)): + return ( + tuple(endpoint.normalized() for endpoint in cached.endpoints) + if isinstance(cached, _EndpointData) + else cached + ) + try: + response: Final = await client.get( + url=url, + headers=dict(headers), + timeout=httpx.Timeout(5.0), + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + parsed: Final = _EndpointResponse.model_validate_json(response.content) + data: Final = parsed.data + if ( + parsed.error is not None + or data.id != model + or any(not endpoint.tag.strip() or endpoint.tag != endpoint.tag.strip() for endpoint in data.endpoints) + ): + raise ValueError("Endpoint inventory must identify the requested model and provider slugs") + except httpx.HTTPStatusError: + unavailable: Final = SupplierInventoryUnavailable("http") + cache.set_cache(cache_key, unavailable, ttl=60) + return unavailable + except (ValidationError, ValueError): + malformed: Final = SupplierInventoryUnavailable("malformed") + cache.set_cache(cache_key, malformed, ttl=60) + return malformed + except Exception: # noqa: BLE001 # optional metadata must not interrupt the last-valid serving generation + transport: Final = SupplierInventoryUnavailable("transport") + cache.set_cache(cache_key, transport, ttl=60) + return transport + cache.set_cache(cache_key, data, ttl=MODEL_INFO_REFRESH_SECONDS) + return tuple(endpoint.normalized() for endpoint in data.endpoints) + + +async def get_openrouter_endpoint_inventory( + *, + inventory: SupplierModelInventory, + models: tuple[str, ...], + previous: SupplierModelInventory | None, + api_base: str, + headers: Mapping[str, str], + client: ModelInventoryHTTPClient, + cache: ModelInventoryCache, + force_refresh: bool, +) -> SupplierModelInventory: + selected: Final = tuple(dict.fromkeys(model for model in models if model in inventory.models)) + retained: Final[Mapping[str, tuple[SupplierModelEndpoint, ...]]] = ( + previous.endpoints + if previous is not None + and inventory.credential_scope is not None + and previous.credential_scope == inventory.credential_scope + else MappingProxyType({}) + ) + results: Final = tuple( + [ + await _fetch_endpoints( + model, api_base=api_base, headers=headers, client=client, cache=cache, force_refresh=force_refresh + ) + for model in selected + ] + ) + return SupplierModelInventory( + inventory.models, + inventory.credential_scope, + MappingProxyType( + { + model: result if isinstance(result, tuple) else retained.get(model, ()) + for model, result in zip(selected, results) + } + ), + ) + + +def openrouter_endpoint_metadata( + model_info: Mapping[str, object], endpoints: tuple[SupplierModelEndpoint, ...], policy: OpenRouterProviderPolicy +) -> Mapping[str, object]: + eligible: Final = tuple(endpoint for endpoint in endpoints if policy.allows_endpoint(endpoint.tag)) + model: Final = ModelMetadata.model_validate(model_info) + sources: Final = tuple(_endpoint_metadata(model, endpoint) for endpoint in eligible) + return MappingProxyType(common_model_metadata(sources).model_dump(mode="json", exclude_none=True)) + + +def _endpoint_metadata(model: ModelMetadata, endpoint: SupplierModelEndpoint) -> ModelMetadata: + supports_effort: Final = ( + endpoint.supported_parameters is not None and "reasoning_effort" in endpoint.supported_parameters + ) + endpoint_levels: Final = endpoint.metadata.reasoning_effort_levels + levels: Final = ( + model.reasoning_effort_levels + if endpoint_levels is None + else endpoint_levels + if model.reasoning_effort_levels is None + else tuple(level for level in model.reasoning_effort_levels if level in endpoint_levels) + ) + return ModelMetadata.model_validate( + { + **endpoint.metadata.model_dump(exclude_none=True), + "reasoning_effort_levels": levels if supports_effort else None, + "default_reasoning_effort": (endpoint.metadata.default_reasoning_effort or model.default_reasoning_effort) + if supports_effort + else None, + } + ) diff --git a/litellm/llms/openrouter/responses/transformation.py b/litellm/llms/openrouter/responses/transformation.py index e166ccaad2d..ef29170cbe5 100644 --- a/litellm/llms/openrouter/responses/transformation.py +++ b/litellm/llms/openrouter/responses/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openrouter.chat.transformation import OpenrouterConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( ResponseCompletedEvent, @@ -135,11 +136,7 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base: str | None, litellm_params: dict, ) -> str: - api_base = ( - api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - ) - - return join_api_base(api_base, "responses") + return join_api_base(OpenrouterConfig.get_api_base(api_base), "responses") def supports_native_websocket(self) -> bool: """OpenRouter does not support native WebSocket for Responses API""" diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index cf633358a41..d371e64f339 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -31,6 +31,10 @@ class _VercelAIGatewayModelList(TypedDict): class VercelAIGatewayConfig(OpenAIGPTConfig): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" + @property def custom_llm_provider(self) -> str | None: return "vercel_ai_gateway" @@ -44,7 +48,7 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: - api_base = api_base or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" + api_base = self.get_api_base(api_base) user_api_key = api_key or get_secret_str("VERCEL_AI_GATEWAY_API_KEY") or get_secret_str("VERCEL_OIDC_TOKEN") return api_base, user_api_key diff --git a/litellm/main.py b/litellm/main.py index f30d93361e5..8c9ac672c78 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3505,7 +3505,7 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR stream: Final = ctx.stream timeout: Final = ctx.timeout - api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + api_base = litellm.OpenrouterConfig.get_api_base(api_base) api_key = ( api_key @@ -7024,9 +7024,7 @@ def embedding( headers=headers, ) elif custom_llm_provider == "openrouter": - api_base = ( - api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - ) + api_base = litellm.OpenrouterConfig.get_api_base(api_base) api_key = ( api_key diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2b01e02bb06..7bf83b6b7f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -88,7 +88,7 @@ from litellm.types.proxy.carried_budget_state import ( from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.request_tag_mappings import RequestTagMapping from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings -from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterConfig +from litellm.types.router import AllowedModelRegion, ModelMetadata, RouterErrors, UpdateRouterConfig from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( @@ -1116,7 +1116,9 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase): id: str -class ModelInfo(LiteLLMPydanticObjectBase): +class ModelInfo( # frozen-ok: native model management assigns team_public_model_name after validation + ModelMetadata, LiteLLMPydanticObjectBase +): id: str | None mode: Literal["embedding", "chat", "completion"] | None input_cost_per_token: float | None = 0.0 @@ -1133,7 +1135,7 @@ class ModelInfo(LiteLLMPydanticObjectBase): ) discoverable: bool | None = None - model_config = ConfigDict(protected_namespaces=(), extra="allow") + model_config = ConfigDict(protected_namespaces=(), extra="allow", frozen=False) @model_validator(mode="before") @classmethod @@ -2459,6 +2461,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ + model_offerings_path: str | None = Field( + None, description="Path to the authoritative external offering file, reloaded without restarting the proxy" + ) request_tag_mappings: Sequence[RequestTagMapping] | None = Field( None, description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.", diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..2304a0fc899 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -12,6 +12,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group @@ -211,12 +212,12 @@ def get_complete_model_list( - If key list is empty -> defer to team list - If team list is empty -> defer to proxy model list - If list contains wildcard -> return known provider models + If list contains wildcard -> include matching proxy names and known provider models """ - unique_models: Final = [] + unique_models: Final[list[str]] = [] - def append_unique(models): + def append_unique(models: Sequence[str]) -> None: for model in models: if model not in unique_models and model != SpecialModelNames.no_default_models.value: unique_models.append(model) @@ -244,6 +245,15 @@ def get_complete_model_list( model_access_groups_to_return.append(model) return model_access_groups_to_return + granted_proxy_models: Final = tuple( + dict.fromkeys( + model + for model in proxy_model_list + if not _check_wildcard_routing(model) + and any(is_model_allowed_by_pattern(model, grant) for grant in unique_models) + ) + ) + all_wildcard_models: Final = _get_wildcard_models( unique_models=unique_models, return_wildcard_routes=return_wildcard_routes, @@ -253,7 +263,7 @@ def get_complete_model_list( complete_model_list: Final = unique_models + all_wildcard_models - return complete_model_list + return complete_model_list + [model for model in granted_proxy_models if model not in complete_model_list] def _hydrate_litellm_credential_name( diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index e5f6bc5dda8..cbd2f6b423a 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,4 +1,6 @@ import base64 +import hashlib +import hmac import os from collections.abc import Mapping from typing import Final, Literal, cast @@ -27,6 +29,7 @@ ALGO_AES_GCM: Final = "aes-256-gcm" _ALGO_AES_GCM: Final = ALGO_AES_GCM _ALGO_XSALSA20: Final = "xsalsa20-poly1305" +_MODEL_OFFERING_PROCESS_KEY: Final = os.urandom(32) def get_salt_key() -> str | None: @@ -43,6 +46,20 @@ def get_salt_key() -> str | None: _get_salt_key: Final = get_salt_key +def model_offering_identity_fingerprint(value: bytes) -> str: + """Opaque stable connection identity, never an unkeyed hash of supplier secrets. + + Use the same server-owned key source as encryption. With a configured salt or + master key identities survive process restarts; standalone/development callers + without either key use a private process key and do not retain that stability. + """ + configured_key: Final = _get_salt_key() + key: Final = ( + configured_key.encode() if isinstance(configured_key, str) and configured_key else _MODEL_OFFERING_PROCESS_KEY + ) + return hmac.new(key, b"litellm:model-offering-identity:v1\0" + value, hashlib.sha256).hexdigest() + + def _get_encryption_algorithm() -> str: """ Resolve the configured at-rest encryption algorithm for *new writes*. diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 175432d1dc1..89f25df2b5a 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -488,7 +488,7 @@ class TeamModelNameTranslator: @staticmethod def resolve_public_name( model_id: str, - available_models: list[str], + available_models: Sequence[str], llm_router: Router | None, general_settings: Mapping[str, object], ) -> str: diff --git a/litellm/proxy/common_utils/openai_endpoint_utils.py b/litellm/proxy/common_utils/openai_endpoint_utils.py index ab44d32f368..3c3d5d105e7 100644 --- a/litellm/proxy/common_utils/openai_endpoint_utils.py +++ b/litellm/proxy/common_utils/openai_endpoint_utils.py @@ -20,21 +20,27 @@ def remove_sensitive_info_from_deployment( excluded_keys: set[str] | None = None, ) -> dict: """ - Removes sensitive information from a deployment dictionary. + Returns a public deployment copy without credentials or supplier headers. Args: deployment_dict (dict): The deployment dictionary to remove sensitive information from. excluded_keys (Optional[Set[str]]): Set of keys that should not be masked (exact match). Returns: - dict: The modified deployment dictionary with sensitive information removed. + dict: A copied deployment dictionary with sensitive information removed. """ - deployment_dict["litellm_params"].pop("api_key", None) - deployment_dict["litellm_params"].pop("client_secret", None) - deployment_dict["litellm_params"].pop("vertex_credentials", None) - deployment_dict["litellm_params"].pop("vertex_ai_credentials", None) - deployment_dict["litellm_params"].pop("aws_access_key_id", None) - deployment_dict["litellm_params"].pop("aws_secret_access_key", None) + _private_params: Final = frozenset( + ( + "api_key", + "client_secret", + "vertex_credentials", + "vertex_ai_credentials", + "aws_access_key_id", + "aws_secret_access_key", + "extra_headers", + "headers", + ) + ) # Rate-limit config fields must never be masked — they are integers, not credentials. # The field names contain "key" which matches the masker's sensitive pattern, so we @@ -45,11 +51,13 @@ def remove_sensitive_info_from_deployment( } _excluded: Final = (excluded_keys or set()) | _rate_limit_config_keys - deployment_dict["litellm_params"] = SENSITIVE_DATA_MASKER.mask_dict( - deployment_dict["litellm_params"], excluded_keys=_excluded - ) - - return deployment_dict + return { + **{key: value for key, value in deployment_dict.items() if key != "provider_policy"}, + "litellm_params": SENSITIVE_DATA_MASKER.mask_dict( + {key: value for key, value in deployment_dict["litellm_params"].items() if key not in _private_params}, + excluded_keys=_excluded, + ), + } async def get_custom_llm_provider_from_request_body(request: Request) -> str | None: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 11f616a5e18..176e3575b2a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1078,7 +1078,7 @@ def update_db_model( _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name merged_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True) - stored_model_info: Final = _dump_without_none(db_model.model_info) + stored_model_info: Final = db_model.model_info.model_dump(mode="json", exclude_none=True) echoed_pricing: Final = echoed_cost_map_pricing_fields(stored_model_info) merged_model_info: Final[dict[str, object]] = { k: v for k, v in stored_model_info.items() if k not in echoed_pricing @@ -1100,7 +1100,7 @@ def update_db_model( # update model info if updated_patch.model_info: - incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + incoming_model_info: Final = updated_patch.model_info.model_dump(mode="json", exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, _cost_map_entry(db_model, incoming_model_info), diff --git a/litellm/proxy/model_discovery.py b/litellm/proxy/model_discovery.py new file mode 100644 index 00000000000..0eb5a3bdc2a --- /dev/null +++ b/litellm/proxy/model_discovery.py @@ -0,0 +1,16 @@ +from typing import Final + +from pydantic import TypeAdapter + +from litellm.types.proxy.model_listing import ModelDiscoveryInfo +from litellm.types.router import ModelMetadata, common_model_metadata + +_ADAPTER: Final = TypeAdapter(ModelDiscoveryInfo) + + +def discover_model_metadata(sources: tuple[ModelMetadata, ...]) -> ModelDiscoveryInfo: + return _ADAPTER.validate_python( + common_model_metadata(sources).model_dump( + mode="json", exclude_none=True, exclude={"max_input_tokens", "max_output_tokens"} + ) + ) diff --git a/litellm/proxy/model_offerings.py b/litellm/proxy/model_offerings.py new file mode 100644 index 00000000000..27edc86782a --- /dev/null +++ b/litellm/proxy/model_offerings.py @@ -0,0 +1,401 @@ +import asyncio +import hashlib +import json +import os +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import yaml + +from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client +from litellm.llms.model_inventory import ( + get_inventory_model_metadata, + get_provider_model_inventory, + model_inventory_api_base, + model_inventory_identity, +) +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_CONCURRENCY +from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_identity_fingerprint +from litellm.proxy.offering_router import OfferingRouterView, OfferingServingSnapshot +from litellm.router import Router +from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory +from litellm.types.proxy.model_offerings import ModelOffering, ModelOfferingsConfig, SupplierConnection +from litellm.types.router import ModelMetadata + + +@dataclass(frozen=True, slots=True) +class _ResolvedConnection: + provider: str + api_base: str | None + api_key: str | None + headers: Mapping[str, str] + fingerprint: str + supplier_identity: str | None = None + + +@dataclass(frozen=True, slots=True) +class _ProviderState: + connection: _ResolvedConnection + inventory: SupplierModelInventory | None + credential_scope: str | None = None + + +@dataclass(frozen=True, slots=True) +class _LoadedConfig: + config: ModelOfferingsConfig + fingerprint: str + connections: Mapping[str, _ResolvedConnection] + + +@dataclass(frozen=True, slots=True) +class _InvalidConfig: + error_kind: str + + +def _resolve_secret(value: str) -> str: + if not value.startswith("os.environ/"): + return value + resolved: Final = os.environ.get(value.removeprefix("os.environ/")) + if resolved is None: + raise ValueError("Offering connection references an unset environment variable") + return resolved + + +def _resolve_connection(connection: SupplierConnection) -> _ResolvedConnection: + api_base: Final = model_inventory_api_base( + connection.provider, str(connection.api_base).rstrip("/") if connection.api_base is not None else None + ) + api_key: Final = _resolve_secret(connection.api_key.get_secret_value()) if connection.api_key is not None else None + configured_headers: Final = { + name.lower(): _resolve_secret(value.get_secret_value()) for name, value in connection.headers.items() + } + headers: Final = MappingProxyType( + {**({"authorization": f"Bearer {api_key}"} if api_key is not None else {}), **configured_headers} + ) + fingerprint: Final = model_offering_identity_fingerprint( + json.dumps((connection.provider, api_base, api_key, sorted(headers.items()))).encode() + ) + return _ResolvedConnection( + connection.provider, + api_base, + api_key, + headers, + fingerprint, + model_inventory_identity(connection.provider), + ) + + +def _load_config(path: Path) -> _LoadedConfig | _InvalidConfig: + try: + content: Final = path.read_bytes() + if len(content) > 1024 * 1024: + return _InvalidConfig("ConfigTooLarge") + config: Final = ModelOfferingsConfig.model_validate(yaml.safe_load(content)) + connections: Final = MappingProxyType( + {name: _resolve_connection(connection) for name, connection in config.providers.items()} + ) + fingerprint: Final = hashlib.sha256( + json.dumps( + ( + hashlib.sha256(content).hexdigest(), + tuple( + (connection.fingerprint, connection.supplier_identity) for connection in connections.values() + ), + ) + ).encode() + ).hexdigest() + return _LoadedConfig(config, fingerprint, connections) + except Exception as error: # noqa: BLE001 # a bad operator edit must leave the complete last-valid snapshot intact + return _InvalidConfig(type(error).__name__) + + +def _provider_after_fetch( + previous: _ProviderState, fetched: SupplierModelInventory | SupplierInventoryUnavailable +) -> _ProviderState: + if isinstance(fetched, SupplierModelInventory): + return _ProviderState(previous.connection, fetched, fetched.credential_scope) + if fetched.credential_scope is not None and previous.credential_scope != fetched.credential_scope: + return _ProviderState(previous.connection, None, fetched.credential_scope) + return previous + + +def _unavailable_reason(offering: ModelOffering, state: _ProviderState) -> str | None: + if not offering.enabled: + return "disabled in the offering configuration" + if offering.source == "manual": + return None + if state.inventory is None: + return "no successful inventory is available for the configured supplier connection" + if offering.upstream_model not in state.inventory.models: + return "the model is absent from the latest successful supplier inventory" + return None + + +def _deployment_id(offering: ModelOffering, connection: _ResolvedConnection) -> str: + identity: Final = ( + offering.model_name, + connection.fingerprint, + offering.upstream_model, + connection.supplier_identity, + ) + return ( + "offering-" + + hashlib.sha256( + json.dumps( + identity + (offering.provider_policy.model_dump(mode="json"),) + if offering.provider_policy is not None + else identity + ).encode() + ).hexdigest() + ) + + +def _endpoint_models(config: ModelOfferingsConfig | None, provider: str) -> tuple[str, ...]: + if config is None: + return () + return tuple( + dict.fromkeys( + offering.upstream_model + for offering in config.offerings + if offering.provider == provider + and offering.enabled + and offering.source == "auto" + and offering.provider_policy is not None + ) + ) + + +def _deployment(offering: ModelOffering, state: _ProviderState) -> Mapping[str, object]: + connection: Final = state.connection + reason: Final = _unavailable_reason(offering, state) + supplier_info: Final[Mapping[str, object]] = ( + get_inventory_model_metadata(state.inventory, offering.upstream_model, offering.provider_policy) + if offering.source == "auto" and state.inventory is not None + else {} + ) + catalog: Final = GetModelCostMap.loaded_model_cost_map() + bundled_info: Final = ( + catalog.get(f"{connection.provider}/{offering.upstream_model}") + or (catalog.get(offering.upstream_model, {}) if connection.provider == "openai" else {}) + if offering.source == "auto" and offering.provider_policy is None + else {} + ) + bundled_metadata: Final = ModelMetadata.model_validate(bundled_info).model_dump(mode="json", exclude_none=True) + bundled_mode: Final = bundled_info.get("mode") + metadata: Final = { + **bundled_metadata, + **({"mode": bundled_mode} if bundled_mode in ("chat", "completion", "embedding") else {}), + **supplier_info, + **offering.model_info.model_dump(mode="json", exclude_none=True), + } + deployment_id: Final = _deployment_id(offering, connection) + return MappingProxyType( + { + "model_name": offering.model_name, + "litellm_params": { + "model": f"{connection.provider}/{offering.upstream_model}", + **({"api_base": connection.api_base} if connection.api_base is not None else {}), + **({"api_key": connection.api_key} if connection.api_key is not None else {}), + **( + { + "extra_headers": { + "Authorization" if name == "authorization" else name: value + for name, value in connection.headers.items() + } + } + if connection.headers + else {} + ), + }, + "model_info": {**metadata, "id": deployment_id, "blocked": reason is not None}, + **({"provider_policy": offering.provider_policy} if offering.provider_policy is not None else {}), + } + ) + + +class ModelOfferingsManager: + def __init__( + self, + *, + path: Path, + template: Router, + client: AsyncHTTPHandler | None = None, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.path = path + self.template = template + self.client = client + self.clock = clock + self.cache = InMemoryCache(max_size_in_memory=64) + self.config: ModelOfferingsConfig | None = None + self.config_fingerprint: str | None = None + self.providers: Mapping[str, _ProviderState] = MappingProxyType({}) + self.deployments: tuple[Mapping[str, object], ...] = () + self.next_inventory_refresh = 0.0 + self.last_error: str | None = None + self.lock = asyncio.Lock() + self.inventory_semaphore = asyncio.Semaphore(MODEL_INFO_REFRESH_CONCURRENCY) + self.router = OfferingRouterView( + OfferingServingSnapshot(template, frozenset(), MappingProxyType({}), frozenset()) + ) + # Globally registered native affinity callbacks must read the same pinned + # snapshot as request routing, rather than the empty bootstrap template. + for callback in template.optional_callbacks or (): + if isinstance(callback, EncryptedContentAffinityCheck) and callback.router is template: + callback.router = self.router + + async def _fetch_provider(self, state: _ProviderState, endpoint_models: tuple[str, ...]) -> _ProviderState: + connection: Final = state.connection + try: + fetched: Final = await get_provider_model_inventory( + provider=connection.provider, + api_base=connection.api_base, + headers=connection.headers, + client=self.client or get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelInventory), + cache=self.cache, + force_refresh=True, + endpoint_models=endpoint_models, + previous=state.inventory, + ) + except Exception: # noqa: BLE001 # supplier failures cannot replace a successful inventory with absence + return state + if isinstance(fetched, SupplierInventoryUnavailable): + verbose_proxy_logger.warning( + "Supplier inventory refresh unavailable (%s); retaining the last successful inventory when connection identity is unchanged", + fetched.reason, + ) + return _provider_after_fetch(state, fetched) + + async def _bounded_fetch_provider(self, state: _ProviderState, endpoint_models: tuple[str, ...]) -> _ProviderState: + async with self.inventory_semaphore: + return await self._fetch_provider(state, endpoint_models) + + async def _refresh_providers( + self, providers: Mapping[str, _ProviderState], config: ModelOfferingsConfig + ) -> Mapping[str, _ProviderState]: + names: Final = tuple(providers) + results: Final = await asyncio.gather( + *(self._bounded_fetch_provider(providers[name], _endpoint_models(config, name)) for name in names) + ) + return MappingProxyType(dict(zip(names, results))) + + def _apply(self, loaded: _LoadedConfig, providers: Mapping[str, _ProviderState]) -> bool: + deployments: Final = tuple( + _deployment(offering, providers[offering.provider]) + for offering in loaded.config.offerings + if _unavailable_reason(offering, providers[offering.provider]) is None + ) + try: + with prevent_device_login(): + candidate: Final = ( + self.template.snapshot_with_model_list(deployments, metadata_authoritative=True) + if deployments != self.deployments + else self.router.latest_snapshot().router + ) + if len(candidate.get_model_ids()) != len(deployments): + self._config_error("DeploymentValidationFailed") + return False + except Exception as error: # noqa: BLE001 # a rejected native deployment must not partially replace serving state + self._config_error(type(error).__name__) + return False + available: Final = frozenset( + offering.model_name + for offering in loaded.config.offerings + if _unavailable_reason(offering, providers[offering.provider]) is None + ) + unavailable: Final = MappingProxyType( + { + offering.model_name: reason + for offering in loaded.config.offerings + if (reason := _unavailable_reason(offering, providers[offering.provider])) is not None + } + ) + allowed_deployments: Final = frozenset( + f"{providers[offering.provider].connection.provider}/{offering.upstream_model}" + for offering in loaded.config.offerings + if offering.model_name in available + ) + snapshot: Final = OfferingServingSnapshot( + candidate, + available, + unavailable, + allowed_deployments, + frozenset(candidate.get_model_ids()), + ) + self.config = loaded.config + self.config_fingerprint = loaded.fingerprint + self.providers = providers + self.deployments = deployments + self.last_error = None + self.router.publish_snapshot(snapshot) + return True + + def _config_error(self, kind: str) -> None: + if self.last_error != kind: + verbose_proxy_logger.error( + "External offering configuration rejected (%s); retaining the complete last-valid serving configuration", + kind, + ) + self.last_error = kind + + async def reload(self, *, initial: bool = False, force_inventory: bool = False) -> bool: + async with self.lock: + loaded: Final = await asyncio.to_thread(_load_config, self.path) + if isinstance(loaded, _InvalidConfig): + self._config_error(loaded.error_kind) + return False + now: Final = self.clock() + config_changed: Final = loaded.fingerprint != self.config_fingerprint + if not config_changed and not force_inventory and now < self.next_inventory_refresh: + return True + providers: Final = MappingProxyType( + { + name: previous + if (previous := self.providers.get(name)) is not None + and previous.connection.fingerprint == connection.fingerprint + and ( + connection.supplier_identity is None + or previous.connection.supplier_identity == connection.supplier_identity + ) + else _ProviderState(connection, None) + for name, connection in loaded.connections.items() + } + ) + connection_changed: Final = any( + name not in self.providers or self.providers[name].connection != state.connection + for name, state in providers.items() + ) + interval_changed: Final = ( + self.config is not None and loaded.config.inventory_poll_seconds != self.config.inventory_poll_seconds + ) + refresh_due: Final = ( + initial + or force_inventory + or connection_changed + or interval_changed + or any( + _endpoint_models(loaded.config, name) != _endpoint_models(self.config, name) for name in providers + ) + or now >= self.next_inventory_refresh + ) + refreshed: Final = await self._refresh_providers(providers, loaded.config) if refresh_due else providers + if not self._apply(loaded, refreshed): + return False + if refresh_due: + self.next_inventory_refresh = now + loaded.config.inventory_poll_seconds + return True + + async def run(self) -> None: + while True: + await asyncio.sleep(self.config.config_poll_seconds if self.config is not None else 5) + await self.reload() diff --git a/litellm/proxy/offering_router.py b/litellm/proxy/offering_router.py new file mode 100644 index 00000000000..680208ed74e --- /dev/null +++ b/litellm/proxy/offering_router.py @@ -0,0 +1,344 @@ +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # native Router dynamic attribute dispatch + +from pydantic import TypeAdapter +from starlette.types import ASGIApp, Receive, Scope, Send + +import litellm +from litellm.caching.caching import DualCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import ( + DeployedModelGroups, + TeamModelNameTranslator, + alias_listing_entries, + alias_target, + caller_alias_maps, +) +from litellm.router import Router +from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.types.utils import CallTypes, CallTypesLiteral + + +@dataclass(frozen=True, slots=True) +class OfferingServingSnapshot: + router: Router + available_models: frozenset[str] + unavailable_models: Mapping[str, str] + allowed_deployments: frozenset[str] + allowed_deployment_ids: frozenset[str] = frozenset() + + +_PINNED_SNAPSHOT: Final[ContextVar[tuple[int, OfferingServingSnapshot] | None]] = ContextVar( + "model_offering_snapshot", default=None +) +_MODEL_MUTATIONS: Final = frozenset(("set_model_list", "add_deployment", "upsert_deployment", "delete_deployment")) +_VIEW_MEMBERS: Final = frozenset(("publish_snapshot", "pin_snapshot", "serving_snapshot", "latest_snapshot")) +_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_POLICY_CALLS: Final = frozenset(("acompletion", "completion", "aresponses", "responses", "anthropic_messages")) +_PARAMETER_CONTROLS: Final = frozenset(("additional_drop_params", "drop_params", "allowed_openai_params")) +_ROUTING_OVERRIDES: Final = ( + frozenset( + ( + "provider", + "models", + "route", + "service_tier", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "model_list", + "provider_policy", + "deployment_id", + "specific_deployment", + "model_id", + "disable_fallbacks", + "mock_response", + "mock_tool_calls", + "mock_timeout", + "mock_provider_error", + ) + ) + | _PARAMETER_CONTROLS +) + + +def _policy_override(data: Mapping[str, object]) -> bool: + extra_body: Final = data.get("extra_body") + return bool(_ROUTING_OVERRIDES.intersection(data)) or ( + isinstance(extra_body, Mapping) + and bool((_ROUTING_OVERRIDES | frozenset(("model", "extra_body"))).intersection(extra_body)) + ) + + +class OfferingRouterView(Router): + """Request-pinned facade delegating native members to initialized Router instances. + + Router initialization belongs to the delegated instances. Reinitializing the + facade would allocate unused clients and register duplicate global callbacks. + """ + + def __init__(self, snapshot: OfferingServingSnapshot) -> None: + self._offering_snapshot = snapshot + + def __getattribute__(self, name: str) -> object: + if name.startswith("_offering_") or name in _VIEW_MEMBERS: + return cast(object, object.__getattribute__(self, name)) # cast-ok: native Router dispatch + if name in _MODEL_MUTATIONS: + return cast( # cast-ok: native Router dispatch + object, object.__getattribute__(self, "_offering_reject_mutation") + ) + snapshot: Final = self.serving_snapshot() + if name == "model_names": + return snapshot.available_models | frozenset(snapshot.unavailable_models) + return cast(object, getattr(snapshot.router, name)) # cast-ok: dynamic native Router compatibility boundary + + def __setattr__(self, name: str, value: object) -> None: + if name.startswith("_offering_"): + object.__setattr__(self, name, value) + return + setattr(self.serving_snapshot().router, name, value) + + def _offering_reject_mutation( + self, + *args: object, + **kwargs: object, # kwargs-ok: blocks all native Router mutation signatures + ) -> None: + raise litellm.BadRequestError( + message="Model definitions are owned by the external offering configuration", model="", llm_provider="" + ) + + def serving_snapshot(self) -> OfferingServingSnapshot: + pinned: Final = _PINNED_SNAPSHOT.get() + return pinned[1] if pinned is not None and pinned[0] == id(self) else self._offering_snapshot + + def latest_snapshot(self) -> OfferingServingSnapshot: + return self._offering_snapshot + + def publish_snapshot(self, snapshot: OfferingServingSnapshot) -> None: + self._offering_snapshot = snapshot + + @contextmanager + def pin_snapshot(self) -> Generator[None]: + token: Final = _PINNED_SNAPSHOT.set((id(self), self._offering_snapshot)) + try: + with prevent_device_login(): + yield + finally: + _PINNED_SNAPSHOT.reset(token) + + +class OfferingSnapshotMiddleware: + def __init__(self, app: ASGIApp, router_getter: Callable[[], Router | None]) -> None: + self.app = app + self.router_getter = router_getter + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + router: Final = self.router_getter() + if scope["type"] != "http" or not isinstance(router, OfferingRouterView): + await self.app(scope, receive, send) + return + with router.pin_snapshot(): + await self.app(scope, receive, send) + + +class OfferingAccessGuard(CustomLogger): + def __init__(self, router: OfferingRouterView, general_settings: Mapping[str, object] | None = None) -> None: + super().__init__() + self.router = router + self.general_settings: Mapping[str, object] = ( + general_settings if general_settings is not None else MappingProxyType({}) + ) + + def _offering_name(self, name: str, auth: UserAPIKeyAuth) -> str: + snapshot: Final = self.router.serving_snapshot() + auth_metadata: Final = _METADATA_ADAPTER.validate_python(auth.model_dump()) + aliases: Final = caller_alias_maps( + _METADATA_ADAPTER.validate_python(auth_metadata.get("aliases") or MappingProxyType({})), + _METADATA_ADAPTER.validate_python(auth_metadata.get("team_model_aliases") or MappingProxyType({})), + auth.team_id, + auth.team_id, + ) + deployed: Final = DeployedModelGroups(snapshot.router, team_id=auth.team_id) + entries: Final = alias_listing_entries( + tuple((name, name) for name in snapshot.available_models | frozenset(snapshot.unavailable_models)), + aliases, + deployed, + ) + caller_target: Final = alias_target(name, aliases, entries, deployed) or dict(entries).get(name, name) + target: Final = resolve_model_group_alias(snapshot.router.model_group_alias, caller_target) or caller_target + rows: Final = TypeAdapter(tuple[Mapping[str, object], ...]).validate_python( + snapshot.router.get_model_list() or () + ) + names: Final = tuple(name for name in (self._visible_model_name(row, auth) for row in rows) if name is not None) + return TeamModelNameTranslator.resolve_public_name(target, names, snapshot.router, self.general_settings) + + @staticmethod + def _visible_model_name(row: Mapping[str, object], auth: UserAPIKeyAuth) -> str | None: + name: Final = row.get("model_name") + if not isinstance(name, str): + return None + raw_model_info: Final = row.get("model_info") + model_info: Final[Mapping[str, object]] = ( + _METADATA_ADAPTER.validate_python(raw_model_info) + if isinstance(raw_model_info, Mapping) + else MappingProxyType({}) + ) + return ( + name + if auth.user_role == LitellmUserRoles.PROXY_ADMIN or model_info.get("team_id") in (None, auth.team_id) + else None + ) + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + snapshot: Final = self.router.serving_snapshot() + return tuple( + name for name in model_names if self._offering_name(name, user_api_key_dict) in snapshot.available_models + ) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: Mapping[str, object], + call_type: CallTypesLiteral, + ) -> dict[str, object] | Exception | None: # mutable-ok: CustomLogger requires a fresh dict for request replacement + if call_type in ("acreate_batch", "create_batch") or ( + call_type in ("acreate_file", "create_file") and data.get("purpose") == "batch" + ): + return litellm.BadRequestError( + message="Native provider batches are unavailable in external offering mode because batch records cannot enforce offering selection", + model="", + llm_provider="", + ) + extra_body: Final = data.get("extra_body") + if isinstance(extra_body, Mapping) and "model" in extra_body: + return litellm.BadRequestError( + message="External offering mode does not accept supplier model overrides in extra_body", + model="", + llm_provider="", + ) + if any( + data.get(name) is not None + for name in ( + "api_base", + "base_url", + "api_key", + "user_config", + "custom_llm_provider", + "extra_headers", + "headers", + "provider_specific_header", + "client", + ) + ): + return litellm.BadRequestError( + message="External offering mode does not accept caller-defined supplier connections", + model="", + llm_provider="", + ) + if any( + name in data + for name in ("specific_deployment", "deployment_id", "model_id", "model_list", "provider_policy") + ): + return litellm.BadRequestError( + message="External offering mode does not accept caller-defined deployment selection", + model="", + llm_provider="", + ) + model: Final = data.get("model") + if not isinstance(model, str): + return None + snapshot: Final = self.router.serving_snapshot() + offering: Final = self._offering_name(model, user_api_key_dict) + if offering in snapshot.available_models: + deployment: Final = snapshot.router.get_deployment_by_model_group_name(offering) + policy: Final = deployment.provider_policy if deployment is not None else None + if policy is None: + return None + if call_type not in _POLICY_CALLS or _policy_override(data): + return litellm.BadRequestError( + message="The offering provider policy does not accept caller routing overrides or this API", + model=model, + llm_provider="", + ) + caller_body: Final = _METADATA_ADAPTER.validate_python(extra_body) if extra_body is not None else {} + return { + **data, + "extra_body": {**caller_body, "provider": dict(policy.request_body())}, + "disable_fallbacks": True, + "caching": False, + "cache": {"no-cache": True, "no-store": True}, + } + reason: Final = snapshot.unavailable_models.get(offering) + if reason is not None: + return litellm.ServiceUnavailableError( + message=f"Model '{model}' is unavailable: {reason}", model=model, llm_provider="" + ) + return litellm.NotFoundError( + message=f"Model '{model}' is not an enabled gateway offering", model=model, llm_provider="" + ) + + async def async_pre_call_deployment_hook( + self, kwargs: Mapping[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: # mutable-ok: CustomLogger requires a fresh dict for request replacement + model: Final = kwargs.get("model") + if not isinstance(model, str): + return + metadata: Final = _METADATA_ADAPTER.validate_python( + kwargs.get("litellm_metadata") or kwargs.get("metadata") or {} + ) + raw_model_info: Final = metadata.get("model_info") + model_info: Final[Mapping[str, object]] = ( + _METADATA_ADAPTER.validate_python(raw_model_info) + if isinstance(raw_model_info, Mapping) + else MappingProxyType({}) + ) + deployment_id: Final = model_info.get("id") + snapshot: Final = self.router.serving_snapshot() + deployment: Final = snapshot.router.get_deployment(deployment_id) if isinstance(deployment_id, str) else None + policy: Final = deployment.provider_policy if deployment is not None else None + if policy is None and call_type not in ( + CallTypes.acompletion, + CallTypes.completion, + CallTypes.aresponses, + CallTypes.responses, + CallTypes.anthropic_messages, + CallTypes.aembedding, + CallTypes.embedding, + ): + return + effective_model: Final = ( + model if model in snapshot.allowed_deployments else f"{kwargs.get('custom_llm_provider')}/{model}" + ) + if effective_model not in snapshot.allowed_deployments or deployment_id not in snapshot.allowed_deployment_ids: + raise litellm.NotFoundError( + message="The requested deployment is not an enabled gateway offering", model=model, llm_provider="" + ) + if policy is None: + return + extra_body: Final = kwargs.get("extra_body") + if ( + call_type is None + or call_type.value not in _POLICY_CALLS + or any(kwargs.get(name) is not None for name in _PARAMETER_CONTROLS) + or any(kwargs.get(name) is not None for name in ("provider", "models", "route", "service_tier")) + or not isinstance(extra_body, Mapping) + or extra_body.get("provider") != policy.request_body() + or any(name in extra_body for name in _PARAMETER_CONTROLS) + or any(name in extra_body for name in ("model", "models", "route", "extra_body", "service_tier")) + ): + raise litellm.BadRequestError( + message="The request does not preserve the configured offering provider policy", + model=model, + llm_provider="", + ) + return {**kwargs, "caching": False, "cache": {"no-cache": True, "no-store": True}} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df948b69a67..2d8d6c1115a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -168,6 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields +from litellm.proxy.model_discovery import discover_model_metadata from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, @@ -190,6 +191,7 @@ from litellm.router_utils.auto_router_tuning_baseline import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.routing_groups import parse_routing_groups from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.router import ModelMetadata, resolve_model_metadata from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CountContextManagement, @@ -1427,11 +1429,11 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries: @asynccontextmanager async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]: + global llm_router # noqa: PLW0603 # proxy routes share the external router published during lifespan global \ prisma_client, \ master_key, \ use_background_health_checks, \ - llm_router, \ llm_model_list, \ general_settings, \ proxy_budget_rescheduler_min_time, \ @@ -1756,19 +1758,57 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState await initialize_shared_aiohttp_session() ) + import pathlib + + from litellm.proxy.model_offerings import ModelOfferingsManager + from litellm.proxy.offering_router import OfferingAccessGuard + + offerings_path: Final = TypeAdapter[str | None](str | None).validate_python( + cast(object, general_settings.get("model_offerings_path")), # cast-ok: validate untyped config + strict=True, + ) + offerings_manager: Final = ( + ModelOfferingsManager( + path=pathlib.Path(str(offerings_path)), + template=llm_router or Router(model_list=[]), + ) + if offerings_path is not None + else None + ) + if offerings_manager is not None: + if llm_model_list or (llm_router is not None and llm_router.get_model_ids()) or store_model_in_db: + raise ValueError("External offering mode requires an empty model_list and store_model_in_db disabled") + if not await offerings_manager.reload(initial=True): + raise ValueError("Initial external offering configuration is invalid") + llm_router = offerings_manager.router # rebind-ok: publish the process-wide router used by native proxy routes + offerings_guard: Final = ( + OfferingAccessGuard( + offerings_manager.router, TypeAdapter(Mapping[str, object]).validate_python(general_settings) + ) + if offerings_manager is not None + else None + ) + if offerings_guard is not None: + litellm.logging_callback_manager.add_litellm_callback(offerings_guard) + offerings_task: Final = asyncio.create_task(offerings_manager.run()) if offerings_manager is not None else None + model_info_refresh_disabled: Final = ( "disable_model_info_refresh" in general_settings and general_settings["disable_model_info_refresh"] is True ) model_info_scheduler: Final = ( - None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() + None + if model_info_refresh_disabled or offerings_manager is not None + else scheduler + if scheduler is not None + else AsyncIOScheduler() ) if model_info_scheduler is not None: + await ProxyStartupEvent.refresh_model_info() model_info_scheduler.add_job( ProxyStartupEvent.refresh_model_info, "interval", seconds=MODEL_INFO_REFRESH_SECONDS, id="refresh_model_info", - next_run_time=datetime.now(timezone.utc), max_instances=1, replace_existing=True, ) @@ -1830,6 +1870,12 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState GracefulShutdownManager.start_shutdown() await GracefulShutdownManager.wait_for_drain() finally: + if offerings_task is not None: + offerings_task.cancel() + await asyncio.gather(offerings_task, return_exceptions=True) + if offerings_guard is not None: + litellm.logging_callback_manager.remove_callback_from_all_lists(offerings_guard, require_self=True) + # Shutdown event - close shared aiohttp session if shared_aiohttp_session is not None: try: @@ -2851,6 +2897,9 @@ app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_b app.add_middleware(RedisRequestBatchMiddleware) app.add_middleware(LensInternalMiddleware) app.add_middleware(InFlightRequestsMiddleware) +from litellm.proxy.offering_router import OfferingSnapshotMiddleware + +app.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: llm_router) app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(GZipBufferedResponseMiddleware) @@ -16009,10 +16058,21 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } - # don't return the api key / vertex credentials - # don't return the llm credentials - model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"}) - return model + gateway_metadata: Final = resolve_model_metadata( + MappingProxyType({}) + if llm_router is not None and getattr(llm_router, "model_metadata_authoritative", False) is True + else MappingProxyType({**litellm_model_info, **discovered_model_info}), + model_info, + ) + published_model: Final = { + **model, + "model_info": { + **{field: value for field, value in model["model_info"].items() if field not in ModelMetadata.model_fields}, + **gateway_metadata.model_dump(mode="json", exclude_none=True), + **discover_model_metadata((gateway_metadata,)), + }, + } + return remove_sensitive_info_from_deployment(published_model, excluded_keys={"litellm_credential_name"}) async def _get_caller_byok_team_scope( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ce7e6afde24..fdd423250be 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -29,7 +29,7 @@ from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from functools import partial -from itertools import takewhile +from itertools import chain, takewhile from operator import attrgetter from types import MappingProxyType from typing import ( @@ -89,10 +89,12 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, with_litellm_call_id, ) +from litellm.proxy.model_discovery import discover_model_metadata from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse +from litellm.types.router import guaranteed_token_limit, resolve_model_metadata from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: @@ -9612,6 +9614,9 @@ async def get_available_models_for_user( return capped +_MODEL_LISTING_RESPONSE_ADAPTER: Final = TypeAdapter(ModelInfoResponse) + + def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None: try: return get_model_info(model) @@ -9694,6 +9699,15 @@ def _group_token_limit(candidate_sets: tuple[tuple[ModelInfo, ...], ...], field: return max(limits) if limits else None +def _merged_listing_info(candidates: tuple[ModelInfo, ...]) -> Mapping[str, object]: + fields: Final = tuple(dict.fromkeys(chain.from_iterable(candidate.keys() for candidate in candidates))) + return { + field: next(candidate.get(field) for candidate in candidates if candidate.get(field) is not None) + for field in fields + if any(candidate.get(field) is not None for candidate in candidates) + } + + def create_model_info_response( model_id: str, provider: str, @@ -9711,13 +9725,6 @@ def create_model_info_response( """ from litellm.proxy.auth.model_checks import get_all_fallbacks - base: Final[ModelInfoResponse] = { - "id": model_id, - "object": "model", - "created": DEFAULT_MODEL_CREATED_AT_TIME, - "owned_by": provider, - } - alias_target: Final = ( resolve_model_group_alias(llm_router.model_group_alias, model_id) if llm_router is not None else None ) @@ -9730,9 +9737,14 @@ def create_model_info_response( deployment_models: Final[tuple[str | None, ...]] = ( listing_info.cost_map_keys if listing_info is not None and listing_info.cost_map_keys else (None,) ) - listed_info: Final = _safe_get_model_info(lookup_model, get_model_info) + authoritative_metadata: Final = ( + llm_router is not None and getattr(llm_router, "model_metadata_authoritative", False) is True + ) + listed_info: Final = None if authoritative_metadata else _safe_get_model_info(lookup_model, get_model_info) candidate_sets: Final = tuple( - _resolve_listing_model_info( + () + if authoritative_metadata + else _resolve_listing_model_info( deployment_model=deployment_model, listed_model=lookup_model, listed_info=listed_info, @@ -9741,8 +9753,6 @@ def create_model_info_response( for deployment_model in deployment_models ) - max_input_tokens: int | None = _group_token_limit(candidate_sets, "max_input_tokens") - max_output_tokens: int | None = _group_token_limit(candidate_sets, "max_output_tokens") mode: Final = next( ( m @@ -9755,27 +9765,55 @@ def create_model_info_response( ), None, ) - if mode is not None: - base["mode"] = mode + configured_mode: Final = llm_router.get_configured_mode(lookup_model) if llm_router is not None else None + effective_mode: Final = configured_mode if isinstance(configured_mode, str) else mode - if listing_info is not None: - if listing_info.max_input_tokens is not None: - max_input_tokens = listing_info.max_input_tokens - if listing_info.max_output_tokens is not None: - max_output_tokens = listing_info.max_output_tokens - - if llm_router is not None: - configured_mode: Final = llm_router.get_configured_mode(lookup_model) - if isinstance(configured_mode, str): - base["mode"] = configured_mode - - if max_input_tokens is not None: - base["max_input_tokens"] = max_input_tokens - if max_output_tokens is not None: - base["max_output_tokens"] = max_output_tokens + candidates_by_model: Final = dict(zip(deployment_models, candidate_sets)) + discovery_sources: Final = ( + tuple( + resolve_model_metadata( + _merged_listing_info(candidates_by_model.get(deployment.cost_map_key, candidate_sets[0])), + deployment.model_info, + ) + for deployment in listing_info.deployments + ) + if listing_info is not None and listing_info.deployments + else tuple(resolve_model_metadata(_merged_listing_info(candidates), {}) for candidates in candidate_sets) + ) + has_context_contract: Final = any(source.context_window is not None for source in discovery_sources) + max_input_tokens: Final = ( + guaranteed_token_limit(discovery_sources, "max_input_tokens") + if has_context_contract + else listing_info.max_input_tokens + if listing_info is not None and listing_info.max_input_tokens is not None + else _group_token_limit(candidate_sets, "max_input_tokens") + ) + max_output_tokens: Final = ( + guaranteed_token_limit(discovery_sources, "max_output_tokens") + if has_context_contract + else listing_info.max_output_tokens + if listing_info is not None and listing_info.max_output_tokens is not None + else _group_token_limit(candidate_sets, "max_output_tokens") + ) + known_limits: Final = { + field: value + for field, value in (("max_input_tokens", max_input_tokens), ("max_output_tokens", max_output_tokens)) + if value is not None + } + discovered_base: Final[ModelInfoResponse] = _MODEL_LISTING_RESPONSE_ADAPTER.validate_python( + { + "id": model_id, + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": provider, + **({"mode": effective_mode} if effective_mode is not None else {}), + **known_limits, + **discover_model_metadata(discovery_sources), + } + ) if not include_metadata: - return base + return discovered_base effective_fallback_type: Final = fallback_type if fallback_type is not None else "general" @@ -9791,7 +9829,7 @@ def create_model_info_response( llm_router=llm_router, fallback_type=effective_fallback_type, ) - return {**base, "metadata": {"fallbacks": fallbacks}} + return {**discovered_base, "metadata": {"fallbacks": fallbacks}} def validate_model_access( diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..35283da6485 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -140,10 +140,8 @@ from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.model_info import ( - MODEL_INFO_DISCOVERY_PROVIDERS, MODEL_INFO_REFRESH_CONCURRENCY, MODEL_INFO_REFRESH_SECONDS, - get_openai_compatible_model_info, ) from litellm.responses.bridge_check import keep_registered_mode from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy @@ -299,6 +297,7 @@ from litellm.router_utils.kubernetes_pod_discovery import ( resolve_pods_after, resolve_pods_after_bound, ) +from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, ContinuationOutput, @@ -397,6 +396,7 @@ from litellm.types.router import ( LiteLLMParamsTypedDict, MockRouterTestingParams, ModelGroupInfo, + ModelListingDeployment, OptionalPreCallChecks, PreRoutingStrategy, RetryPolicy, @@ -451,7 +451,6 @@ from litellm.utils import ( lookup_model_info_for_registration, model_info_with_pricing_overrides, provider_rejectable_params, - resolve_api_key_secret_reference, set_live_deployment_replay, ) @@ -1428,6 +1427,7 @@ class Router: self.routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] # Initialize model_group_alias early since it's used in set_model_list + self.model_metadata_authoritative = False self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( model_group_alias or {} ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group @@ -3124,10 +3124,22 @@ class Router: ) # Check for silent model experiment # Make a local copy of litellm_params to avoid mutating the Router's state - litellm_params: Final = self._deployment_params_with_request_reasoning_override( + selected_params: Final = self._deployment_params_with_request_reasoning_override( deployment["litellm_params"], kwargs ) - silent_model: Final = litellm_params.pop("silent_model", None) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(selected_params), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + "completion", + ) + litellm_params: Final = deployment_params_with_request_defaults( + {key: value for key, value in selected_params.items() if key != "silent_model"}, + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + ) + silent_model: Final = selected_params.get("silent_model") for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model @@ -3169,6 +3181,7 @@ class Router: "caching": self.cache_responses, "client": model_client, **kwargs, + **request_defaults, } ) response: Final = litellm.completion(**input_kwargs) @@ -4493,10 +4506,22 @@ class Router: # Check for silent model experiment # Make a local copy of litellm_params to avoid mutating the Router's state - litellm_params: Final = self._deployment_params_with_request_reasoning_override( + selected_params: Final = self._deployment_params_with_request_reasoning_override( deployment["litellm_params"], kwargs ) - silent_model: Final = litellm_params.pop("silent_model", None) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(selected_params), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + "completion", + ) + litellm_params: Final = deployment_params_with_request_defaults( + {key: value for key, value in selected_params.items() if key != "silent_model"}, + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + ) + silent_model: Final = selected_params.get("silent_model") for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model @@ -4528,6 +4553,7 @@ class Router: "caching": self.cache_responses, "client": model_client, **kwargs, + **request_defaults, } ) @@ -6221,6 +6247,12 @@ class Router: return await original_generic_function(model=model, **kwargs) raise e + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + handler_name or "", + ) self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, @@ -6228,7 +6260,12 @@ class Router: splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) - data: Final = deployment["litellm_params"].copy() + data: Final = deployment_params_with_request_defaults( + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + ) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -6245,6 +6282,7 @@ class Router: **kwargs, "model": model_name, **_with_router_resolved_session_model(kwargs.get("session"), model_name), + **request_defaults, **( {"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)} if generic_function_name == "_aresponses_websocket" @@ -6992,8 +7030,19 @@ class Router: function_name="generic_api_call", splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) + request_defaults: Final = model_request_defaults( + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + handler_name, + ) - data: Final = deployment["litellm_params"].copy() + data: Final = deployment_params_with_request_defaults( + _MODEL_INFO_ADAPTER.validate_python(deployment["litellm_params"]), + request_defaults, + deployment.get("model_info", {}), + _MODEL_INFO_ADAPTER.validate_python(kwargs), + ) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -7015,6 +7064,7 @@ class Router: "custom_llm_provider": custom_llm_provider, "caching": self.cache_responses, **kwargs, + **request_defaults, } ) @@ -10446,7 +10496,7 @@ class Router: deployment = self._add_deployment(deployment=with_admin_api_key_secret_reference(deployment)) - model: Final = deployment.to_json(exclude_none=True) + model: Final = deployment.to_router_dict() self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id) return deployment @@ -10927,7 +10977,9 @@ class Router: - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values - ValueError: If supported_environments is not set in model_info or not one of the valid values """ - return model_info_is_active_for_environment(model_info=deployment.model_info) + return model_info_is_active_for_environment( + model_info=_MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True)) + ) def set_model_list(self, model_list: list): original_model_list: Final = copy.deepcopy(model_list) @@ -11021,6 +11073,32 @@ class Router: # deployments have been registered. self._finalize_adaptive_router_if_configured() + def snapshot_with_model_list( + self, model_list: Sequence[Mapping[str, object]], *, metadata_authoritative: bool = False + ) -> "Router": + snapshot: Final = copy.copy(self) + snapshot.model_metadata_authoritative = metadata_authoritative + snapshot.pattern_router = PatternMatchRouter() + snapshot.deployment_names = [] + snapshot.provider_default_deployment_ids = [] + snapshot._zero_cost_cache = {} + snapshot._discovered_model_info_cache = InMemoryCache( + max_size_in_memory=max(len(model_list), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS + ) + snapshot.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + snapshot.get_deployment_model_info + ) + snapshot.set_model_list([dict(model) for model in model_list]) + snapshot.healthy_deployments = snapshot.get_model_list() or [] + # Factory closures and the OpenAI-compatible wrapper must dispatch through + # this snapshot, rather than the template whose attributes were copied. + # Rebinding endpoints does not register callbacks or recreate clients. + snapshot.initialize_assistants_endpoint() + snapshot.initialize_router_endpoints() + snapshot.chat = litellm.Chat(params=snapshot.default_litellm_params, router_obj=snapshot) + _live_routers.add(snapshot) + return snapshot + def _add_deployment(self, deployment: Deployment) -> Deployment: import os @@ -11093,7 +11171,7 @@ class Router: if "*" in deployment.model_name: # store this as a regex pattern - all deployments matching this pattern will be sent to this deployment # Store deployment.model_name as a regex pattern - self.pattern_router.add_pattern(deployment.model_name, deployment.to_json(exclude_none=True)) + self.pattern_router.add_pattern(deployment.model_name, deployment.to_router_dict()) if deployment.model_info.id: self.provider_default_deployment_ids.append(deployment.model_info.id) @@ -11102,9 +11180,7 @@ class Router: if _team_id is not None and _team_public_model_name is not None and "*" in _team_public_model_name: if _team_id not in self.team_pattern_routers: self.team_pattern_routers[_team_id] = PatternMatchRouter() - self.team_pattern_routers[_team_id].add_pattern( - _team_public_model_name, deployment.to_json(exclude_none=True) - ) + self.team_pattern_routers[_team_id].add_pattern(_team_public_model_name, deployment.to_router_dict()) # Azure GPT-Vision Enhancements, users can pass os.environ/ data_sources: Final = deployment.litellm_params.get("dataSources", []) or [] @@ -11218,7 +11294,7 @@ class Router: served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) # add to model list - _deployment: Final = served.to_json(exclude_none=True) + _deployment: Final = served.to_router_dict() # initialize client self._add_deployment(deployment=served) @@ -11464,7 +11540,7 @@ class Router: deployment alone, which is what lets a refresh rebuild the same entries. """ model_info: Final = dict( - _MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True)) + _MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(mode="json", exclude_none=True)) ) for field in CustomPricingLiteLLMParams.model_fields: field_value = deployment.litellm_params.get(field) @@ -11808,35 +11884,7 @@ class Router: } ) ) - model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) - if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: - return - if api_base is None or "*" in model or params.get("use_clientside_credentials"): - return - api_key: Final = resolve_api_key_secret_reference( - params.api_key or dynamic_api_key, - model=model, - llm_provider=provider, - upstream=MappingProxyType({"api_base": params.api_base}), - ) - headers: Final = TypeAdapter(Mapping[str, str]).validate_python( - params.get("extra_headers") or params.get("headers") or MappingProxyType({}) - ) - auth_headers: Final = ( - MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) - ) - limits: Final = await get_openai_compatible_model_info( - model=model, - api_base=api_base, - headers=MappingProxyType( - { - **auth_headers, - **MappingProxyType({key.lower(): value for key, value in headers.items()}), - } - ), - client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), - cache=self.cache.in_memory_cache, - ) + limits: Final = await self._aget_deployment_model_metadata(params, client=client) model_id: Final = deployment.model_info.id if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment: return @@ -11847,13 +11895,22 @@ class Router: ) self._invalidate_model_group_info_cache() - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + async def _aget_deployment_model_metadata( + self, params: LiteLLM_Params, *, client: AsyncHTTPHandler | None + ) -> Mapping[str, object]: + from litellm.llms.model_inventory import get_deployment_model_metadata + + return await get_deployment_model_metadata( + params=params, + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self.cache.in_memory_cache, + ) + + def get_discovered_model_info(self, model_id: object) -> Mapping[str, object]: + if not isinstance(model_id, str): + return MappingProxyType({}) cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) - if ( - model_id is not None - and isinstance(cached, DiscoveredDeploymentModelInfo) - and cached.deployment is self.get_model_info(model_id) - ): + if isinstance(cached, DiscoveredDeploymentModelInfo) and cached.deployment is self.get_model_info(model_id): configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) return MappingProxyType({}) @@ -11881,15 +11938,21 @@ class Router: if not indices: return None - deployments: Final = tuple(self.model_list[index] for index in indices) + deployments: Final = tuple(_MODEL_INFO_ADAPTER.validate_python(self.model_list[index]) for index in indices) model_infos: Final = tuple( MappingProxyType( { - **self.get_discovered_model_info((deployment.get("model_info") or MappingProxyType({})).get("id")), + **self.get_discovered_model_info( + _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or MappingProxyType({})).get( + "id" + ) + ), **MappingProxyType( { k: v - for k, v in (deployment.get("model_info") or MappingProxyType({})).items() + for k, v in _MODEL_INFO_ADAPTER.validate_python( + deployment.get("model_info") or MappingProxyType({}) + ).items() if v is not None } ), @@ -11898,12 +11961,19 @@ class Router: for deployment in deployments ) raw_deployments: Final[tuple[object, ...]] = deployments - pricing_keys: Final = (deployment_pricing_models(deployment) for deployment in raw_deployments) + pricing_keys: Final = tuple(deployment_pricing_models(deployment) for deployment in raw_deployments) cost_map_keys: Final = tuple(dict.fromkeys(keys[0] for keys in pricing_keys if keys)) return DeploymentModelListingInfo( cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), max_output_tokens=self._widest_configured_limit(model_infos, "max_output_tokens"), + deployments=tuple( + ModelListingDeployment( + cost_map_key=keys[0] if keys else None, + model_info=model_info, + ) + for model_info, keys in zip(model_infos, pricing_keys) + ), ) @staticmethod diff --git a/litellm/router_utils/model_request_defaults.py b/litellm/router_utils/model_request_defaults.py new file mode 100644 index 00000000000..ef457c45dd4 --- /dev/null +++ b/litellm/router_utils/model_request_defaults.py @@ -0,0 +1,76 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.types.router import ModelMetadata + +_METADATA_ADAPTER: Final = TypeAdapter(ModelMetadata) +_EMPTY_DEFAULTS: Final[Mapping[str, int]] = MappingProxyType({}) +_OUTPUT_BUDGET_PARAMS: Final = frozenset(("max_tokens", "max_completion_tokens", "max_output_tokens")) + + +class _Reasoning(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + effort: str | int | None = None + + +class _RequestEffort(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + reasoning_effort: str | int | _Reasoning | None = None + reasoning: _Reasoning | None = None + + def effort(self) -> str | int | None: + direct: Final = ( + self.reasoning_effort.effort if isinstance(self.reasoning_effort, _Reasoning) else self.reasoning_effort + ) + return direct if direct is not None else self.reasoning.effort if self.reasoning is not None else None + + +def model_request_defaults( + model_info: object, + deployment_params: Mapping[str, object], + request_params: Mapping[str, object], + call_type: str, +) -> Mapping[str, int]: + if call_type not in ("completion", "acompletion", "responses", "aresponses", "anthropic_messages"): + return _EMPTY_DEFAULTS + if any(request_params.get(key) is not None for key in _OUTPUT_BUDGET_PARAMS): + return _EMPTY_DEFAULTS + metadata: Final = _METADATA_ADAPTER.validate_python(model_info) + defaults: Final = metadata.request_defaults + if defaults is None: + return _EMPTY_DEFAULTS + requested_effort: Final = _RequestEffort.model_validate(request_params).effort() + deployment_effort: Final = _RequestEffort.model_validate(deployment_params).effort() + effort: Final = ( + requested_effort if requested_effort is not None else deployment_effort or metadata.default_reasoning_effort + ) + budget: Final = defaults.output_budget(str(effort) if effort is not None else None) + if budget is None: + return _EMPTY_DEFAULTS + key: Final = "max_output_tokens" if call_type in ("responses", "aresponses") else "max_tokens" + return MappingProxyType({key: budget}) + + +def deployment_params_with_request_defaults( + deployment_params: Mapping[str, object], + defaults: Mapping[str, int], + model_info: object = None, + request_params: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + caller_budget: Final = request_params is not None and any( + request_params.get(key) is not None for key in _OUTPUT_BUDGET_PARAMS + ) + configured_policy: Final = ( + model_info is not None and _METADATA_ADAPTER.validate_python(model_info).request_defaults is not None + ) + replace_budget: Final = bool(defaults) or caller_budget and configured_policy + return MappingProxyType( + { + key: value + for key, value in deployment_params.items() + if not replace_budget or key not in _OUTPUT_BUDGET_PARAMS + } + ) diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 47f80c52845..65f72be2d45 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -34,6 +34,7 @@ class httpxSpecialProvider(str, Enum): ROICalculator = "roi_calculator" Sandbox = "sandbox" ModelCostMap = "model_cost_map" + ModelInventory = "model_inventory" PasswordBreachCheck = "password_breach_check" ASGI = "asgi" AgentHarness = "agent_harness" diff --git a/litellm/types/llms/openrouter.py b/litellm/types/llms/openrouter.py index 1558ff576b5..be7f7ec7640 100644 --- a/litellm/types/llms/openrouter.py +++ b/litellm/types/llms/openrouter.py @@ -1,7 +1,52 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from pydantic import ConfigDict, Field, field_validator from typing_extensions import TypedDict +from litellm.types.llms.base import LiteLLMBaseModel + +OPENROUTER_REASONING_EFFORTS: Final = ("max", "xhigh", "high", "medium", "low", "minimal", "none") + class OpenRouterErrorMessage(TypedDict): message: str code: int metadata: dict + + +class OpenRouterProviderPolicy(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["openrouter"] + only: Annotated[tuple[Annotated[str, Field(min_length=1, strict=True)], ...], Field(min_length=1)] + allow_fallbacks: Literal[False] = False + + @field_validator("only") + @classmethod + def exact_endpoints(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(frozenset(value)) != len(value) or any( + endpoint != endpoint.strip() or not endpoint.strip() or "*" in endpoint or "://" in endpoint + for endpoint in value + ): + raise ValueError("Provider endpoints must be unique exact slugs, not URLs or wildcards") + return value + + @field_validator("allow_fallbacks", mode="before") + @classmethod + def no_fallbacks(cls, value: object) -> Literal[False]: + if value is not False: + raise ValueError("Offering provider policies require allow_fallbacks=false") + return False + + def request_body(self) -> Mapping[str, object]: + return MappingProxyType({"only": list(self.only), "allow_fallbacks": self.allow_fallbacks}) + + def allows_endpoint(self, tag: str) -> bool: + return any( + tag == slug + or tag.startswith(slug + "/") + and not {"fast", "flex"}.intersection(tag[len(slug) + 1 :].split("/")) + for slug in self.only + ) diff --git a/litellm/types/proxy/model_inventory.py b/litellm/types/proxy/model_inventory.py new file mode 100644 index 00000000000..5511ffeecad --- /dev/null +++ b/litellm/types/proxy/model_inventory.py @@ -0,0 +1,46 @@ +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Literal, Protocol + +import httpx + +from litellm.types.router import ModelMetadata + + +@dataclass(frozen=True, slots=True) +class SupplierModelEndpoint: + tag: str + metadata: ModelMetadata + supported_parameters: tuple[str, ...] | None + + +@dataclass(frozen=True, slots=True) +class SupplierModelInventory: + models: Mapping[str, Mapping[str, object]] + credential_scope: str | None = None + endpoints: Mapping[str, tuple[SupplierModelEndpoint, ...]] = field(default_factory=lambda: MappingProxyType({})) + + +@dataclass(frozen=True, slots=True) +class SupplierInventoryUnavailable: + reason: Literal["authentication", "http", "transport", "malformed"] + credential_scope: str | None = None + + +class ModelInventoryCache(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: object, *, ttl: int) -> None: ... + + +class ModelInventoryHTTPClient(Protocol): + async def get( + self, + *, + url: str, + headers: Mapping[str, str], + timeout: httpx.Timeout, + follow_redirects: bool, + max_response_bytes: int, + ) -> httpx.Response: ... diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py index 24cfa85eee4..4e6d3be01d1 100644 --- a/litellm/types/proxy/model_listing.py +++ b/litellm/types/proxy/model_listing.py @@ -1,15 +1,32 @@ +from collections.abc import Sequence + """Response types for the model listing/retrieve endpoints (/v1/models, /models).""" from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict + +from litellm.types.utils import ModelRequestDefaultsInfo + + +class ModelDiscoveryInfo(TypedDict, total=False): + context_window: ReadOnly[int] + supports_function_calling: ReadOnly[bool] + supports_parallel_function_calling: ReadOnly[bool] + supports_reasoning: ReadOnly[bool] + reasoning_effort_levels: ReadOnly[Sequence[str]] + default_reasoning_effort: ReadOnly[str] + supported_endpoints: ReadOnly[Sequence[str]] + supported_modalities: ReadOnly[Sequence[str]] + supported_output_modalities: ReadOnly[Sequence[str]] + request_defaults: ReadOnly[ModelRequestDefaultsInfo] class ModelInfoMetadata(TypedDict): fallbacks: list[str] -class ModelInfoResponse(TypedDict): +class ModelInfoResponse(ModelDiscoveryInfo): """OpenAI-compatible model object. `mode`, `max_input_tokens`, and `max_output_tokens` are attached when the cost map or deployment config knows them; `metadata` is present only with include_metadata=true. diff --git a/litellm/types/proxy/model_offerings.py b/litellm/types/proxy/model_offerings.py new file mode 100644 index 00000000000..0d6488ae07a --- /dev/null +++ b/litellm/types/proxy/model_offerings.py @@ -0,0 +1,114 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from pydantic import ( + AnyHttpUrl, + BaseModel, + ConfigDict, + Field, + SecretStr, + ValidationInfo, + field_validator, + model_validator, +) + +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.router import ModelMetadata + + +class OfferingModelInfo(ModelMetadata): + model_config = ConfigDict(extra="forbid", frozen=True) + mode: Literal["chat", "completion", "embedding"] | None = None + + @field_validator("*", mode="before") + @classmethod + def strict_configured_metadata(cls, value: object, info: ValidationInfo) -> object: + if info.field_name is not None: + cls.validate_field_input(info.field_name, value) + return value + + +class SupplierConnection(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + provider: Literal["openai", "openrouter", "vercel_ai_gateway", "chatgpt"] + api_base: AnyHttpUrl | None = None + api_key: SecretStr | None = None + headers: Mapping[str, SecretStr] = Field(default_factory=lambda: MappingProxyType({})) + + @field_validator("headers") + @classmethod + def immutable_headers(cls, value: Mapping[str, SecretStr]) -> Mapping[str, SecretStr]: + return MappingProxyType(dict(value)) + + @model_validator(mode="after") + def generic_base_required(self) -> "SupplierConnection": + if self.provider == "openai" and self.api_base is None: + raise ValueError("Generic OpenAI connections require api_base") + if self.provider == "chatgpt" and (self.api_key is not None or self.headers): + raise ValueError( + "ChatGPT connections use the active native OAuth account rather than configured keys or headers" + ) + return self + + +class ModelOffering(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + model_name: Annotated[str, Field(min_length=1, strict=True)] + source: Literal["auto", "manual"] + provider: Annotated[str, Field(min_length=1, strict=True)] + upstream_model: Annotated[str, Field(min_length=1, strict=True)] + enabled: bool = Field(default=True, strict=True) + model_info: OfferingModelInfo = Field(default_factory=OfferingModelInfo) + provider_policy: OpenRouterProviderPolicy | None = None + + @field_validator("model_name", "provider", "upstream_model") + @classmethod + def exact_identifier(cls, value: str) -> str: + if not value.strip() or value != value.strip() or "*" in value: + raise ValueError("Offering identifiers must be nonempty exact names without wildcards") + return value + + +class ModelOfferingsConfig(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + version: Literal[1] + config_poll_seconds: int = Field(default=5, ge=1, le=300, strict=True) + inventory_poll_seconds: int = Field(default=300, ge=1, le=86400, strict=True) + providers: Mapping[str, SupplierConnection] + offerings: tuple[ModelOffering, ...] + + @field_validator("version", mode="before") + @classmethod + def supported_version(cls, value: object) -> Literal[1]: + if isinstance(value, int) and not isinstance(value, bool) and value == 1: + return 1 + raise ValueError("Offering config version must be integer 1") + + @field_validator("providers") + @classmethod + def immutable_providers(cls, value: Mapping[str, SupplierConnection]) -> Mapping[str, SupplierConnection]: + if any(not name.strip() or name != name.strip() or "*" in name for name in value): + raise ValueError("Connection names must be nonempty exact names without wildcards") + return MappingProxyType(dict(value)) + + @model_validator(mode="after") + def known_connections_and_unique_names(self) -> "ModelOfferingsConfig": + names: Final = tuple(offering.model_name for offering in self.offerings) + if len(frozenset(names)) != len(names): + raise ValueError("Each offering must have a unique model_name") + if any(offering.provider not in self.providers for offering in self.offerings): + raise ValueError("Each offering must reference a configured connection") + if any( + offering.provider_policy is not None + and ( + offering.provider_policy.provider != self.providers[offering.provider].provider + or offering.model_info.mode != "chat" + ) + for offering in self.offerings + ): + raise ValueError("Provider policies require a matching supplier and explicit chat mode") + return self diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..a4c2d9d53e2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,6 +6,7 @@ import datetime import enum from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -16,19 +17,32 @@ from typing import ( Literal, TypeAlias, TypeVar, + cast, # noqa: TID251 # runtime Pydantic field schema boundary get_type_hints, ) from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout -from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator +from pydantic import ( + BeforeValidator, + ConfigDict, + Field, + JsonValue, + StrictInt, + TypeAdapter, + ValidationError, + ValidationInfo, + field_serializer, + field_validator, + model_validator, +) from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import coerce_token_limit, normalize_drop_params from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router_weights import RouterWeights @@ -41,6 +55,7 @@ from .embedding import EmbeddingRequest from .litellm_params import RoutingStrategyName from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject +from .llms.openrouter import OpenRouterProviderPolicy from .search import SearchProvider from .utils import ( # private alias: `from .types.router import *` would rebind a public Final in litellm/__init__.py @@ -260,11 +275,211 @@ class ModelAccessWindow(LiteLLMBaseModel): return self +_STRING_VALUES_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_VALIDATED_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +class ModelRequestDefaults(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + output_token_budget: Annotated[int, Field(gt=0, strict=True)] | None = Field( + default=None, description="Default requested output budget, not a physical model limit" + ) + output_token_budget_by_reasoning_effort: Mapping[str, Annotated[int, Field(gt=0, strict=True)]] | None = Field( + default=None, description="Requested output budgets keyed by exact reasoning effort" + ) + + @field_validator("output_token_budget_by_reasoning_effort") + @classmethod + def immutable_budgets(cls, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return MappingProxyType(dict(value)) if value is not None else None + + @field_serializer("output_token_budget_by_reasoning_effort") + def serialize_budgets(self, value: Mapping[str, int] | None) -> Mapping[str, int] | None: + return dict(value) if value is not None else None + + def output_budget(self, effort: str | None) -> int | None: + return ( + (self.output_token_budget_by_reasoning_effort or {}).get(effort, self.output_token_budget) + if effort is not None + else self.output_token_budget + ) + + +class ModelMetadata(LiteLLMBaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + context_window: int | None = Field( + default=None, gt=0, description="Maximum combined input and generated output tokens" + ) + max_input_tokens: int | None = Field(default=None, gt=0, description="Independent maximum input tokens") + max_output_tokens: int | None = Field(default=None, gt=0, description="Independent maximum generated output tokens") + supports_function_calling: bool | None = Field(default=None, description="Whether this model accepts tool calls") + supports_parallel_function_calling: bool | None = Field( + default=None, description="Whether this model accepts parallel tool calls" + ) + supports_reasoning: bool | None = Field(default=None, description="Whether this model supports reasoning controls") + reasoning_effort_levels: Sequence[str] | None = Field( + default=None, description="Exact effort values this route accepts" + ) + default_reasoning_effort: str | None = Field(default=None, description="Effort used when the request omits it") + supported_endpoints: Sequence[str] | None = Field( + default=None, description="Inference endpoints exposed by this route" + ) + supported_modalities: Sequence[str] | None = Field(default=None, description="Accepted input modalities") + supported_output_modalities: Sequence[str] | None = Field(default=None, description="Supported output modalities") + request_defaults: ModelRequestDefaults | None = Field( + default=None, description="Configured request defaults, independent of supplier capability limits" + ) + + @classmethod + def validate_field_input(cls, field_name: str, value: object) -> None: + TypeAdapter[object]( + cast(object, cls.model_fields[field_name].rebuild_annotation()) # cast-ok: runtime Pydantic field schema + ).validate_python(value, strict=True) + + @classmethod + def validate_supplier_field(cls, field_name: str | None, value: object) -> object: + normalized: Final = ( + None + if field_name in ("context_window", "max_input_tokens", "max_output_tokens") + and type(value) is int + and value == 0 + else value + ) + if field_name is not None and field_name in cls.model_fields: + cls.validate_field_input(field_name, normalized) + return normalized + + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") + @classmethod + def positive_limit(cls, value: object, _info: ValidationInfo) -> int | None: + return coerce_token_limit(value, strict_positive=True) + + @field_validator( + "supports_function_calling", "supports_parallel_function_calling", "supports_reasoning", mode="before" + ) + @classmethod + def known_boolean(cls, value: object) -> bool | None: + return value if isinstance(value, bool) else None + + @field_validator( + "reasoning_effort_levels", + "supported_endpoints", + "supported_modalities", + "supported_output_modalities", + mode="before", + ) + @classmethod + def known_string_list(cls, value: object) -> Sequence[str] | None: + try: + return tuple(dict.fromkeys(_STRING_VALUES_ADAPTER.validate_python(value))) + except ValidationError: + return None + + @field_validator("default_reasoning_effort", mode="before") + @classmethod + def known_string(cls, value: object) -> str | None: + return value if isinstance(value, str) else None + + @field_validator("reasoning_effort_levels") + @classmethod + def enabled_reasoning_levels(cls, value: Sequence[str] | None, info: ValidationInfo) -> Sequence[str] | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + return None if data.get("supports_reasoning") is False else value + + @field_validator("default_reasoning_effort") + @classmethod + def supported_default(cls, value: str | None, info: ValidationInfo) -> str | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(info.data) + levels: Final = data.get("reasoning_effort_levels") + if data.get("supports_reasoning") is False: + return None + return value if levels is None or isinstance(levels, (list, tuple)) and value in levels else None + + +def guaranteed_token_limit( + sources: tuple[ModelMetadata, ...], field: Literal["context_window", "max_input_tokens", "max_output_tokens"] +) -> int | None: + values: Final = tuple( + source.context_window + if field == "context_window" + else source.max_input_tokens + if field == "max_input_tokens" + else source.max_output_tokens + for source in sources + ) + if not values or any(value is None for value in values): + return None + return min(value for value in values if value is not None) + + +def _common_metadata_list(values: tuple[Sequence[str] | None, ...]) -> Sequence[str] | None: + if not values or any(value is None for value in values): + return None + first: Final = values[0] + if first is None: + return None + return tuple(item for item in first if all(value is not None and item in value for value in values)) + + +def _common_metadata_boolean(values: tuple[bool | None, ...]) -> bool | None: + return all(values) if values and all(value is not None for value in values) else None + + +def common_model_metadata(sources: tuple[ModelMetadata, ...]) -> ModelMetadata: + efforts: Final = _common_metadata_list(tuple(source.reasoning_effort_levels for source in sources)) + defaults: Final = tuple(source.default_reasoning_effort for source in sources) + default: Final = defaults[0] if defaults and all(value == defaults[0] for value in defaults) else None + request_defaults: Final = tuple(source.request_defaults for source in sources) + return ModelMetadata( + context_window=guaranteed_token_limit(sources, "context_window"), + max_input_tokens=guaranteed_token_limit(sources, "max_input_tokens"), + max_output_tokens=guaranteed_token_limit(sources, "max_output_tokens"), + supports_function_calling=_common_metadata_boolean( + tuple(source.supports_function_calling for source in sources) + ), + supports_parallel_function_calling=_common_metadata_boolean( + tuple(source.supports_parallel_function_calling for source in sources) + ), + supports_reasoning=_common_metadata_boolean(tuple(source.supports_reasoning for source in sources)), + reasoning_effort_levels=efforts, + default_reasoning_effort=default if efforts is None or default in efforts else None, + supported_endpoints=_common_metadata_list(tuple(source.supported_endpoints for source in sources)), + supported_modalities=_common_metadata_list(tuple(source.supported_modalities for source in sources)), + supported_output_modalities=_common_metadata_list( + tuple(source.supported_output_modalities for source in sources) + ), + request_defaults=request_defaults[0] + if request_defaults and all(value == request_defaults[0] for value in request_defaults) + else None, + ) + + +def resolve_model_metadata(catalog: Mapping[str, object], configured: Mapping[str, object]) -> ModelMetadata: + values: Final = { + field: configured[field] if configured.get(field) is not None else catalog.get(field) + for field in ModelMetadata.model_fields + } + return ModelMetadata.model_validate(values) + + def _whole_float_as_int(value: object) -> object: return int(value) if isinstance(value, float) and value.is_integer() else value -class ModelInfo(MirroredPricingParams): +class ModelInfo(ModelMetadata, MirroredPricingParams): + max_input_tokens: int | None = Field(default=None, description="Independent maximum input tokens") + max_output_tokens: int | None = Field(default=None, description="Independent maximum generated output tokens") + + @field_validator("context_window", "max_input_tokens", "max_output_tokens", mode="before") + @classmethod + def positive_limit(cls, value: object, _info: ValidationInfo) -> int | None: + data: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(_info.data) + return coerce_token_limit( + value, strict_positive=_info.field_name == "context_window" or data.get("context_window") is not None + ) + id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. updated_at: datetime.datetime | None = None @@ -350,7 +565,7 @@ class ModelInfo(MirroredPricingParams): raise ValueError("ptu_effective_to must be after ptu_effective_from") return self - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=False) def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator @@ -812,6 +1027,7 @@ class DeploymentTypedDict(TypedDict, total=False): model_name: Required[str] litellm_params: Required[LiteLLMParamsTypedDict] model_info: dict + provider_policy: ReadOnly[OpenRouterProviderPolicy] SPECIAL_MODEL_INFO_PARAMS = tuple(MirroredPricingParams.model_fields) @@ -821,6 +1037,9 @@ class Deployment(LiteLLMBaseModel): model_name: str litellm_params: LiteLLM_Params model_info: ModelInfo + provider_policy: OpenRouterProviderPolicy | None = Field( + default=None, exclude=True, description="Gateway-owned supplier selection, excluded from public model metadata" + ) # admin-toggled pause flag; mirrors LiteLLM_ProxyModelTable.blocked. None means "don't set it # on create" -- the Prisma column defaults to False -- rather than "explicitly unblocked". blocked: bool | None = None @@ -858,6 +1077,27 @@ class Deployment(LiteLLMBaseModel): # if using pydantic v1 return self.dict(**kwargs) + def to_router_dict(self) -> dict[str, object]: # mutable-ok: native Router stores mutable deployment records + return { + **self.model_dump(exclude_none=True), + **({"provider_policy": self.provider_policy} if self.provider_policy is not None else {}), + } + + @model_validator(mode="after") + def apply_provider_policy(self) -> "Deployment": + policy: Final = self.provider_policy + if policy is None: + return self + provider: Final = self.litellm_params.custom_llm_provider or self.litellm_params.model.partition("/")[0] + if provider != policy.provider: + raise ValueError("The deployment provider must match its server-owned provider policy") + body: Final = _VALIDATED_FIELDS_ADAPTER.validate_python(self.litellm_params.get("extra_body") or {}) + routing: Final = dict(policy.request_body()) + if body.get("provider") is not None and body["provider"] != routing: + raise ValueError("The deployment routing body conflicts with its server-owned provider policy") + self.litellm_params = self.litellm_params.model_copy(update={"extra_body": {**body, "provider": routing}}) + return self + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -878,7 +1118,13 @@ class Deployment(LiteLLMBaseModel): @dataclass(frozen=True, slots=True) class DiscoveredDeploymentModelInfo: deployment: Mapping[str, object] - limits: Mapping[str, int] + limits: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class ModelListingDeployment: + cost_map_key: str | None + model_info: Mapping[str, object] @dataclass(frozen=True, slots=True) @@ -897,6 +1143,7 @@ class DeploymentModelListingInfo: cost_map_keys: tuple[str, ...] max_input_tokens: int | None max_output_tokens: int | None + deployments: tuple[ModelListingDeployment, ...] = () class RouterErrors(enum.Enum): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..4e820d4983e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -225,7 +225,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_xhigh_reasoning_effort: bool | None supports_max_reasoning_effort: bool | None reasoning_effort_levels: ReadOnly[Sequence[str] | None] - default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] + default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None] supports_output_config: bool | None supports_image_size: bool | None supports_anthropic_thinking_payload: ReadOnly[bool | None] @@ -302,7 +302,16 @@ class OffPeakPricing(TypedDict, total=False): cache_creation_input_token_cost: ReadOnly[float] +class ModelRequestDefaultsInfo(TypedDict, total=False): + output_token_budget: ReadOnly[int] + output_token_budget_by_reasoning_effort: ReadOnly[Mapping[str, int]] + + class ModelInfoBase(ProviderSpecificModelInfo, total=False): + context_window: ReadOnly[int | None] + supported_modalities: ReadOnly[Sequence[str] | None] + supported_output_modalities: ReadOnly[Sequence[str] | None] + request_defaults: ReadOnly[ModelRequestDefaultsInfo | None] key: Required[str] # the key in litellm.model_cost which is returned max_tokens: Required[int | None] diff --git a/litellm/utils.py b/litellm/utils.py index e2820f9c682..5044cb49a34 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6740,6 +6740,10 @@ def get_model_info_helper( returned_model_info: Final = ModelInfoBase( key=key, + context_window=_model_info.get("context_window", None), + supported_modalities=_model_info.get("supported_modalities", None), + supported_output_modalities=_model_info.get("supported_output_modalities", None), + request_defaults=_model_info.get("request_defaults", None), max_tokens=_model_info.get("max_tokens", None), max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 890fa0fd773..ef1b868ee32 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "fastuuid>=0.14.0,<1.0", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<3.20.4", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dateutil>=2.8.2,<3.0", diff --git a/pyproject.toml b/pyproject.toml index 817c50df90b..95d9a277229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<3.20.4", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dotenv>=1.0.0,<2.0", diff --git a/tests/unit/integrations/otel/test_otel_v2_presets.py b/tests/unit/integrations/otel/test_otel_v2_presets.py index 570c27a24e1..877ac071e23 100644 --- a/tests/unit/integrations/otel/test_otel_v2_presets.py +++ b/tests/unit/integrations/otel/test_otel_v2_presets.py @@ -93,13 +93,14 @@ def _agentops_exporter(): def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch(): pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - with respx.mock() as auth, requests_mock.Mocker() as collector: + with respx.mock() as auth: token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"}) - collector.post(_AGENTOPS_ENDPOINT, status_code=200) exporter = _agentops_exporter() assert token_route.call_count == 0, "building the exporter must not fetch the JWT" - results = [exporter.export(()), exporter.export(())] + with requests_mock.Mocker(session=exporter._agentops_session) as collector: + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + results = [exporter.export(()), exporter.export(())] assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS] assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}] @@ -111,11 +112,13 @@ def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_ def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails(): pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - with respx.mock() as auth, requests_mock.Mocker() as collector: + with respx.mock() as auth: auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down")) - collector.post(_AGENTOPS_ENDPOINT, status_code=200) + exporter = _agentops_exporter() + with requests_mock.Mocker(session=exporter._agentops_session) as collector: + collector.post(_AGENTOPS_ENDPOINT, status_code=200) - result = _agentops_exporter().export(()) + result = exporter.export(()) assert result is SpanExportResult.SUCCESS assert [request.headers.get("Authorization") for request in collector.request_history] == [None] diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 658aa009683..711aa84b2e2 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -4845,7 +4845,7 @@ def test_completion_cost_on_translated_anthropic_response_includes_web_search(): "model, provider, carried", [ ("databricks/databricks-claude-opus-4-7", "databricks", "max"), - ("openrouter/anthropic/claude-opus-4.7", "openrouter", "xhigh"), + ("openrouter/anthropic/claude-opus-4.7", "openrouter", "max"), ], ) def test_a_summary_bearing_adaptive_request_still_delivers_its_tier(model, provider, carried): @@ -4855,8 +4855,8 @@ def test_a_summary_bearing_adaptive_request_still_delivers_its_tier(model, provi exactly the path this translator exists to serve. Each case names the exact tier that provider ends up sending, not merely that something arrived: - bedrock and databricks rebuild `output_config`, and openrouter applies its own max to xhigh - remap, so asserting presence alone would pass on a mapping that silently changed the tier.""" + bedrock and databricks rebuild `output_config`, while OpenRouter keeps its gateway effort + unchanged, so asserting presence alone would pass on a mapping that silently changed the tier.""" from litellm.types.llms.anthropic import AnthropicMessagesRequest from litellm.utils import get_optional_params diff --git a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py index e9450d025a6..35f891c8a97 100644 --- a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py @@ -23,6 +23,14 @@ from litellm.router_utils.reasoning_effort_capability import ( from litellm.utils import get_model_info +@pytest.mark.parametrize("effort", ("max", "xhigh", "high", "minimal", "none", "future-effort")) +@pytest.mark.parametrize(("model", "provider"), (("org/off-map", "openrouter"), ("openrouter/org/off-map", None))) +def test_openrouter_gateway_efforts_are_never_degraded_from_backend_flags( + effort: str, model: str, provider: str | None +) -> None: + assert normalize_reasoning_effort_value(effort, model, provider) == effort + + def _load_model_registry() -> Dict[str, Any]: """Load the root model_prices_and_context_window.json.""" json_path = os.path.join( diff --git a/tests/unit/llms/chatgpt/chat/test_transformation.py b/tests/unit/llms/chatgpt/chat/test_transformation.py new file mode 100644 index 00000000000..bfa99cc64ce --- /dev/null +++ b/tests/unit/llms/chatgpt/chat/test_transformation.py @@ -0,0 +1,26 @@ +import json +from pathlib import Path +from typing import Final + +import pytest + +from litellm.llms.chatgpt.authenticator import prevent_device_login +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig + + +def test_configured_native_endpoint_is_used_with_saved_authorization( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_API_BASE", "https://configured.test/native") + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text(json.dumps({"access_token": "local-fixture-token", "expires_at": 9999999999})) + config: Final = ChatGPTConfig() + config.authenticator.auth_file = str(auth_file) + with prevent_device_login(): + result: Final = config._get_openai_compatible_provider_info( + model="local-fixture-model", + api_base="https://untrusted.test/capture", + api_key=None, + custom_llm_provider="chatgpt", + ) + assert result == ("https://configured.test/native", "local-fixture-token", "chatgpt") diff --git a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py index 43a4c26b6d2..2357ba6fcd0 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py @@ -161,3 +161,13 @@ class TestChatGPTAuthenticator: assert account_id == "acct-123" mock_write.assert_called_once() assert mock_write.call_args[0][0]["account_id"] == "acct-123" + + +def test_request_scope_never_starts_device_login(tmp_path): + from litellm.llms.chatgpt.authenticator import prevent_device_login + from litellm.llms.chatgpt.common_utils import GetAccessTokenError + + authenticator = Authenticator() + authenticator.auth_file = str(tmp_path / "missing-auth.json") + with prevent_device_login(), pytest.raises(GetAccessTokenError, match="cannot start device login"): + authenticator.get_access_token() diff --git a/tests/unit/llms/chatgpt/test_model_info.py b/tests/unit/llms/chatgpt/test_model_info.py new file mode 100644 index 00000000000..a5177ba5350 --- /dev/null +++ b/tests/unit/llms/chatgpt/test_model_info.py @@ -0,0 +1,305 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.chatgpt.authenticator import Authenticator +from litellm.llms.chatgpt.common_utils import GetAccessTokenError +from litellm.llms.chatgpt.model_info import ( + _ChatGPTModel, + chatgpt_model_inventory_identity, + get_chatgpt_model_inventory, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.model_inventory import SupplierInventoryUnavailable, SupplierModelInventory + + +@pytest.mark.parametrize( + "invalid", + ( + {"error": "denied", "models": []}, + {"models": [{"slug": " duplicate "}]}, + {"models": [{"slug": ""}]}, + {"models": [{"slug": "same"}, {"slug": "same"}]}, + {"models": [{"slug": "fixture", "context_window": "invalid"}]}, + {"models": [{"slug": "fixture", "supports_parallel_tool_calls": "false"}]}, + ), +) +async def test_malformed_refresh_never_replaces_a_successful_native_inventory( + invalid: dict[str, object], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + (tmp_path / "auth.json").write_text( + json.dumps({"access_token": "fixture-token", "account_id": "fixture-account", "expires_at": 10**30}) + ) + responses: Final = iter(({"models": [{"slug": "fixture", "context_window": 0}]}, invalid, {"models": []})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json=next(responses))) + ) as client: + handler.client = client + cache: Final = InMemoryCache() + first: Final = await get_chatgpt_model_inventory(client=handler, cache=cache) + assert isinstance(first, SupplierModelInventory) + assert set(first.models) == {"fixture"} + assert "codex_context_window" not in first.models["fixture"] + failed: Final = await get_chatgpt_model_inventory(client=handler, cache=cache, force_refresh=True) + assert isinstance(failed, SupplierInventoryUnavailable) + assert failed.reason == "malformed" + assert failed.credential_scope == first.credential_scope + assert await get_chatgpt_model_inventory(client=handler, cache=cache) is first + empty: Final = await get_chatgpt_model_inventory(client=handler, cache=cache, force_refresh=True) + assert isinstance(empty, SupplierModelInventory) + assert not empty.models + assert await get_chatgpt_model_inventory(client=handler, cache=cache) is empty + + +async def test_native_transport_failure_is_unavailable_and_never_an_empty_inventory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "fixture-token", "expires_at": 10**30})) + + def timeout(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("fixture timeout", request=request) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(timeout)) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory(client=handler, cache=InMemoryCache()) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "transport" + assert result.credential_scope is None + + +@pytest.mark.parametrize("content", (b"not valid json", b"\xff")) +def test_unreadable_native_account_does_not_claim_a_credential_change( + content: bytes, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_bytes(content) + assert chatgpt_model_inventory_identity() is None + + +def test_native_explicit_capabilities_override_native_defaults_without_conflating_context() -> None: + metadata: Final = _ChatGPTModel.model_validate( + { + "slug": "fixture", + "context_window": 900, + "max_context_window": 0, + "max_input_tokens": 600, + "max_output_tokens": 300, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_parallel_tool_calls": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "ultra"], + "default_reasoning_effort": "ultra", + "supported_modalities": ["text"], + "input_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + } + ).metadata() + assert metadata["default_reasoning_effort"] == "high" + assert metadata["reasoning_effort_levels"] == ("low", "high") + assert metadata["supports_reasoning"] is True + assert metadata["supports_parallel_function_calling"] is False + assert metadata["supported_modalities"] == ("text",) + assert metadata["supported_output_modalities"] == ("text",) + assert metadata["max_input_tokens"] == 600 + assert metadata["max_output_tokens"] == 300 + assert metadata["codex_context_window"] == 900 + assert "context_window" not in metadata + assert "codex_max_context_window" not in metadata + + +@pytest.mark.parametrize("levels", (None, [])) +def test_unresolvable_ui_default_never_advertises_an_unsupported_api_effort( + levels: list[dict[str, str]] | None, +) -> None: + metadata: Final = _ChatGPTModel.model_validate( + {"slug": "fixture", "default_reasoning_level": "ultra", "supported_reasoning_levels": levels} + ).metadata() + assert "default_reasoning_effort" not in metadata + + +async def test_account_catalog_preserves_client_policy_separately_from_model_limits( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text( + json.dumps({"access_token": "local-token", "account_id": "local-account", "expires_at": 10**30}) + ) + card: Final = { + "slug": "fixture-model", + "context_window": 600, + "max_context_window": 1200, + "supported_reasoning_levels": [{"effort": "high"}, {"effort": "future-effort"}], + "default_reasoning_level": "future-effort", + "input_modalities": ["text", "image"], + "supports_parallel_tool_calls": False, + } + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/backend-api/codex/models" + assert request.url.params.get("client_version") + assert request.headers["authorization"] == "Bearer local-token" + assert request.headers["chatgpt-account-id"] == "local-account" + assert request.headers["accept"] == "application/json" + return httpx.Response(200, json={"models": [card]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + cache: Final = InMemoryCache() + result: Final = await get_chatgpt_model_inventory( + api_base="https://account.test/backend-api/codex/", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + assert isinstance(result, SupplierModelInventory) + assert result.models["fixture-model"] == { + "codex_context_window": card["context_window"], + "codex_max_context_window": card["max_context_window"], + "supports_parallel_function_calling": card["supports_parallel_tool_calls"], + "supports_reasoning": True, + "reasoning_effort_levels": tuple(level["effort"] for level in card["supported_reasoning_levels"]), + "default_reasoning_effort": card["default_reasoning_level"], + "supported_endpoints": ("/responses",), + "supported_modalities": tuple(card["input_modalities"]), + } + cached: Final = await get_chatgpt_model_inventory( + api_base="https://account.test/backend-api/codex/", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + assert cached is result + assert cached.models.get("not-selected", {}) == {} + + +async def test_switching_accounts_invalidates_catalog_without_exposing_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + auth_file: Final = tmp_path / "auth.json" + cache: Final = InMemoryCache() + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def respond(request: httpx.Request) -> httpx.Response: + capacity: Final = 1024 if request.headers["chatgpt-account-id"] == "first" else 2048 + return httpx.Response(200, json={"models": [{"slug": "fixture", "context_window": capacity}]}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + for account, capacity in (("first", 1024), ("second", 2048)): + auth_file.write_text( + json.dumps({"access_token": "local-token", "account_id": account, "expires_at": 10**30}) + ) + result: Final = await get_chatgpt_model_inventory( + api_base="https://account.test/backend-api/codex", + authenticator=Authenticator(), + client=handler, + cache=cache, + ) + assert isinstance(result, SupplierModelInventory) + assert result.models["fixture"]["codex_context_window"] == capacity + assert "local-token" not in json.dumps(dict(result.models["fixture"])) + + +async def test_discovery_without_authorization_never_starts_device_login( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + with pytest.raises(GetAccessTokenError, match="model discovery cannot start device login"): + Authenticator().get_access_token(allow_device_login=False) + + def reject(request: httpx.Request) -> httpx.Response: + pytest.fail("Discovery without credentials must not send any request") + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory( + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), + ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "authentication" + assert not (tmp_path / "auth.json").exists() + + +@pytest.mark.parametrize(("status", "body"), ((503, {}), (200, {"models": "invalid"}))) +async def test_invalid_or_unavailable_catalog_preserves_fallback( + status: int, body: dict[str, object], tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_API_BASE", "https://account.test/backend-api/codex") + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "local", "expires_at": 10**30})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(status, json=body))) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory( + authenticator=Authenticator(), + client=handler, + cache=InMemoryCache(), + ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == ("http" if status == 503 else "malformed") + + +@pytest.mark.parametrize( + ("default", "multi_agent", "expected"), + (("ultra", "high", "high"), ("ultra", None, "max"), ("persistent", None, "disabled")), +) +def test_codex_ui_efforts_are_not_advertised_as_api_efforts( + default: str, multi_agent: str | None, expected: str +) -> None: + metadata: Final = _ChatGPTModel.model_validate( + { + "slug": "fixture", + "supported_reasoning_levels": [{"effort": effort} for effort in ("high", "max", "ultra", "persistent")], + "default_reasoning_level": default, + "multi_agent_reasoning_effort": multi_agent, + } + ).metadata() + assert metadata["reasoning_effort_levels"] == ("high", "max", "disabled") + assert metadata["default_reasoning_effort"] == expected + + +async def test_caller_api_base_cannot_exfiltrate_saved_server_oauth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CHATGPT_API_BASE", "https://trusted.test/backend-api/codex") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "auth.json").write_text(json.dumps({"access_token": "fixture-server-token", "expires_at": 10**30})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def reject(request: httpx.Request) -> httpx.Response: + raise AssertionError("Saved OAuth must not reach a caller-controlled destination") + + async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: + handler.client = client + result: Final = await get_chatgpt_model_inventory( + client=handler, cache=InMemoryCache(), api_base="https://attacker.test/capture" + ) + assert isinstance(result, SupplierInventoryUnavailable) + assert result.reason == "authentication" diff --git a/tests/unit/llms/openai_like/test_model_info.py b/tests/unit/llms/openai_like/test_model_info.py index 15a7d9e7fc6..840e26cb409 100644 --- a/tests/unit/llms/openai_like/test_model_info.py +++ b/tests/unit/llms/openai_like/test_model_info.py @@ -16,14 +16,14 @@ from litellm.llms.openai_like.model_info import ( @pytest.mark.parametrize( ("card", "expected"), ( - ({"max_model_len": 8192}, {"max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192}), + ({"max_model_len": 8192}, {"context_window": 8192}), ( {"context_length": 4096, "max_output_tokens": 1024}, - {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 1024}, + {"context_window": 4096, "max_output_tokens": 1024}, ), ( {"max_model_len": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192}, - {"max_tokens": 4096, "max_input_tokens": 2048, "max_output_tokens": 4096}, + {"context_window": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192}, ), ({"max_input_tokens": 2048}, {"max_input_tokens": 2048}), ({"max_output_tokens": 1024}, {"max_output_tokens": 1024}), @@ -32,7 +32,9 @@ from litellm.llms.openai_like.model_info import ( ({}, {}), ), ) -async def test_discovers_only_valid_advertised_limits(card: Mapping[str, object], expected: Mapping[str, int]) -> None: +async def test_discovers_only_valid_advertised_limits( + card: Mapping[str, object], expected: Mapping[str, object] +) -> None: def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/tenant/v1/models" assert request.headers["authorization"] == "Bearer local-key" @@ -81,18 +83,18 @@ async def test_cache_is_scoped_to_endpoint_and_authentication_and_expires() -> N handler.client = client cache: Final = InMemoryCache(clock=clock) - async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, int]: + async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, object]: return await get_openai_compatible_model_info( model=model, api_base=f"https://{host}", headers={"Authorization": key}, client=handler, cache=cache ) - assert (await lookup())["max_input_tokens"] == 1024 - assert (await lookup("second"))["max_input_tokens"] == 2048 + assert (await lookup())["context_window"] == 1024 + assert (await lookup("second"))["context_window"] == 2048 assert responder.call_count == 1 - assert (await lookup(key="two"))["max_input_tokens"] == 4096 - assert (await lookup(host="two.test"))["max_input_tokens"] == 8192 + assert (await lookup(key="two"))["context_window"] == 4096 + assert (await lookup(host="two.test"))["context_window"] == 8192 clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1 - assert (await lookup())["max_input_tokens"] == 16384 + assert (await lookup())["context_window"] == 16384 assert responder.call_count == 4 @@ -124,3 +126,102 @@ async def test_unavailable_metadata_is_best_effort_and_negative_cached( == {} ) assert responder.call_count == 1 + + +@pytest.mark.parametrize( + ("provider", "card", "expected"), + ( + ( + "vercel_ai_gateway", + { + "context_window": 1200, + "max_tokens": 200, + "modalities": {"input": ["text", "image", "pdf"], "output": ["text"]}, + "supported_parameters": ["tools", "reasoning"], + "reasoning_options": [{"type": "effort", "values": ["low", "xhigh", "max"]}], + }, + { + "context_window": 1200, + "max_output_tokens": 200, + "supported_modalities": ["text", "image", "pdf"], + "supported_output_modalities": ["text"], + "supports_function_calling": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "xhigh", "max"], + }, + ), + ( + "openrouter", + { + "context_length": 1200, + "top_provider": {"context_length": 1100, "max_completion_tokens": 200}, + "architecture": {"input_modalities": ["text", "file"], "output_modalities": ["text"]}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["low", "max"], "default_effort": "low"}, + }, + { + "context_window": 1100, + "max_output_tokens": 200, + "supported_modalities": ["text", "file"], + "supported_output_modalities": ["text"], + "supports_function_calling": True, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + }, + ), + ( + "openai", + {"max_tokens": 200, "supported_parameters": []}, + {"supports_function_calling": False, "supports_reasoning": False}, + ), + ), +) +async def test_normalizes_supplier_metadata_without_inventing_input_limit(provider, card, expected): + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "selected", **card}]})) + ) as client: + handler.client = client + result: Final = await get_openai_compatible_model_info( + model="selected", + provider=provider, + api_base="https://supplier.test/v1", + headers={}, + client=handler, + cache=InMemoryCache(), + ) + assert result == expected + + +async def test_full_inventory_preserves_model_kind_without_fabricating_embedding_output_cap() -> None: + from litellm.llms.openai_like.model_info import get_openai_compatible_model_inventory + from litellm.types.proxy.model_inventory import SupplierModelInventory + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={ + "data": [ + {"id": "embedding", "type": "embedding", "context_window": 4096, "max_tokens": 4096}, + {"id": "language", "type": "language", "max_tokens": 1024}, + ] + }, + ) + ) + ) as client: + handler.client = client + result: Final = await get_openai_compatible_model_inventory( + api_base="https://supplier.test/v1", + provider="vercel_ai_gateway", + headers={}, + client=handler, + cache=InMemoryCache(), + ) + assert isinstance(result, SupplierModelInventory) + assert result.models["embedding"] == {"mode": "embedding", "context_window": 4096} + assert result.models["language"] == {"mode": "chat", "max_output_tokens": 1024} diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 59355dfabcb..1d3043600a5 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -562,9 +562,9 @@ def test_openrouter_non_reasoning_models_support_reasoning_params(): assert "thinking" in supported_params -def test_openrouter_reasoning_effort_max_maps_to_xhigh(): +def test_openrouter_reasoning_effort_max_is_preserved(): """ - OpenRouter expects 'xhigh' instead of 'max' for reasoning_effort. + Preserve the exact gateway effort, which need not be the backend's alias. """ config = OpenrouterConfig() @@ -575,7 +575,7 @@ def test_openrouter_reasoning_effort_max_maps_to_xhigh(): drop_params=False, ) - assert result["reasoning_effort"] == "xhigh" + assert result["reasoning_effort"] == "max" def test_openrouter_reasoning_effort_max_does_not_mutate_caller_dict(): diff --git a/tests/unit/llms/openrouter/test_model_info.py b/tests/unit/llms/openrouter/test_model_info.py new file mode 100644 index 00000000000..c829f2dcfe7 --- /dev/null +++ b/tests/unit/llms/openrouter/test_model_info.py @@ -0,0 +1,364 @@ +from collections.abc import Mapping +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.model_inventory import get_inventory_model_metadata, get_provider_model_inventory +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS +from litellm.types.llms.openrouter import OPENROUTER_REASONING_EFFORTS, OpenRouterProviderPolicy +from litellm.types.proxy.model_inventory import SupplierModelInventory + + +def _endpoint(tag: str = "supplier/fp8", **fields: object) -> Mapping[str, object]: + return { + "tag": tag, + "context_length": 1200, + "max_prompt_tokens": None, + "max_completion_tokens": 300, + "supported_parameters": ["tools", "reasoning", "reasoning_effort"], + **fields, + } + + +async def _inventory( + handler: AsyncHTTPHandler, + cache: InMemoryCache, + *, + only: tuple[str, ...] = ("supplier/fp8",), + key: str = "fixture-key", + base: str = "https://supplier.test/custom/v1", + refresh: bool = False, + previous: SupplierModelInventory | None = None, + model: str = "org/model", +) -> tuple[SupplierModelInventory, Mapping[str, object]]: + inventory: Final = await get_provider_model_inventory( + provider="openrouter", + api_base=base, + headers={"authorization": key}, + client=handler, + cache=cache, + endpoint_models=(model, model), + force_refresh=refresh, + previous=previous, + ) + assert isinstance(inventory, SupplierModelInventory) + return inventory, get_inventory_model_metadata( + inventory, model, OpenRouterProviderPolicy(provider="openrouter", only=only) + ) + + +def _catalog(reasoning: Mapping[str, object] | None = None) -> Mapping[str, object]: + return { + "data": [ + { + "id": "org/model", + "context_length": 9000, + "top_provider": {"context_length": 8000, "max_completion_tokens": 7000}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["max", "high", "low"], "default_effort": "high"} + if reasoning is None + else reasoning, + }, + {"id": "org/second"}, + ] + } + + +@pytest.mark.parametrize( + ("endpoints", "only", "context", "output", "input_limit", "efforts", "default", "reasoning"), + ( + ( + [_endpoint(), _endpoint("other", context_length=9000)], + ("supplier/fp8",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("supplier/fp8/eu", context_length=900, max_completion_tokens=200)], + ("supplier",), + 900, + 200, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(max_prompt_tokens=500), _endpoint("second", max_prompt_tokens=400)], + ("supplier", "second"), + 1200, + 300, + 400, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("second", supported_parameters=["reasoning"])], + ("supplier", "second"), + 1200, + 300, + None, + None, + None, + True, + ), + ([_endpoint(supported_parameters=[])], ("supplier",), 1200, 300, None, None, None, False), + ( + [_endpoint(supported_parameters=None, context_length=None, max_completion_tokens=None)], + ("supplier",), + None, + None, + None, + None, + None, + None, + ), + ([_endpoint()], ("supplier/fp4",), None, None, None, None, None, None), + ([], ("supplier",), None, None, None, None, None, None), + ( + [_endpoint(), _endpoint("supplier/fast", context_length=10)], + ("supplier",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(), _endpoint("supplier/fast", context_length=10)], + ("supplier/fast",), + 10, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [_endpoint(status=-1, uptime_last_30m=0)], + ("supplier",), + 1200, + 300, + None, + ["max", "high", "low"], + "high", + True, + ), + ( + [ + _endpoint(reasoning_effort_levels=["high", "low"]), + _endpoint("second", reasoning_effort_levels=["max", "high"], default_reasoning_effort="max"), + ], + ("supplier", "second"), + 1200, + 300, + None, + ["high"], + None, + True, + ), + ), +) +async def test_pinned_metadata_guarantees_only_eligible_endpoint_contracts( + endpoints: list[Mapping[str, object]], + only: tuple[str, ...], + context: int | None, + output: int | None, + input_limit: int | None, + efforts: list[str] | None, + default: str | None, + reasoning: bool | None, +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] == "fixture-key" + if request.url.path.endswith("/endpoints"): + assert request.url.path == "/custom/v1/models/org/model/endpoints" + return httpx.Response(200, json={"data": {"id": "org/model", "endpoints": endpoints}}) + assert request.url.path == "/custom/v1/models" + return httpx.Response(200, json=_catalog()) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + inventory, metadata = await _inventory(handler, InMemoryCache(), only=only) + assert metadata.get("context_window") == context + assert metadata.get("max_output_tokens") == output + assert metadata.get("max_input_tokens") == input_limit + assert metadata.get("reasoning_effort_levels") == efforts + assert metadata.get("default_reasoning_effort") == default + assert metadata.get("supports_reasoning") == reasoning + assert all(value is not None for value in metadata.values()) + assert get_inventory_model_metadata(inventory, "org/model")["context_window"] == 8000 + assert get_inventory_model_metadata(inventory, "org/model")["max_output_tokens"] == 7000 + + +@pytest.mark.parametrize( + ("reasoning", "expected"), + ( + ({}, {}), + ( + {"supported_efforts": None, "mandatory": True, "default_effort": "none"}, + {"reasoning_effort_levels": [level for level in OPENROUTER_REASONING_EFFORTS if level != "none"]}, + ), + ( + {"supported_efforts": ["high", "none"], "default_effort": "none", "default_enabled": False}, + {"reasoning_effort_levels": ["high", "none"], "default_reasoning_effort": "none"}, + ), + ({"supported_efforts": ["future-effort", "high"]}, {"reasoning_effort_levels": ["future-effort", "high"]}), + ( + {"supported_efforts": None, "default_effort": "high"}, + {"reasoning_effort_levels": list(OPENROUTER_REASONING_EFFORTS), "default_reasoning_effort": "high"}, + ), + ({"supported_efforts": [], "default_effort": "high"}, {"reasoning_effort_levels": []}), + ( + {"supported_efforts": ["xhigh", "high"], "default_effort": "xhigh", "default_enabled": False}, + {"reasoning_effort_levels": ["xhigh", "high"]}, + ), + ({"supported_efforts": ["max", "high"], "default_effort": "low"}, {"reasoning_effort_levels": ["max", "high"]}), + ), +) +async def test_model_reasoning_contract_is_not_invented_from_boolean_support( + reasoning: Mapping[str, object], expected: Mapping[str, object] +) -> None: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={"data": {"id": "org/model", "endpoints": [_endpoint()]}} + if request.url.path.endswith("/endpoints") + else _catalog(reasoning), + ) + ) + ) as client: + handler.client = client + _, metadata = await _inventory(handler, InMemoryCache()) + assert { + k: v for k, v in metadata.items() if k in ("reasoning_effort_levels", "default_reasoning_effort") + } == expected + + +@pytest.mark.parametrize( + "failure", + ( + httpx.Response(401), + httpx.Response(503), + httpx.Response(302, headers={"location": "https://foreign.test"}), + httpx.Response(200, content=b"bad json"), + httpx.Response(200, json={"data": {"id": "wrong/model", "endpoints": [_endpoint()]}}), + httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint(context_length=True)]}}), + httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint(max_prompt_tokens=-1)]}}), + httpx.Response(200, content=b"x" * (2 * 1024 * 1024 + 1)), + httpx.ReadTimeout("fixture timeout"), + ), +) +async def test_endpoint_failure_retains_same_identity_metadata_but_never_imports_aggregate_limits( + failure: httpx.Response | Exception, +) -> None: + endpoint_response: Final = Mock( + return_value=httpx.Response(200, json={"data": {"id": "org/model", "endpoints": [_endpoint()]}}) + ) + + def respond(request: httpx.Request) -> httpx.Response: + return endpoint_response() if request.url.path.endswith("/endpoints") else httpx.Response(200, json=_catalog()) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond), follow_redirects=True) as client: + handler.client = client + cache: Final = InMemoryCache() + first, metadata = await _inventory(handler, cache) + endpoint_response.side_effect = failure if isinstance(failure, Exception) else None + endpoint_response.return_value = failure + _, retained = await _inventory(handler, cache, refresh=True, previous=first) + assert retained == metadata + _, unavailable = await _inventory(handler, cache) + assert unavailable == {} + assert endpoint_response.call_count == 2 + _, other_identity = await _inventory(handler, cache, key="other-key", previous=first) + assert other_identity == {} + + +async def test_endpoint_cache_separates_models_credentials_and_bases_and_refreshes() -> None: + clock: Final = Mock(return_value=0) + responder: Final = Mock( + side_effect=lambda request: httpx.Response( + 200, + json={ + "data": { + "id": "org/second" if "/org/second/" in request.url.path else "org/model", + "endpoints": [_endpoint()], + } + } + if request.url.path.endswith("/endpoints") + else _catalog(), + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + cache: Final = InMemoryCache(clock=clock) + await _inventory(handler, cache) + await _inventory(handler, cache, only=("supplier",)) + assert responder.call_count == 2 + await _inventory(handler, cache, model="org/second") + assert responder.call_count == 3 + await _inventory(handler, cache, key="other") + assert responder.call_count == 5 + await _inventory(handler, cache, base="https://other.test/v1") + assert responder.call_count == 7 + await _inventory(handler, cache, refresh=True) + assert responder.call_count == 9 + clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1 + await _inventory(handler, cache) + assert responder.call_count == 11 + + +@pytest.mark.parametrize("endpoints", ([], [_endpoint("other")])) +async def test_successful_endpoint_absence_clears_last_known_metadata(endpoints: list[Mapping[str, object]]) -> None: + response: Final = Mock(return_value=[_endpoint()]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={"data": {"id": "org/model", "endpoints": response()}} + if request.url.path.endswith("/endpoints") + else _catalog(), + ) + ) + ) as client: + handler.client = client + cache: Final = InMemoryCache() + first, metadata = await _inventory(handler, cache) + assert metadata["context_window"] == 1200 + response.return_value = endpoints + current, absent = await _inventory(handler, cache, refresh=True, previous=first) + assert absent == {} + assert "org/model" in current.models + + +@pytest.mark.parametrize("model", ("org/../escape", "../model", "org/..")) +async def test_inventory_identifiers_cannot_change_endpoint_url_structure(model: str) -> None: + responder: Final = Mock(return_value=httpx.Response(200, json={"data": [{"id": model}]})) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + _, metadata = await _inventory(handler, InMemoryCache(), model=model) + assert metadata == {} + assert responder.call_count == 1 diff --git a/tests/unit/llms/test_model_inventory.py b/tests/unit/llms/test_model_inventory.py new file mode 100644 index 00000000000..069f2615a15 --- /dev/null +++ b/tests/unit/llms/test_model_inventory.py @@ -0,0 +1,97 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.model_inventory import ( + get_provider_model_inventory, + model_inventory_api_base, + model_inventory_identity, +) +from litellm.llms.openrouter.responses.transformation import OpenRouterResponsesAPIConfig +from litellm.llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig +from litellm.types.proxy.model_inventory import SupplierModelInventory + + +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +@pytest.mark.parametrize("source", ("explicit", "environment", "global", "default")) +def test_inventory_base_matches_native_resolution_and_preserves_paths( + monkeypatch: pytest.MonkeyPatch, provider: str, source: str +) -> None: + explicit: Final = "https://explicit.test/custom/v1/" if source == "explicit" else None + environment: Final = "https://environment.test/custom/v1" + global_base: Final = "https://global.test/custom/v1" + monkeypatch.setattr(litellm, "api_base", global_base if source in ("explicit", "global") else None) + variable: Final = "OPENROUTER_API_BASE" if provider == "openrouter" else "VERCEL_AI_GATEWAY_API_BASE" + if source in ("explicit", "environment"): + monkeypatch.setenv(variable, environment) + else: + monkeypatch.delenv(variable, raising=False) + expected: Final = ( + explicit or OpenRouterResponsesAPIConfig().get_complete_url(None, {}).removesuffix("/responses") + if provider == "openrouter" + else VercelAIGatewayConfig().get_openai_compatible_provider_info(explicit, "fixture-key")[0] + ) + assert model_inventory_api_base(provider, explicit) == expected + + +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +async def test_native_base_resolution_does_not_borrow_ambient_discovery_credentials( + monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + base: Final = "https://supplier.test/custom/v1" + variable: Final = "OPENROUTER" if provider == "openrouter" else "VERCEL_AI_GATEWAY" + monkeypatch.setenv(f"{variable}_API_BASE", base) + monkeypatch.setenv(f"{variable}_API_KEY", "ambient-provider-key") + monkeypatch.setenv("VERCEL_OIDC_TOKEN", "ambient-oidc") + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "api_key", "ambient-global-key") + + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == f"{base}/models" + assert "authorization" not in request.headers + return httpx.Response(200, json={"data": [{"id": "org/backend"}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + inventory: Final = await get_provider_model_inventory( + provider=provider, api_base=None, headers={}, client=handler, cache=InMemoryCache() + ) + assert isinstance(inventory, SupplierModelInventory) + assert frozenset(inventory.models) == {"org/backend"} + + +@pytest.mark.parametrize( + "provider,expected", + [ + ("openrouter", "https://openrouter.ai/api/v1"), + ("vercel_ai_gateway", "https://ai-gateway.vercel.sh/v1"), + ("openai", None), + ], +) +def test_inventory_connection_defaults_and_explicit_endpoints(provider, expected): + assert model_inventory_api_base(provider, None) == expected + assert model_inventory_api_base(provider, "https://configured.test/v1") == "https://configured.test/v1" + assert model_inventory_identity(provider) is None + + +def test_native_inventory_registry_uses_server_owned_backend_and_identity( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + trusted: Final = "https://server-owned.test/backend-api/codex" + monkeypatch.setenv("CHATGPT_API_BASE", trusted) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "auth.json") + (tmp_path / "auth.json").write_text(json.dumps({"account_id": "fixture-account"})) + assert model_inventory_api_base("chatgpt", None) == trusted + assert model_inventory_api_base("chatgpt", trusted + "/") == trusted + assert model_inventory_identity("chatgpt") == "fixture-account" + with pytest.raises(ValueError, match="server-configured API base"): + model_inventory_api_base("chatgpt", "https://caller-owned.test/capture") diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 83761620cc9..1388ab9081f 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -8,6 +8,38 @@ import pytest import respx +@pytest.mark.parametrize( + "key_grants,team_grants,groups_only,expected", + ( + (("public/*",), (), False, ("public/selected",)), + (("public/sel*",), (), False, ("public/selected",)), + (("*",), (), False, ("public/selected", "private/selected")), + ((), ("public/*",), False, ("public/selected",)), + (("private/selected",), ("public/*",), False, ("private/selected",)), + (("denied/*",), (), False, ()), + (("public/*",), (), True, ()), + (("no-default-models",), ("public/*",), False, ()), + ), +) +def test_wildcard_grants_list_deployed_public_names_without_provider_inventory( + key_grants: tuple[str, ...], + team_grants: tuple[str, ...], + groups_only: bool, + expected: tuple[str, ...], +) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + result: Final = get_complete_model_list( + key_models=key_grants, + team_models=team_grants, + proxy_model_list=["public/selected", "public/selected", "private/selected", "public/*"], + user_model=None, + infer_model_from_keys=False, + only_model_access_groups=groups_only, + ) + assert tuple(result) == expected + + def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 1163090c510..417bec744d9 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -266,3 +266,21 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +def test_model_offering_identity_uses_server_key_and_not_a_public_credential_hash(monkeypatch): + import hashlib + import hmac + + from litellm.proxy.common_utils.encrypt_decrypt_utils import model_offering_identity_fingerprint + + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + low_entropy_header = b'Basic dXNlcjpwYXNz' + identity = model_offering_identity_fingerprint(low_entropy_header) + assert identity == hmac.new( + b"fixture-server-owned-key", b"litellm:model-offering-identity:v1\0" + low_entropy_header, hashlib.sha256 + ).hexdigest() + assert identity != hashlib.sha256(low_entropy_header).hexdigest() + assert identity != model_offering_identity_fingerprint(b'Basic dXNlcjpuZXctcGFzcw==') + monkeypatch.setenv("LITELLM_SALT_KEY", "other-server-owned-key") + assert identity != model_offering_identity_fingerprint(low_entropy_header) diff --git a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py index 3e3e89f8117..1bc428649a0 100644 --- a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py +++ b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py @@ -77,9 +77,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ), ], ) -def test_remove_sensitive_info_from_deployment( - model_config: dict, expected_config: dict -): +def test_remove_sensitive_info_from_deployment(model_config: dict, expected_config: dict): sanitized_config = remove_sensitive_info_from_deployment(model_config) assert sanitized_config == expected_config @@ -105,15 +103,10 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): assert "*" in sanitized_config["litellm_params"]["access_token"] # With excluded_keys, litellm_credentials_name should NOT be masked. - # ``remove_sensitive_info_from_deployment`` mutates its input, so feed it - # a fresh copy rather than the already-sanitized one. sanitized_config = remove_sensitive_info_from_deployment( copy.deepcopy(base_config), excluded_keys={"litellm_credentials_name"} ) - assert ( - sanitized_config["litellm_params"]["litellm_credentials_name"] - == "my-credential-name" - ) + assert sanitized_config["litellm_params"]["litellm_credentials_name"] == "my-credential-name" # access_token should still be masked (not in excluded_keys) assert sanitized_config["litellm_params"]["access_token"] != "token-12345" @@ -121,3 +114,22 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): # api_key should still be removed (popped) regardless of excluded_keys assert "api_key" not in sanitized_config["litellm_params"] + + +def test_public_deployment_omits_all_headers_without_mutating_runtime_parameters(): + configured = { + "model_name": "selected", + "litellm_params": { + "model": "openai/backend", + "api_key": "private-key", + "extra_headers": {"X-Session": "opaque-session", "Trace-Value": "private-trace"}, + "headers": {"Unclassified": "another-secret"}, + "temperature": 0.25, + }, + } + original = copy.deepcopy(configured) + public = remove_sensitive_info_from_deployment(configured, excluded_keys={"extra_headers", "headers"}) + assert public["litellm_params"] == {"model": "openai/backend", "temperature": 0.25} + assert configured == original + assert public is not configured + assert public["litellm_params"] is not configured["litellm_params"] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..ca2c58d8d04 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4396,6 +4396,45 @@ class TestModelInfoCostMapEchoFilter: "cost-map metadata must not be persisted from an unchanged /model/info echo" ) + @pytest.mark.parametrize("case", ("echo", "edited", "stored")) + def test_typed_metadata_compares_wire_values_without_freezing_catalog_echoes( + self, monkeypatch: pytest.MonkeyPatch, case: str + ) -> None: + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + + catalog: Final = { + "mode": "chat", + "litellm_provider": "openai", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + "request_defaults": {"output_token_budget": 100, "output_token_budget_by_reasoning_effort": {"max": 180}}, + } + monkeypatch.setattr(litellm, "model_cost", {"openai/metadata-fixture": catalog}) + entry: Final = litellm.get_model_info("openai/metadata-fixture") + fields: Final = frozenset(catalog) - {"mode", "litellm_provider"} + stored: Final = {field: catalog[field] for field in fields} if case == "stored" else {} + edited: Final = {"supported_modalities": ["text"]} if case == "edited" else {} + deployment: Final = Deployment( + model_name="public-model", + litellm_params=LiteLLM_Params(model="openai/metadata-fixture"), + model_info=ModelInfo(id="private-deployment", **stored), + ) + result: Final = update_db_model( + db_model=deployment, + updated_patch=updateDeployment(model_info=ModelInfo(**{**entry, **edited, "access_groups": ["allowed"]})), + loaded_catalog=lambda: {"openai/metadata-fixture": catalog}, + ) + saved: Final = json.loads(result["model_info"]) + assert saved["access_groups"] == ["allowed"] + assert {field: value for field, value in saved.items() if field in fields} == ( + stored if case == "stored" else edited + ) + def test_an_edited_value_survives_the_echo_filter(self): import litellm diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 11a44a0ec16..799fd569f36 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -1408,3 +1408,65 @@ async def test_prometheus_fallback_stats_job_runs_when_the_lock_is_free_or_absen await jobs["prometheus_fallback_stats_job"]() assert send_fallback_stats.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("configuration", ["valid", "invalid", "mixed"]) +async def test_external_inventory_lifespan_does_not_close_shared_clients(tmp_path, monkeypatch, configuration): + import httpx + import yaml + + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.router import Router + from litellm.types.llms.custom_http import httpxSpecialProvider + + path = tmp_path / "offerings.yaml" + path.write_text( + "invalid: true\n" + if configuration == "invalid" + else yaml.safe_dump( + { + "version": 1, + "providers": {"source": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "manual", "source": "manual", "provider": "source", "upstream_model": "backend"} + ], + } + ) + ) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + inventory = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + inference = get_async_httpx_client("openai") + assert inventory is not inference + await inventory.client.aclose() + inventory.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200))) + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + monkeypatch.setattr(ps, "llm_router", Router(model_list=[])) + monkeypatch.setattr(ps, "llm_model_list", [] if configuration != "mixed" else [{"model_name": "unmanaged"}]) + monkeypatch.setattr(ps, "store_model_in_db", False) + monkeypatch.setattr(ps, "scheduler", None) + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + ps.general_settings, {"model_offerings_path": str(path), "disable_model_info_refresh": True}, clear=True + ), + ): + try: + if configuration == "valid": + async with proxy_startup_event(app=None): + assert ps.llm_router.model_names == {"manual"} + else: + with pytest.raises(ValueError, match=r"external offering|External offering"): + async with proxy_startup_event(app=None): + pytest.fail("Invalid external config must reject startup") + assert get_async_httpx_client(httpxSpecialProvider.ModelInventory) is inventory + assert not inventory.client.is_closed + assert not inference.client.is_closed + assert (await inventory.get("https://supplier.test/v1/models")).status_code == 200 + finally: + await inventory.close() + await inference.close() diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 2c0e0944edb..f175df3523b 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -13,6 +13,7 @@ import asyncio import copy from collections.abc import Callable from contextlib import AbstractContextManager +from pathlib import Path from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -86,6 +87,7 @@ async def test_discovery_preserves_model_info_fallbacks( { "id": backend_model.split("/", 1)[1], "max_model_len": advertised_limit, + "max_input_tokens": advertised_limit, } ] }, @@ -143,7 +145,9 @@ async def test_upstream_limits_reach_model_info_routes( def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/models" - return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": 4096}]}) + return httpx.Response( + 200, json={"data": [{"id": "org/local-model", "max_model_len": 4096, "max_input_tokens": 4096}]} + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -1307,3 +1311,102 @@ def test_v2_model_info_access_group_paginates_over_the_filtered_set(client, auth assert _model_names(payload) == ["openai/*"] assert payload["total_count"] == 2 assert payload["total_pages"] == 2 + + +async def test_external_headers_are_private_on_permitted_nonadmin_discovery( + tmp_path: Path, + mock_prisma: MagicMock, + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + import yaml + from openai import AsyncOpenAI + + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.model_offerings import ModelOfferingsManager + + secret: Final = "opaque-session-fixture" + monkeypatch.setenv("SUPPLIER_SESSION", secret) + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "source": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": "supplier-fixture-key", + "headers": {"X-Session": "os.environ/SUPPLIER_SESSION"}, + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "source", + "upstream_model": "fixture-backend", + "model_info": {"mode": "chat", "context_window": 1000}, + } + ], + } + ) + ) + catalog_handler: Final = AsyncHTTPHandler() + await catalog_handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as catalog_client: + catalog_handler.client = catalog_client + manager: Final = ModelOfferingsManager( + path=path, template=litellm.Router(model_list=[], num_retries=0), client=catalog_handler + ) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(proxy_server, "user_config_file_path", None) + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + deployment_id: Final = manager.router.get_model_ids()[0] + with auth_as(LitellmUserRoles.INTERNAL_USER, models=["selected"]): + for route, query in ( + ("/model/info", {}), + ("/v1/model/info", {"litellm_model_id": deployment_id}), + ("/v2/model/info", {}), + ("/v2/model/info", {"modelId": deployment_id}), + ("/model_group/info", {}), + ("/v1/models", {}), + ("/v1/models/selected", {}), + ): + response: Final = client.get(route, params=query) + assert response.status_code == 200, response.text + assert "selected" in response.text + assert secret not in response.text + assert "x-session" not in response.text.lower() + runtime_headers: Final = manager.router.get_model_list()[0]["litellm_params"]["extra_headers"] + assert runtime_headers["x-session"] == secret + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["x-session"] == secret + return httpx.Response( + 200, + json={ + "id": "fixture-response", + "object": "chat.completion", + "created": 1, + "model": "fixture-backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + async with AsyncOpenAI(api_key="supplier-fixture-key", http_client=http_client, max_retries=0) as sdk_client: + result: Final = await manager.router.acompletion( + model="selected", messages=[{"role": "user", "content": "fixture"}], client=sdk_client + ) + assert result.choices[0].message.content == "accepted" diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index 679959b5728..1e2f0596048 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -1137,6 +1137,11 @@ async def test_v1_models_team_alias_inherits_token_limits_and_chat_mode(monkeypa "mode": "chat", "max_input_tokens": 876000, "max_output_tokens": 128000, + "supports_function_calling": True, + "supports_parallel_function_calling": True, + "supported_endpoints": ["/v1/chat/completions", "/v1/batch", "/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], "metadata": {"fallbacks": []}, } ] @@ -1183,6 +1188,7 @@ async def test_v1_models_team_image_alias_inherits_image_generation_mode(monkeyp "created": 1677610602, "owned_by": "openai", "mode": "image_generation", + "supported_endpoints": ["/v1/images/generations", "/v1/images/edits"], } ] diff --git a/tests/unit/proxy/test_model_discovery.py b/tests/unit/proxy/test_model_discovery.py new file mode 100644 index 00000000000..43c875f1121 --- /dev/null +++ b/tests/unit/proxy/test_model_discovery.py @@ -0,0 +1,67 @@ +from typing import Final + +import pytest + +from litellm.proxy.model_discovery import discover_model_metadata +from litellm.types.router import ModelMetadata, guaranteed_token_limit, resolve_model_metadata + + +def test_configured_overrides_preserve_false_empty_and_independent_limits(): + metadata: Final = resolve_model_metadata( + { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "supports_function_calling": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + }, + {"context_window": 1000, "supports_function_calling": False, "reasoning_effort_levels": []}, + ) + assert metadata.context_window == 1000 + assert metadata.max_input_tokens == 1100 + assert metadata.max_output_tokens == 200 + assert discover_model_metadata((metadata,)) == { + "context_window": 1000, + "supports_function_calling": False, + "reasoning_effort_levels": [], + } + + +def test_mixed_deployments_advertise_only_guaranteed_metadata(): + sources: Final = ( + ModelMetadata( + context_window=1200, + max_input_tokens=1100, + max_output_tokens=200, + supports_function_calling=True, + supports_parallel_function_calling=True, + reasoning_effort_levels=["low", "high", "max"], + default_reasoning_effort="high", + supported_modalities=["text", "image"], + ), + ModelMetadata( + context_window=800, + max_input_tokens=700, + max_output_tokens=100, + supports_function_calling=False, + reasoning_effort_levels=["low", "high"], + default_reasoning_effort="low", + supported_modalities=["text"], + ), + ) + assert discover_model_metadata(sources) == { + "context_window": 800, + "supports_function_calling": False, + "reasoning_effort_levels": ["low", "high"], + "supported_modalities": ["text"], + } + assert guaranteed_token_limit(sources, "max_input_tokens") == 700 + assert guaranteed_token_limit(sources, "max_output_tokens") == 100 + + +@pytest.mark.parametrize("value", [None, True, 0, -1, "1200", 1.5]) +def test_unknown_or_invalid_context_never_fabricates_capacity(value): + metadata: Final = ModelMetadata(context_window=value, max_input_tokens=1100, max_output_tokens=200) + assert discover_model_metadata((metadata,)) == {} + assert discover_model_metadata((ModelMetadata(context_window=1200), metadata)) == {} diff --git a/tests/unit/proxy/test_model_offerings.py b/tests/unit/proxy/test_model_offerings.py new file mode 100644 index 00000000000..a5dd099927e --- /dev/null +++ b/tests/unit/proxy/test_model_offerings.py @@ -0,0 +1,1422 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from fastapi import FastAPI +from openai import AsyncOpenAI +from starlette.requests import Request + +import litellm +from litellm.caching.caching import Cache, DualCache +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.model_offerings import ModelOfferingsManager +from litellm.proxy.offering_router import OfferingAccessGuard, OfferingSnapshotMiddleware +from litellm.proxy.utils import ProxyLogging +from litellm.router import Router +from litellm.types.utils import CallTypesLiteral + + +@pytest.mark.parametrize("provider", ("openrouter", "vercel_ai_gateway")) +async def test_native_base_changes_invalidate_connection_inventory_and_preserve_pinned_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + variable: Final = "OPENROUTER" if provider == "openrouter" else "VERCEL_AI_GATEWAY" + first_base: Final = "https://first.test/custom/v1" + second_base: Final = "https://second.test/custom/v1" + monkeypatch.setenv(f"{variable}_API_BASE", first_base) + monkeypatch.setenv(f"{variable}_API_KEY", "ambient-provider-key") + monkeypatch.setattr(litellm, "api_base", None) + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"supplier": {"provider": provider, "api_key": "configured-key"}}, + "offerings": [ + { + "model_name": "selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + } + ], + } + ) + ) + responses: Final = iter( + ( + (first_base, 200), + (second_base, 401), + (second_base, 200), + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + expected_base, status = next(responses) + assert str(request.url) == f"{expected_base}/models" + assert request.headers["authorization"] == "Bearer configured-key" + return httpx.Response(status, json={"data": [{"id": "org/backend"}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + original_ids: Final = manager.router.get_model_ids() + with manager.router.pin_snapshot(): + monkeypatch.setenv(f"{variable}_API_BASE", second_base) + assert await manager.reload() + assert manager.router.get_model_ids() == original_ids + assert manager.router.latest_snapshot().available_models == set() + assert manager.router.get_model_names() == [] + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + assert manager.router.get_model_ids() != original_ids + + +@pytest.mark.parametrize("catalog_state", ("missing", "known", "different_route_provider")) +async def test_wildcard_listing_tracks_selected_offerings_and_pinned_reload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, catalog_state: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "openrouter/org/new-model", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/new-model", + "model_info": {"mode": "chat", "context_window": 4000}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + existing: Final = { + "model_name": "chatgpt/existing", + "source": "manual", + "provider": "secondary", + "upstream_model": "org/existing", + } + other_existing: Final = tuple( + {**existing, "model_name": name} for name in ("chatgpt/second", "chatgpt/third", "chatgpt/fourth") + ) + existing_names: Final = frozenset((existing["model_name"], *(row["model_name"] for row in other_existing))) + denied: Final = {**existing, "model_name": "restricted/other"} + disabled: Final = {**selected, "model_name": "openrouter/org/disabled", "enabled": False} + absent: Final = {**selected, "model_name": "openrouter/org/absent", "upstream_model": "org/absent"} + config: Final = { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "private-fixture-key", + }, + "secondary": {"provider": "openai", "api_base": "https://secondary.test/v1"}, + }, + "offerings": [existing, *other_existing, denied, disabled, absent], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response(200, json={"data": [{"id": "org/new-model"}, {"id": "org/unselected"}]}) + ) + ) as client: + handler.client = client + template: Final = Router( + model_list=[], + model_group_alias={"openrouter/*": existing["model_name"]} + if catalog_state == "different_route_provider" + else {}, + ) + manager: Final = ModelOfferingsManager(path=path, template=template, client=handler, clock=lambda: 0.0) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(litellm, "callbacks", [OfferingAccessGuard(manager.router)]) + monkeypatch.setattr( + litellm, + "models_by_provider", + { + "openrouter": ["org/catalog-only"] if catalog_state == "missing" else [selected["model_name"]], + "chatgpt": sorted(existing_names), + "openai": ["fixture/catalog-only"], + }, + ) + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + assert (selected["model_name"] in get_known_models_from_wildcard("openrouter/*")) == ( + catalog_state != "missing" + ) + wildcard: Final = UserAPIKeyAuth( + api_key="wildcard", user_role=LitellmUserRoles.INTERNAL_USER, models=["openrouter/*", "chatgpt/*"] + ) + exact: Final = UserAPIKeyAuth( + api_key="exact", user_role=LitellmUserRoles.INTERNAL_USER, models=sorted(existing_names) + ) + denied_key: Final = UserAPIKeyAuth( + api_key="denied", user_role=LitellmUserRoles.INTERNAL_USER, models=["denied/*"] + ) + team_key: Final = UserAPIKeyAuth( + api_key="team", user_role=LitellmUserRoles.INTERNAL_USER, team_id="team-one", team_models=["openrouter/*"] + ) + alias_key: Final = UserAPIKeyAuth( + api_key="alias", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["shortcut"], + aliases={"shortcut": selected["model_name"]}, + ) + admin: Final = UserAPIKeyAuth(api_key="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + credentials: Final = MappingProxyType( + {auth.api_key: auth for auth in (wildcard, exact, denied_key, team_key, alias_key, admin)} + ) + application: Final = FastAPI() + application.include_router(proxy_server.router) + application.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: manager.router) + + async def scoped_auth(request: Request) -> UserAPIKeyAuth: + if edit := request.headers.get("x-fixture-offerings"): + path.write_text(yaml.safe_dump(json.loads(edit))) + assert await manager.reload() + return credentials[request.headers["authorization"].removeprefix("Bearer ")] + + application.dependency_overrides[proxy_server.user_api_key_auth] = scoped_auth + + async def listed(auth: UserAPIKeyAuth, edit: str | None = None) -> frozenset[str]: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=application), base_url="http://gateway.test" + ) as gateway: + response: Final = await gateway.get( + "/v1/models", + headers={ + "Authorization": f"Bearer {auth.api_key}", + **({"x-fixture-offerings": edit} if edit is not None else {}), + }, + ) + assert response.status_code == 200, response.text + assert "private-fixture-key" not in response.text + assert "supplier.test" not in response.text + assert "secondary.test" not in response.text + assert "org/existing" not in response.text + assert "synthetic/fp8" not in response.text + assert "offering-" not in response.text + return frozenset(row["id"] for row in response.json()["data"]) + + assert await listed(exact) == existing_names + assert await listed(admin) == existing_names | {"restricted/other"} + assert await listed(admin, json.dumps({**config, "offerings": [*config["offerings"], selected]})) == ( + existing_names | {"restricted/other"} + ) + assert await listed(admin) == existing_names | {"restricted/other", selected["model_name"]} + assert await listed(wildcard) == existing_names | {selected["model_name"]} + assert await listed(exact) == existing_names + assert await listed(denied_key) == set() + assert await listed(team_key) == {selected["model_name"]} + assert await listed(alias_key) == {"shortcut"} + assert await listed(wildcard, json.dumps(config)) == existing_names | {selected["model_name"]} + assert await listed(wildcard) == existing_names + assert await listed(alias_key) == set() + assert await listed(team_key) == set() + assert await listed(admin) == existing_names | {"restricted/other"} + + +async def test_authoritative_inventory_failure_recovery_and_atomic_config_reload(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": "backend"}, + { + "model_name": "manual", + "source": "manual", + "provider": "supplier", + "upstream_model": "handwritten", + "model_info": {"context_window": 12345}, + }, + ], + } + path.write_text(yaml.safe_dump(config)) + responses: Final = iter( + ( + httpx.Response(200, json={"data": [{"id": "backend", "context_window": 8000}, {"id": "unselected"}]}), + httpx.Response(401, json={"error": "expired"}), + httpx.Response(200, json={"missing": "data"}), + httpx.ReadTimeout("supplier timed out"), + httpx.Response(200, json={"data": [], "error": {"message": "failed"}}), + httpx.Response(200, json={"data": [{"id": "backend", "context_window": "bad"}]}), + httpx.Response(200, json={"data": [{"id": "backend", "supports_function_calling": "bad"}]}), + httpx.Response(200, json={"data": [{"id": "backend", "supported_endpoints": 123}]}), + httpx.Response(200, json={"data": [{"id": " backend "}]}), + httpx.Response(200, json={"data": []}), + httpx.Response(200, json={"data": [{"id": "backend", "context_window": 16000}]}), + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + response: Final = next(responses) + if isinstance(response, Exception): + raise response + return response + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.model_names == {"automatic", "manual"} + assert manager.router.latest_snapshot().available_models == {"automatic", "manual"} + first: Final = manager.router.latest_snapshot() + for _ in range(8): + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == first.available_models + assert ( + manager.router.get_model_listing_info("automatic").deployments[0].model_info["context_window"] == 8000 + ) + pinned: Final = manager.router.latest_snapshot() + with manager.router.pin_snapshot(): + assert await manager.reload(force_inventory=True) + assert manager.router.serving_snapshot() is pinned + assert manager.router.latest_snapshot().available_models == {"manual"} + assert manager.router.serving_snapshot().available_models == {"manual"} + assert manager.router.get_model_group_info("automatic") is None + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"automatic", "manual"} + assert manager.router.get_model_listing_info("automatic").deployments[0].model_info["context_window"] == 16000 + valid: Final = manager.router.latest_snapshot() + path.write_text("version: 1\nofferings: broken\n") + assert not await manager.reload() + assert manager.router.latest_snapshot() is valid + path.write_text(yaml.safe_dump({**config, "offerings": [config["offerings"][1]]})) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == {"manual"} + assert manager.router.model_names == {"manual"} + + +async def test_disabled_manual_offering_and_invalid_secrets_keep_authority_atomic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.offering_router import OfferingAccessGuard + + path: Final = tmp_path / "offerings.yaml" + monkeypatch.setenv("FIXTURE_OFFERING_KEY", "fixture-secret") + config: Final = { + "version": 1, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": "os.environ/FIXTURE_OFFERING_KEY", + "headers": {"X-Session": "os.environ/FIXTURE_OFFERING_KEY"}, + } + }, + "offerings": [ + {"model_name": "manual", "source": "manual", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + active: Final = manager.router.latest_snapshot() + assert active.available_models == {"manual"} + assert await manager.reload() + assert manager.router.latest_snapshot() is active + monkeypatch.delenv("FIXTURE_OFFERING_KEY") + assert not await manager.reload() + assert manager.router.latest_snapshot() is active + assert manager.last_error == "ValueError" + path.write_bytes(b" " * (1024 * 1024 + 1)) + assert not await manager.reload() + assert manager.router.latest_snapshot() is active + assert manager.last_error == "ConfigTooLarge" + monkeypatch.setenv("FIXTURE_OFFERING_KEY", "fixture-secret") + path.write_text(yaml.safe_dump({**config, "offerings": [{**config["offerings"][0], "enabled": False}]})) + assert await manager.reload() + assert not manager.router.latest_snapshot().available_models + assert "disabled" in manager.router.latest_snapshot().unavailable_models["manual"] + rejected: Final = await OfferingAccessGuard(manager.router).async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "manual"}, "acompletion" + ) + assert isinstance(rejected, litellm.ServiceUnavailableError) + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == {"manual"} + + +async def test_new_connection_cannot_reuse_another_credentials_inventory(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + first_config: Final = { + "version": 1, + "providers": { + "supplier": {"provider": "openai", "api_base": "https://supplier.test/v1", "api_key": "first-private-key"} + }, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(first_config)) + responses: Final = iter( + ( + httpx.Response(200, json={"data": [{"id": "backend"}]}), + httpx.Response(401), + httpx.Response(200, json={"data": [{"id": "backend"}]}), + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + changed: Final = { + **first_config, + "providers": {"supplier": {**first_config["providers"]["supplier"], "api_key": "second-private-key"}}, + } + path.write_text(yaml.safe_dump(changed)) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == set() + assert manager.router.get_model_group_info("selected") is None + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + + +async def test_pinned_metadata_does_not_read_new_snapshot_shared_catalog(tmp_path: Path) -> None: + from litellm.proxy.utils import create_model_info_response + + path: Final = tmp_path / "offerings.yaml" + initial: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + { + "model_name": "snapshot-only-model", + "source": "manual", + "provider": "supplier", + "upstream_model": "snapshot-only-backend", + "model_info": {"context_window": 8000}, + } + ], + } + path.write_text(yaml.safe_dump(initial)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + with manager.router.pin_snapshot(): + before: Final = create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) + changed: Final = { + **initial, + "offerings": [ + {**initial["offerings"][0], "model_info": {"context_window": 16000, "max_output_tokens": 5001}} + ], + } + path.write_text(yaml.safe_dump(changed)) + assert await manager.reload() + assert create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) == before + after: Final = create_model_info_response("snapshot-only-model", "openai", llm_router=manager.router) + assert before["context_window"] == 8000 + assert "max_output_tokens" not in before + assert after["context_window"] == 16000 + assert after["max_output_tokens"] == 5001 + + +async def test_snapshot_factory_dispatch_and_reload_resources_are_isolated() -> None: + import gc + import weakref + + import litellm + + template: Final = Router(model_list=[], num_retries=0) + callback_counts: Final = tuple( + len(callbacks) + for callbacks in ( + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ) + ) + models: Final = ( + {"model_name": "selected", "litellm_params": {"model": "openai/fixture", "api_key": "local-only"}}, + ) + first: Final = template.snapshot_with_model_list(models) + response: Final = await first.aresponses(model="selected", input="fixture", mock_response="local-only") + assert response.output[0].content[0].text == "local-only" + assert template.get_model_names() == [] + assert template.deployment_names == [] + assert first.deployment_names == ["openai/fixture"] + references: Final = tuple(weakref.ref(template.snapshot_with_model_list(models)) for _ in range(30)) + gc.collect() + assert all(reference() is None for reference in references) + assert template.deployment_names == [] + assert first.deployment_names == ["openai/fixture"] + assert callback_counts == tuple( + len(callbacks) + for callbacks in ( + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ) + ) + + +async def test_auto_catalog_fallback_is_frozen_and_manual_metadata_stays_handwritten(tmp_path: Path) -> None: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + path: Final = tmp_path / "offerings.yaml" + upstream: Final = "text-embedding-3-large" + bundled: Final = GetModelCostMap.loaded_model_cost_map()[upstream] + assert bundled["max_input_tokens"] > 0 + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": upstream}, + {"model_name": "manual", "source": "manual", "provider": "supplier", "upstream_model": upstream}, + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"data": [{"id": upstream, "context_window": 12345}]}) + ) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + auto: Final = manager.router.get_model_listing_info("automatic").deployments[0].model_info + manual: Final = manager.router.get_model_listing_info("manual").deployments[0].model_info + assert auto["context_window"] == 12345 + assert auto["max_input_tokens"] == bundled["max_input_tokens"] + assert auto["mode"] == "embedding" + assert manual.get("context_window") is None + assert manual.get("max_input_tokens") is None + + +async def test_changed_inventory_interval_refreshes_without_retaining_old_deadline(tmp_path: Path) -> None: + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "inventory_poll_seconds": 86400, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "automatic", "source": "auto", "provider": "supplier", "upstream_model": "backend"} + ], + } + path.write_text(yaml.safe_dump(config)) + responses: Final = iter( + (httpx.Response(200, json={"data": [{"id": "backend"}]}), httpx.Response(200, json={"data": []})) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"automatic"} + path.write_text(yaml.safe_dump({**config, "inventory_poll_seconds": 1})) + assert await manager.reload() + assert manager.router.latest_snapshot().available_models == set() + assert manager.next_inventory_refresh == 1.0 + + +@pytest.mark.parametrize("authorization_name", (None, "authorization", "AUTHORIZATION")) +async def test_native_client_forwards_one_supplier_authorization_value( + tmp_path: Path, authorization_name: str | None +) -> None: + path: Final = tmp_path / "offerings.yaml" + api_key: Final = "supplier-fixture-key.with-symbols=" + expected_authorization: Final = ( + "Bearer explicit-fixture-override" if authorization_name is not None else f"Bearer {api_key}" + ) + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "api_key": api_key, + "headers": ( + {authorization_name: expected_authorization} if authorization_name is not None else {} + ), + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "fixture-backend", + } + ], + } + ) + ) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers.get_list("authorization") == [expected_authorization] + return httpx.Response( + 200, + json={ + "id": "fixture-response", + "object": "chat.completion", + "created": 1, + "model": "fixture-backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + }, + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[], num_retries=0), client=handler) + assert await manager.reload(initial=True) + async with AsyncOpenAI(api_key=api_key, http_client=http_client, max_retries=0) as sdk_client: + response: Final = await manager.router.acompletion( + model="selected", messages=[{"role": "user", "content": "authenticate"}], client=sdk_client + ) + assert response.choices[0].message.content == "accepted" + + +async def test_global_affinity_preserves_encrypted_content_across_switch_and_pinned_reload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + monkeypatch.setattr(litellm, "callbacks", []) + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": { + "source": {"provider": "openai", "api_base": "https://supplier.test/v1", "api_key": "fixture-key"} + }, + "offerings": [ + {"model_name": name, "source": "manual", "provider": "source", "upstream_model": "gpt-4o-mini"} + for name in ("parent", "child") + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + template: Final = Router(model_list=[], optional_pre_call_checks=["encrypted_content_affinity"]) + manager: Final = ModelOfferingsManager(path=path, template=template, client=handler) + assert await manager.reload(initial=True) + old: Final = manager.router.latest_snapshot() + origin: Final = old.router.get_model_ids("parent")[0] + wrapped: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("opaque-fixture", origin) + with manager.router.pin_snapshot(): + config["offerings"] = config["offerings"][1:] + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + request: Final = { + "input": [{"type": "reasoning", "encrypted_content": wrapped}, {"role": "user", "content": "fixture"}] + } + child: Final = [row for row in old.router.get_model_list() or [] if row["model_name"] == "child"] + result: Final = await manager.router.async_callback_filter_deployments( + model="child", + healthy_deployments=child, + messages=None, + parent_otel_span=None, + request_kwargs=request, + ) + assert result == child + assert request["input"][0]["encrypted_content"] == wrapped + next_request: Final = { + "input": [{"type": "reasoning", "encrypted_content": wrapped}, {"role": "user", "content": "fixture"}] + } + await manager.router.async_callback_filter_deployments( + model="child", + healthy_deployments=child, + messages=None, + parent_otel_span=None, + request_kwargs=next_request, + ) + assert next_request["input"] == [{"role": "user", "content": "fixture"}] + assert template.deployment_names == [] + + +async def test_private_connection_identity_is_stable_and_changes_on_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + path: Final = tmp_path / "offerings.yaml" + config: Final = { + "version": 1, + "providers": { + "source": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + "headers": {"Authorization": "Basic dXNlcjpwYXNz"}, + } + }, + "offerings": [ + {"model_name": "selected", "source": "manual", "provider": "source", "upstream_model": "gpt-4o-mini"} + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + before: Final = manager.router.get_model_ids() + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids() == before + config["providers"]["source"]["headers"]["Authorization"] = "Basic dXNlcjpuZXctcGFzcw==" + path.write_text(yaml.safe_dump(config)) + assert await manager.reload() + assert manager.router.get_model_ids() != before + + +async def test_native_account_change_changes_private_deployment_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-server-owned-key") + monkeypatch.setenv("CHATGPT_API_BASE", "https://native.test/backend-api/codex") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text(json.dumps({"access_token": "fixture-oauth", "account_id": "first", "expires_at": 10**30})) + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"native": {"provider": "chatgpt"}}, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "native", "upstream_model": "fixture"} + ], + } + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + responses: Final = iter((200, 401, 200)) + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(next(responses), json={"models": [{"slug": "fixture"}]}) + ) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), client=handler) + assert await manager.reload(initial=True) + first: Final = manager.router.get_model_ids() + assert len(first) == 1 + auth_file.write_text(json.dumps({"access_token": "second-oauth", "account_id": "second", "expires_at": 10**30})) + assert await manager.reload(force_inventory=True) + assert not manager.router.latest_snapshot().available_models + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids() != first + assert manager.router.latest_snapshot().available_models == {"selected"} + + +async def test_shared_inventory_client_is_reacquired_after_cache_eviction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": {"source": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + {"model_name": "selected", "source": "auto", "provider": "source", "upstream_model": "backend"} + ], + } + ) + ) + cache: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache) + first: Final = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + await first.client.aclose() + first.client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [{"id": "backend"}]})) + ) + manager: Final = ModelOfferingsManager(path=path, template=Router(model_list=[]), clock=lambda: 0.0) + assert await manager.reload(initial=True) + assert manager.router.latest_snapshot().available_models == {"selected"} + cache.delete_cache(cache.update_cache_key_with_event_loop("async_httpx_clientmodel_inventory")) + await first.close() + second: Final = get_async_httpx_client(httpxSpecialProvider.ModelInventory) + assert second is not first + await second.client.aclose() + second.client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": []})) + ) + try: + assert await manager.reload(force_inventory=True) + assert manager.router.latest_snapshot().available_models == set() + assert not second.client.is_closed + finally: + await second.close() + + +@pytest.mark.parametrize("surface", ("chat", "chat_unified", "responses", "responses_chat", "messages")) +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize("effort", (None, "high", "max")) +@pytest.mark.parametrize("source", ("manual", "auto")) +async def test_provider_policy_reaches_upstream_body_on_every_chat_surface( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, surface: str, stream: bool, effort: str | None, source: str +) -> None: + path: Final = tmp_path / "offerings.yaml" + policy: Final = {"only": ["synthetic/fp8"], "allow_fallbacks": False} + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "fixture-key", + } + }, + "offerings": [ + { + "model_name": "selected", + "source": source, + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat"}, + "provider_policy": {"provider": "openrouter", **policy}, + } + ], + } + ) + ) + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/models"): + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "reasoning": { + "supported_efforts": ["max", "high", "low"], + "default_effort": "high", + }, + } + ] + }, + ) + if request.url.path.endswith("/endpoints"): + return httpx.Response( + 200, + json={ + "data": { + "id": "org/backend", + "endpoints": [ + { + "tag": "synthetic/fp8", + "context_length": 1200, + "max_completion_tokens": 300, + "supported_parameters": ["reasoning", "reasoning_effort"], + } + ], + } + }, + ) + requests.append(request) + body: Final = json.loads(request.content) + assert body["model"] == "org/backend" + assert body["provider"] == policy + assert body["temperature"] == 0.5 + assert (body.get("reasoning") or {}).get("effort", body.get("reasoning_effort")) == effort + if request.url.path.endswith("/responses"): + response: Final = { + "id": "resp_fixture", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "org/backend", + "output": [ + { + "id": "msg_fixture", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "accepted", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + if stream: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + "event: response.completed\ndata: " + + json.dumps( + { + "type": "response.completed", + "sequence_number": 1, + "response": response, + } + ) + + "\n\n" + ), + ) + return httpx.Response(200, json=response) + if stream: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + "data: " + + json.dumps( + { + "id": "fixture", + "object": "chat.completion.chunk", + "created": 1, + "model": "org/backend", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "accepted"}, + "finish_reason": "stop", + } + ], + } + ) + + "\n\ndata: [DONE]\n\n" + ), + ) + return httpx.Response( + 200, + json={ + "id": "fixture", + "object": "chat.completion", + "created": 1, + "model": "org/backend", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "accepted"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + cache: Final = LLMClientCache() + cache.set_cache("async_httpx_clientssl_verify_Noneopenrouter", handler) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", cache) + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[], num_retries=0), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + if source == "auto": + advertised: Final = manager.router.get_deployment_by_model_group_name("selected") + assert advertised is not None + assert advertised.model_info.reasoning_effort_levels == ("max", "high", "low") + assert advertised.model_info.default_reasoning_effort == "high" + guard: Final = OfferingAccessGuard(manager.router) + monkeypatch.setattr(litellm, "callbacks", [guard]) + call_type: Final[CallTypesLiteral] = ( + "acompletion" + if surface in ("chat", "chat_unified") + else "anthropic_messages" + if surface == "messages" + else "aresponses" + ) + response_cache: Final = Cache( + type="local", supported_call_types=["acompletion", "aresponses", "anthropic_messages"] + ) + monkeypatch.setattr(litellm, "cache", response_cache) + response_cache.cache.set_cache("shared-fixture-key", {"response": "unrelated endpoint output"}) + shaped: Final = await ProxyLogging(UserApiKeyCache()).pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + data={ + "model": "selected", + "stream": stream, + "extra_body": { + "temperature": 0.5, + **({"reasoning": {"effort": effort}} if surface == "chat_unified" and effort is not None else {}), + }, + "caching": True, + "preset_cache_key": "shared-fixture-key", + "cache": {"no-cache": False, "no-store": False}, + **({"use_chat_completions_api": True} if surface == "responses_chat" else {}), + **( + ( + {"reasoning": {"effort": effort}} + if surface in ("responses", "responses_chat") + else {"thinking": {"type": "adaptive"}, "output_config": {"effort": effort}} + if surface == "messages" + else {} + if surface == "chat_unified" + else {"reasoning_effort": effort} + ) + if effort is not None + else {} + ), + }, + call_type=call_type, + ) + assert isinstance(shaped, dict) + result: Final = ( + await manager.router.aresponses(input="fixture", client=handler, **shaped) + if surface in ("responses", "responses_chat") + else await manager.router.anthropic_messages( + messages=[{"role": "user", "content": "fixture"}], max_tokens=10, client=handler, **shaped + ) + if surface == "messages" + else await manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **shaped + ) + ) + if stream: + chunks: Final = tuple([chunk async for chunk in result]) + assert chunks + assert len(requests) == 1 + assert requests[0].url.path.endswith("/responses" if surface == "responses" else "/chat/completions") + assert response_cache.cache.get_cache("shared-fixture-key") == {"response": "unrelated endpoint output"} + for control in ( + {"service_tier": "priority"}, + {"extra_body": {"service_tier": "priority"}}, + {"additional_drop_params": ["provider"]}, + {"additional_drop_params": ["extra_body"]}, + {"additional_drop_params": ["extra_body.provider"]}, + {"additional_drop_params": ["provider.only"]}, + {"drop_params": True}, + {"allowed_openai_params": ["provider"]}, + {"extra_body": {"provider": policy, "temperature": 0.5, "additional_drop_params": ["provider"]}}, + ): + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + await ( + manager.router.aresponses(input="fixture", client=handler, **{**shaped, **control}) + if surface in ("responses", "responses_chat") + else manager.router.anthropic_messages( + messages=[{"role": "user", "content": "fixture"}], + max_tokens=10, + client=handler, + **{**shaped, **control}, + ) + if surface == "messages" + else manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **{**shaped, **control} + ) + ) + assert len(requests) == 1 + + +async def test_endpoint_policy_reload_isolates_aliases_identity_and_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.utils import create_model_info_response + + monkeypatch.setenv("LITELLM_SALT_KEY", "fixture-identity-key") + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat", "context_window": 4000}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openrouter", "api_base": "https://supplier.test/v1"}}, + "offerings": [ + selected, + { + "model_name": "ordinary", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + }, + ], + } + path.write_text(yaml.safe_dump(config)) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "context_length": 16000, + "top_provider": {"max_completion_tokens": 8000}, + } + ] + }, + ) + ) + ) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + selected_deployment: Final = manager.router.get_deployment_by_model_group_name("selected") + assert selected_deployment is not None + assert selected_deployment.provider_policy.only == ("synthetic/fp8",) + assert "provider_policy" not in selected_deployment.model_dump() + selected_id: Final = manager.router.get_model_ids("selected") + ordinary_id: Final = manager.router.get_model_ids("ordinary") + pinned_metadata: Final = create_model_info_response("selected", "openrouter", llm_router=manager.router) + ordinary_metadata: Final = create_model_info_response("ordinary", "openrouter", llm_router=manager.router) + assert pinned_metadata["context_window"] == 4000 + assert "max_output_tokens" not in pinned_metadata + assert ordinary_metadata["context_window"] == 16000 + assert ordinary_metadata["max_output_tokens"] == 8000 + assert await manager.reload(force_inventory=True) + assert manager.router.get_model_ids("selected") == selected_id + with manager.router.pin_snapshot(): + path.write_text( + yaml.safe_dump( + { + **config, + "offerings": [ + { + **selected, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/bf16"]}, + }, + config["offerings"][1], + ], + } + ) + ) + assert await manager.reload() + assert ( + manager.router.get_deployment_by_model_group_name("selected").provider_policy + == selected_deployment.provider_policy + ) + assert manager.router.get_model_ids("selected") == selected_id + assert manager.router.get_model_ids("selected") != selected_id + assert manager.router.get_model_ids("ordinary") == ordinary_id + assert manager.router.get_deployment_by_model_group_name("selected").provider_policy.only == ("synthetic/bf16",) + assert manager.router.get_deployment_by_model_group_name("ordinary").provider_policy is None + valid: Final = manager.router.latest_snapshot() + path.write_text( + yaml.safe_dump( + { + **config, + "providers": { + "supplier": { + "provider": "openai", + "api_base": "https://supplier.test/v1", + } + }, + } + ) + ) + assert not await manager.reload() + assert manager.router.latest_snapshot() is valid + + +async def test_endpoint_failure_never_uses_gateway_fallback_and_tampering_never_forwards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path: Final = tmp_path / "offerings.yaml" + path.write_text( + yaml.safe_dump( + { + "version": 1, + "providers": { + "supplier": { + "provider": "openrouter", + "api_base": "https://supplier.test/v1", + "api_key": "fixture-key", + } + }, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "org/pinned", + "model_info": {"mode": "chat"}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + }, + { + "model_name": "ordinary", + "source": "manual", + "provider": "supplier", + "upstream_model": "org/other", + }, + ], + } + ) + ) + forwarded: Final[list[dict[str, object]]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/models"): + return httpx.Response(200, json={"data": []}) + forwarded.append(json.loads(request.content)) + return httpx.Response(503, json={"error": {"message": "synthetic endpoint unavailable", "code": 503}}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, + client=handler, + template=Router( + model_list=[], + num_retries=0, + fallbacks=[{"selected": ["ordinary"]}], + ), + ) + assert await manager.reload(initial=True) + guard: Final = OfferingAccessGuard(manager.router) + monkeypatch.setattr(litellm, "callbacks", [guard]) + shaped: Final = await guard.async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "selected"}, "acompletion" + ) + assert isinstance(shaped, dict) + with pytest.raises(litellm.ServiceUnavailableError, match="synthetic endpoint unavailable"): + await manager.router.acompletion( + messages=[{"role": "user", "content": "fixture"}], client=handler, **shaped + ) + assert tuple(body["model"] for body in forwarded) == ("org/pinned",) + for extra_body in ({"provider": {"only": ["other"]}}, {"model": "org/other"}, {"models": ["org/other"]}): + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + await manager.router.acompletion( + model="selected", + messages=[{"role": "user", "content": "fixture"}], + client=handler, + extra_body=extra_body, + disable_fallbacks=True, + ) + assert len(forwarded) == 1 + + +@pytest.mark.parametrize( + "overrides", + ( + {}, + {"context_window": 900, "max_output_tokens": 150, "max_input_tokens": 500}, + {"default_reasoning_effort": "max"}, + {"supports_reasoning": False, "supports_function_calling": False}, + {"reasoning_effort_levels": []}, + ), +) +async def test_auto_pinned_catalog_preserves_overrides_scope_and_endpoint_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, overrides: Mapping[str, object] +) -> None: + from litellm.proxy import proxy_server + + path: Final = tmp_path / "offerings.yaml" + selected: Final = { + "model_name": "openrouter/org/selected", + "source": "auto", + "provider": "supplier", + "upstream_model": "org/backend", + "model_info": {"mode": "chat", **overrides}, + "provider_policy": {"provider": "openrouter", "only": ["supplier/fp8"], "allow_fallbacks": False}, + } + config: Final = { + "version": 1, + "providers": { + "supplier": {"provider": "openrouter", "api_base": "https://supplier.test/v1", "api_key": "fixture-key"} + }, + "offerings": [ + selected, + {**selected, "model_name": "ordinary", "provider_policy": None}, + {**selected, "model_name": "manual", "source": "manual"}, + ], + } + path.write_text(yaml.safe_dump(config)) + generations: Final = iter((1200, 700, None, None)) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] in ("Bearer fixture-key", "Bearer changed-key") + if request.url.path.endswith("/models"): + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/backend", + "context_length": 9000, + "top_provider": {"max_completion_tokens": 6000}, + "supported_parameters": ["tools", "reasoning_effort"], + "reasoning": {"supported_efforts": ["max", "high", "low"], "default_effort": "high"}, + }, + {"id": "org/unselected"}, + ] + }, + ) + assert request.url.path == "/v1/models/org/backend/endpoints" + context: Final = next(generations) + return ( + httpx.Response(503) + if context is None + else httpx.Response( + 200, + json={ + "data": { + "id": "org/backend", + "endpoints": [ + { + "tag": "supplier/fp8", + "context_length": context, + "max_completion_tokens": 300, + "max_prompt_tokens": None, + "supported_parameters": ["tools", "reasoning", "reasoning_effort"], + }, + {"tag": "foreign", "context_length": 9000, "max_completion_tokens": 6000}, + ], + } + }, + ) + ) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + manager: Final = ModelOfferingsManager( + path=path, template=Router(model_list=[]), client=handler, clock=lambda: 0.0 + ) + assert await manager.reload(initial=True) + monkeypatch.setattr(proxy_server, "llm_router", manager.router) + monkeypatch.setattr(proxy_server, "llm_model_list", manager.router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + application: Final = FastAPI() + application.include_router(proxy_server.router) + application.add_middleware(OfferingSnapshotMiddleware, router_getter=lambda: manager.router) + + async def scoped_auth(request: Request) -> UserAPIKeyAuth: + if request.headers.get("x-fixture-refresh"): + assert await manager.reload(force_inventory=True) + return UserAPIKeyAuth( + api_key="fixture-client", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["openrouter/*"] if request.headers.get("authorization") == "allowed" else ["denied/*"], + ) + + application.dependency_overrides[proxy_server.user_api_key_auth] = scoped_auth + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=application), base_url="http://gateway.test" + ) as gateway: + denied: Final = await gateway.get("/v1/models", headers={"authorization": "denied"}) + assert denied.status_code == 200 and denied.json()["data"] == [] + first: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert first.status_code == 200 + row: Final = first.json()["data"][0] + assert row["id"] == selected["model_name"] + assert row["context_window"] == overrides.get("context_window", 1200) + assert row["max_output_tokens"] == overrides.get("max_output_tokens", 300) + assert row.get("max_input_tokens") == overrides.get("max_input_tokens") + assert row["supports_function_calling"] == overrides.get("supports_function_calling", True) + if overrides.get("supports_reasoning") is False: + assert row["supports_reasoning"] is False + assert "reasoning_effort_levels" not in row and "default_reasoning_effort" not in row + else: + assert row["reasoning_effort_levels"] == overrides.get( + "reasoning_effort_levels", ["max", "high", "low"] + ) + assert row.get("default_reasoning_effort") == ( + None + if overrides.get("reasoning_effort_levels") == [] + else overrides.get("default_reasoning_effort", "high") + ) + assert all(value is not None for value in row.values()) + assert "supplier/fp8" not in first.text and "fixture-key" not in first.text + ordinary: Final = manager.router.get_deployment_by_model_group_name("ordinary") + manual: Final = manager.router.get_deployment_by_model_group_name("manual") + assert ordinary is not None and ordinary.model_info.context_window == overrides.get("context_window", 9000) + assert manual is not None and manual.model_info.context_window == overrides.get("context_window") + first_ids: Final = manager.router.get_model_ids(selected["model_name"]) + pinned: Final = await gateway.get( + "/v1/models", headers={"authorization": "allowed", "x-fixture-refresh": "1"} + ) + assert pinned.json() == first.json() + changed: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert changed.json()["data"][0]["context_window"] == overrides.get("context_window", 700) + assert manager.router.get_model_ids(selected["model_name"]) == first_ids + assert await manager.reload(force_inventory=True) + retained: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert retained.json() == changed.json() + path.write_text( + yaml.safe_dump( + { + **config, + "providers": { + "supplier": { + **config["providers"]["supplier"], + "api_key": "changed-key", + } + }, + } + ) + ) + assert await manager.reload() + rotated: Final = await gateway.get("/v1/models", headers={"authorization": "allowed"}) + assert rotated.json()["data"][0].get("context_window") == overrides.get("context_window") + assert rotated.json()["data"][0].get("reasoning_effort_levels") == overrides.get("reasoning_effort_levels") + assert manager.router.get_model_ids(selected["model_name"]) != first_ids diff --git a/tests/unit/proxy/test_offering_router.py b/tests/unit/proxy/test_offering_router.py new file mode 100644 index 00000000000..b8f1d4ce3e4 --- /dev/null +++ b/tests/unit/proxy/test_offering_router.py @@ -0,0 +1,357 @@ +from types import MappingProxyType +from typing import Final + +import pytest +from starlette.types import Message, Receive, Scope, Send + +import litellm +from litellm.caching.caching import DualCache +from litellm.proxy._types import ( + LiteLLM_EndUserTable, + LitellmUserRoles, + ModelAccessDeniedProxyException, + ProxyErrorTypes, + UserAPIKeyAuth, +) +from litellm.proxy.auth.auth_checks import can_customer_access_model, can_key_call_model +from litellm.proxy.offering_router import ( + OfferingAccessGuard, + OfferingRouterView, + OfferingServingSnapshot, + OfferingSnapshotMiddleware, +) +from litellm.router import Router +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.router import Deployment, LiteLLM_Params +from litellm.types.utils import CallTypes + + +def _policy_snapshot(policy: OpenRouterProviderPolicy) -> OfferingServingSnapshot: + native: Final = Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": { + "model": "openrouter/backend", + "extra_body": {"provider": dict(policy.request_body())}, + }, + "model_info": {"id": "selected-id"}, + "provider_policy": policy, + }, + {"model_name": "ordinary", "litellm_params": {"model": "openai/ordinary", "api_key": "fixture-key"}}, + ] + ) + return OfferingServingSnapshot( + native, + frozenset({"selected", "ordinary"}), + MappingProxyType({}), + frozenset({"openrouter/backend", "openai/ordinary"}), + frozenset(native.get_model_ids()), + ) + + +async def test_asgi_stream_pins_authority_until_final_chunk_and_passes_lifespan_through() -> None: + first: Final = OfferingServingSnapshot( + Router(model_list=[]), frozenset({"first"}), MappingProxyType({}), frozenset() + ) + second: Final = OfferingServingSnapshot( + Router(model_list=[]), frozenset({"second"}), MappingProxyType({}), frozenset() + ) + view: Final = OfferingRouterView(first) + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + async def stream(scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "lifespan": + assert view.serving_snapshot() is second + await send({"type": "lifespan.startup.complete"}) + return + assert view.serving_snapshot() is first + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"first", "more_body": True}) + view.publish_snapshot(second) + assert view.serving_snapshot() is first + assert view.latest_snapshot() is second + await send({"type": "http.response.body", "body": b"last", "more_body": False}) + + middleware: Final = OfferingSnapshotMiddleware(stream, lambda: view) + await middleware({"type": "http"}, receive, send) + assert view.serving_snapshot() is second + assert tuple(message["body"] for message in sent if message["type"] == "http.response.body") == (b"first", b"last") + await middleware({"type": "lifespan"}, receive, send) + assert sent[-1] == {"type": "lifespan.startup.complete"} + + +def test_native_mutations_cannot_publish_unmanaged_routes_in_external_mode() -> None: + native: Final = Router(model_list=[]) + view: Final = OfferingRouterView(OfferingServingSnapshot(native, frozenset(), MappingProxyType({}), frozenset())) + with pytest.raises(litellm.BadRequestError, match="external offering configuration"): + view.add_deployment(Deployment(model_name="unmanaged", litellm_params=LiteLLM_Params(model="openai/unmanaged"))) + assert not native.get_model_list() + view.num_retries = 2 + assert native.num_retries == 2 + + +async def test_native_batches_cannot_bypass_offering_selection_with_uploaded_record_models() -> None: + view: Final = OfferingRouterView( + OfferingServingSnapshot(Router(model_list=[]), frozenset({"selected"}), MappingProxyType({}), frozenset()) + ) + guard: Final = OfferingAccessGuard(view) + auth: Final = UserAPIKeyAuth() + cache: Final = DualCache() + for data in ({"model": "selected"}, {"input_file_id": "file-already-uploaded"}): + rejected: Final = await guard.async_pre_call_hook(auth, cache, data, "acreate_batch") + assert isinstance(rejected, litellm.BadRequestError) + assert "batch records" in rejected.message + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "selected", "purpose": "batch"}, "acreate_file"), + litellm.BadRequestError, + ) + assert ( + await guard.async_pre_call_hook(auth, cache, {"model": "selected", "purpose": "assistants"}, "acreate_file") + is None + ) + + +async def test_unavailable_and_unselected_offerings_cannot_forward_or_override_connection() -> None: + router: Final = OfferingRouterView( + OfferingServingSnapshot( + Router(model_list=[]), + frozenset({"selected"}), + MappingProxyType({"missing": "absent from supplier"}), + frozenset({"openai/backend"}), + frozenset({"selected-deployment"}), + ) + ) + guard: Final = OfferingAccessGuard(router) + auth: Final = UserAPIKeyAuth() + cache: Final = DualCache() + assert await guard.async_filter_listed_models(auth, ("selected", "missing", "unselected")) == ("selected",) + assert await guard.async_pre_call_hook(auth, cache, {"model": "selected"}, "acompletion") is None + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "missing"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + assert isinstance( + await guard.async_pre_call_hook(auth, cache, {"model": "unselected"}, "acompletion"), litellm.NotFoundError + ) + assert isinstance( + await guard.async_pre_call_hook( + auth, cache, {"model": "selected", "api_base": "https://other.test"}, "acompletion" + ), + litellm.BadRequestError, + ) + await guard.async_pre_call_deployment_hook( + {"model": "openai/backend", "metadata": {"model_info": {"id": "selected-deployment"}}}, + CallTypes.acompletion, + ) + try: + await guard.async_pre_call_deployment_hook({"model": "openai/unselected"}, CallTypes.acompletion) + except litellm.NotFoundError: + pass + else: + raise AssertionError("An unselected concrete deployment was allowed") + try: + await guard.async_pre_call_deployment_hook( + {"model": "openai/backend", "metadata": {"model_info": {"id": "unavailable-other-connection"}}}, + CallTypes.acompletion, + ) + except litellm.NotFoundError: + pass + else: + raise AssertionError("An unavailable deployment sharing the backend model was allowed") + + +async def test_globally_available_offerings_preserve_customer_and_key_model_permissions() -> None: + native: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": f"openai/{name}", "api_key": "fixture-key"}} + for name in ("allowed", "restricted") + ] + ) + router: Final = OfferingRouterView( + OfferingServingSnapshot( + native, + frozenset({"allowed", "restricted"}), + MappingProxyType({}), + frozenset({"openai/allowed", "openai/restricted"}), + ) + ) + guard: Final = OfferingAccessGuard(router) + key: Final = UserAPIKeyAuth(models=["allowed", "restricted"]) + customer: Final = LiteLLM_EndUserTable(user_id="fixture-customer", blocked=False, models=["allowed"]) + with router.pin_snapshot(): + assert await guard.async_pre_call_hook(key, DualCache(), {"model": "restricted"}, "acompletion") is None + assert await can_key_call_model("restricted", None, key, router) is True + with pytest.raises(ModelAccessDeniedProxyException) as denied_customer: + can_customer_access_model("restricted", customer, router, key) + assert denied_customer.value.code == "403" + assert denied_customer.value.type == ProxyErrorTypes.customer_model_access_denied + assert can_customer_access_model("allowed", customer, router, key) is True + assert await can_key_call_model("allowed", None, key, router) is True + with pytest.raises(ModelAccessDeniedProxyException) as denied_key: + await can_key_call_model("restricted", None, UserAPIKeyAuth(models=["allowed"]), router) + assert denied_key.value.code == "403" + assert denied_key.value.type == ProxyErrorTypes.key_model_access_denied + + +async def test_public_and_router_aliases_follow_offering_availability_without_crossing_teams() -> None: + active: Final = "model_name_team1_active" + missing: Final = "model_name_team1_missing" + other_team: Final = "model_name_team2_active" + native: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "fixture-key"}, + "model_info": {"team_id": team, "team_public_model_name": public}, + } + for name, team, public in ( + (active, "team1", "fast"), + (missing, "team1", "gone"), + (other_team, "team2", "gone"), + ) + ], + model_group_alias={"active-alias": active, "missing-alias": missing}, + ) + view: Final = OfferingRouterView( + OfferingServingSnapshot( + native, frozenset({active, other_team}), MappingProxyType({missing: "absent"}), frozenset() + ) + ) + auth: Final = UserAPIKeyAuth(team_id="team1", aliases={"my-fast": active, "my-gone": missing}) + guard: Final = OfferingAccessGuard(view) + with view.pin_snapshot(): + assert await guard.async_filter_listed_models( + auth, ("fast", "gone", "active-alias", "missing-alias", "my-fast", "my-gone") + ) == ( + "fast", + "active-alias", + "my-fast", + ) + assert await guard.async_pre_call_hook(auth, DualCache(), {"model": "fast"}, "acompletion") is None + assert isinstance( + await guard.async_pre_call_hook(auth, DualCache(), {"model": "gone"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + assert isinstance( + await guard.async_pre_call_hook(auth, DualCache(), {"model": "missing-alias"}, "acompletion"), + litellm.ServiceUnavailableError, + ) + admin: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + assert await guard.async_filter_listed_models(admin, ("fast",)) == ("fast",) + legacy: Final = OfferingAccessGuard(view, {"use_team_public_model_name": False}) + assert await legacy.async_filter_listed_models(auth, ("fast", active)) == (active,) + + +async def test_body_model_override_cannot_bypass_offering_authority() -> None: + view: Final = OfferingRouterView( + OfferingServingSnapshot(Router(model_list=[]), frozenset({"selected"}), MappingProxyType({}), frozenset()) + ) + guard: Final = OfferingAccessGuard(view) + assert guard.message_logging is True + assert guard.turn_off_message_logging is False + auth: Final = UserAPIKeyAuth() + for call_type in ("acompletion", "aresponses"): + assert isinstance( + await guard.async_pre_call_hook( + auth, DualCache(), {"model": "selected", "extra_body": {"model": "unselected"}}, call_type + ), + litellm.BadRequestError, + ) + assert ( + await guard.async_pre_call_hook( + auth, DualCache(), {"model": "selected", "extra_body": {"metadata": {"fixture": "allowed"}}}, call_type + ) + is None + ) + + +@pytest.mark.parametrize( + "override", + ( + {"provider_policy": {"provider": "openrouter", "only": ["other"]}}, + {"extra_body": {"provider_policy": {"provider": "openrouter", "only": ["other"]}}}, + {"provider": {"only": ["other"]}}, + {"service_tier": "priority"}, + {"extra_body": {"service_tier": "flex"}}, + {"extra_body": {"provider": {"allow_fallbacks": True}}}, + {"extra_body": {"extra_body": {"provider": {"only": ["other"]}}}}, + {"models": ["other"]}, + {"route": "fallback"}, + {"extra_body": {"models": ["other"]}}, + {"extra_body": {"route": "fallback"}}, + {"fallbacks": [{"selected": ["other"]}]}, + {"context_window_fallbacks": [{"selected": ["other"]}]}, + {"content_policy_fallbacks": [{"selected": ["other"]}]}, + {"specific_deployment": True}, + {"model_id": "unrestricted"}, + {"disable_fallbacks": False}, + {"mock_response": "bypassed"}, + {"additional_drop_params": ["provider"]}, + {"additional_drop_params": ["extra_body"]}, + {"additional_drop_params": ["extra_body.provider"]}, + {"additional_drop_params": ["provider.only"]}, + {"additional_drop_params": ["provider.allow_fallbacks"]}, + {"drop_params": True}, + {"allowed_openai_params": ["provider"]}, + ), +) +async def test_pinned_policy_rejects_caller_routing_even_through_alias(override: dict[str, object]) -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView(_policy_snapshot(policy)) + guard: Final = OfferingAccessGuard(view) + auth: Final = UserAPIKeyAuth(aliases={"alias": "selected"}) + for call_type in ("acompletion", "aresponses", "anthropic_messages"): + rejected: Final = await guard.async_pre_call_hook(auth, DualCache(), {"model": "alias", **override}, call_type) + assert isinstance(rejected, litellm.BadRequestError) + assert "caller" in rejected.message + + +async def test_pinned_policy_shapes_requests_without_mutation_and_blocks_unsupported_apis() -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView(_policy_snapshot(policy)) + guard: Final = OfferingAccessGuard(view) + original: Final = {"model": "selected", "extra_body": {"temperature": 0.5}} + assert await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), original, "acompletion") == { + "model": "selected", + "extra_body": {"temperature": 0.5, "provider": {"only": ["synthetic/fp8"], "allow_fallbacks": False}}, + "disable_fallbacks": True, + "caching": False, + "cache": {"no-cache": True, "no-store": True}, + } + assert original == {"model": "selected", "extra_body": {"temperature": 0.5}} + assert ( + await guard.async_pre_call_hook( + UserAPIKeyAuth(), DualCache(), {"model": "ordinary", "provider": {"only": ["other"]}}, "acompletion" + ) + is None + ) + assert isinstance( + await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "selected"}, "aembedding"), + litellm.BadRequestError, + ) + assert isinstance( + await guard.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": "openrouter/backend"}, "acompletion"), + litellm.NotFoundError, + ) + + +@pytest.mark.parametrize("call_type", (None, CallTypes.aresponses_websocket, CallTypes.aimage_generation)) +async def test_pinned_deployment_rejects_apis_that_cannot_carry_policy(call_type: CallTypes | None) -> None: + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + view: Final = OfferingRouterView(_policy_snapshot(policy)) + with pytest.raises(litellm.BadRequestError, match="configured offering provider policy"): + await OfferingAccessGuard(view).async_pre_call_deployment_hook( + { + "model": "openrouter/backend", + "metadata": {"model_info": {"id": "selected-id"}}, + "extra_body": {"provider": dict(policy.request_body())}, + }, + call_type, + ) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index b5940e4c212..fb3fb1f4b11 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -4002,3 +4002,70 @@ def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog) assert provider_result.code == internal_result.code == "500" assert ISSUE_URL_BASE in caplog.text assert ISSUE_URL_BASE not in internal_result.message + + +def test_model_listing_advertises_complete_configured_metadata_and_no_secrets(): + metadata: Final = { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "xhigh", "max"], + "default_reasoning_effort": "xhigh", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "request_defaults": { + "output_token_budget": 100, + "output_token_budget_by_reasoning_effort": {"xhigh": 150}, + }, + } + router: Final = litellm.Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-test-model", "api_key": "private-fixture-key"}, + "model_info": {**metadata, "internal_secret": "must-not-advertise"}, + } + ] + ) + response: Final = create_model_info_response( + "selected", "openai", llm_router=router, get_model_info=_raise_unmapped + ) + assert response == { + "id": "selected", + "object": "model", + "created": response["created"], + "owned_by": "openai", + **metadata, + } + + +def test_model_listing_context_contract_uses_minimum_and_omits_unknown_group_limits(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-one"}, + "model_info": {"context_window": 1200, "max_input_tokens": 1100, "max_output_tokens": 200}, + }, + { + "model_name": "selected", + "litellm_params": {"model": "openai/unknown-two"}, + "model_info": {"context_window": 800, "max_output_tokens": 100}, + }, + ] + ) + response: Final = create_model_info_response( + "selected", "openai", llm_router=router, get_model_info=_raise_unmapped + ) + assert response == { + "id": "selected", + "object": "model", + "created": response["created"], + "owned_by": "openai", + "context_window": 800, + "max_output_tokens": 100, + } diff --git a/tests/unit/router_utils/test_model_request_defaults.py b/tests/unit/router_utils/test_model_request_defaults.py new file mode 100644 index 00000000000..5b0fb8468b0 --- /dev/null +++ b/tests/unit/router_utils/test_model_request_defaults.py @@ -0,0 +1,107 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.proxy.model_discovery import discover_model_metadata +from litellm.router_utils.model_request_defaults import deployment_params_with_request_defaults, model_request_defaults +from litellm.types.router import ModelMetadata, ModelRequestDefaults + +_POLICY: Final = { + "output_token_budget": 8192, + "output_token_budget_by_reasoning_effort": {"low": 65536, "high": 65536, "xhigh": 65536, "max": 131072}, +} + + +@pytest.mark.parametrize( + "effort,expected", [(None, 8192), ("off", 8192), ("low", 65536), ("high", 65536), ("xhigh", 65536), ("max", 131072)] +) +def test_request_budget_follows_effective_reasoning_without_claiming_supplier_limit(effort, expected): + metadata: Final = {"context_window": 262144, "request_defaults": _POLICY} + request: Final = {"reasoning_effort": effort} + assert model_request_defaults(metadata, {}, request, "completion") == {"max_tokens": expected} + assert "max_output_tokens" not in discover_model_metadata((ModelMetadata.model_validate(metadata),)) + assert request == {"reasoning_effort": effort} + + +@pytest.mark.parametrize("key", ["max_tokens", "max_completion_tokens", "max_output_tokens"]) +def test_explicit_caller_budget_wins(key): + assert ( + model_request_defaults({"request_defaults": _POLICY}, {}, {key: 123, "reasoning_effort": "max"}, "completion") + == {} + ) + + +@pytest.mark.parametrize( + "surface,key", + [ + ("completion", "max_tokens"), + ("acompletion", "max_tokens"), + ("responses", "max_output_tokens"), + ("aresponses", "max_output_tokens"), + ("anthropic_messages", "max_tokens"), + ], +) +def test_budget_uses_endpoint_parameter_and_request_reasoning_overrides_deployment(surface, key): + assert model_request_defaults( + {"request_defaults": _POLICY}, + {"reasoning_effort": "max", "max_tokens": 456}, + {"reasoning": {"effort": "low"}}, + surface, + ) == {key: 65536} + + +def test_deployment_and_metadata_default_efforts_are_resolved(): + metadata: Final = {"request_defaults": _POLICY, "default_reasoning_effort": "high"} + assert model_request_defaults(metadata, {}, {}, "completion") == {"max_tokens": 65536} + assert model_request_defaults(metadata, {"reasoning": {"effort": "max"}}, {}, "responses") == { + "max_output_tokens": 131072 + } + assert model_request_defaults(metadata, {}, {}, "aembedding") == {} + assert model_request_defaults({}, {}, {}, "completion") == {} + assert model_request_defaults(metadata, {}, {"reasoning_effort": 50}, "completion") == {"max_tokens": 8192} + + +def test_selected_policy_replaces_competing_static_budget_aliases(): + deployment: Final = {"model": "openai/fixture", "max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3} + defaults: Final = model_request_defaults( + {"request_defaults": _POLICY}, deployment, {"reasoning_effort": "max"}, "responses" + ) + params: Final = deployment_params_with_request_defaults(deployment, defaults) + assert {**params, **defaults} == {"model": "openai/fixture", "max_output_tokens": 131072} + assert deployment_params_with_request_defaults(deployment, {}) == deployment + assert deployment_params_with_request_defaults( + deployment, {}, {"request_defaults": _POLICY}, {"max_tokens": 123} + ) == {"model": "openai/fixture"} + assert deployment["max_completion_tokens"] == 2 + + +@pytest.mark.parametrize("value", [0, -1, True, "8192", 1.5]) +@pytest.mark.parametrize("field", ["output_token_budget", "output_token_budget_by_reasoning_effort"]) +def test_invalid_request_budgets_are_rejected(value, field): + invalid: Final = {field: {"high": value} if field.endswith("effort") else value} + with pytest.raises(ValidationError): + ModelRequestDefaults.model_validate(invalid) + + +def test_request_budget_maps_are_immutable_and_only_common_policy_is_advertised(): + metadata: Final = ModelMetadata.model_validate({"request_defaults": _POLICY}) + assert discover_model_metadata((metadata, metadata)) == {"request_defaults": _POLICY} + assert discover_model_metadata((metadata, ModelMetadata())) == {} + other: Final = ModelMetadata(request_defaults=ModelRequestDefaults(output_token_budget=16384)) + assert discover_model_metadata((metadata, other)) == {} + defaults: Final = metadata.request_defaults + assert defaults is not None + with pytest.raises(TypeError): + defaults.output_token_budget_by_reasoning_effort["max"] = 1 + + +@pytest.mark.parametrize("surface,key", [("acompletion", "max_tokens"), ("aresponses", "max_output_tokens")]) +def test_structured_reasoning_effort_preserves_summary_and_caller_precedence(surface, key): + metadata: Final = {"request_defaults": _POLICY} + deployment: Final = {"reasoning_effort": {"effort": "max", "summary": "auto"}} + request: Final = {"reasoning_effort": {"effort": "high", "summary": "auto"}} + assert model_request_defaults(metadata, deployment, request, surface) == {key: 65536} + assert model_request_defaults(metadata, deployment, {}, surface) == {key: 131072} + assert request == {"reasoning_effort": {"effort": "high", "summary": "auto"}} + assert deployment == {"reasoning_effort": {"effort": "max", "summary": "auto"}} diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 60a75c688c4..e04a227f5c6 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -53,6 +53,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/litellm_core_utils/specialty_caches/service_trace_id_cache.py", "litellm/llms/azure/common_utils.py", "litellm/llms/bedrock/base_aws_llm.py", + "litellm/llms/chatgpt/model_info.py", "litellm/llms/custom_httpx/http_handler.py", "litellm/llms/databricks/agent/transformation.py", "litellm/llms/gigachat/authenticator.py", @@ -60,6 +61,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/llms/litellm_proxy/skills/handler.py", "litellm/llms/openai/common_utils.py", "litellm/llms/openai_like/model_info.py", + "litellm/llms/openrouter/model_info.py", "litellm/llms/vertex_ai/vertex_ai_non_gemini.py", "litellm/llms/watsonx/common_utils.py", "litellm/proxy/_experimental/mcp_server/byok_credential_cache.py", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index c0abb18ca8c..13e9bd38a5c 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -736,6 +736,36 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] +def test_native_model_info_and_lookup_preserve_reasoning_capabilities_and_request_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.types.router import ModelInfo + + metadata: Final = { + "context_window": 1200, + "max_input_tokens": 1100, + "max_output_tokens": 200, + "max_tokens": 200, + "supports_function_calling": True, + "supports_parallel_function_calling": False, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "max"], + "default_reasoning_effort": "low", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "request_defaults": { + "output_token_budget": 100, + "output_token_budget_by_reasoning_effort": {"max": 180}, + }, + } + native: Final = ModelInfo(**metadata) + assert native.request_defaults.output_budget("max") == 180 + monkeypatch.setattr(litellm, "model_cost", {"fixture-model": {"litellm_provider": "openai", **metadata}}) + info: Final = litellm.get_model_info("fixture-model") + assert {field: info.get(field) for field in metadata} == metadata + + def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) return MappingProxyType( diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f001008ee2c..015b9cffc46 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -18,6 +18,7 @@ import queue import re import sys import threading +from collections.abc import Mapping from pathlib import Path from types import MappingProxyType from typing import Final, cast @@ -31,8 +32,8 @@ import litellm from litellm import Router from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE -from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.types.router import ( @@ -105,7 +106,19 @@ async def test_discovered_limits_survive_deployment_growth_and_removal( await handler.client.aclose() async with httpx.AsyncClient( transport=httpx.MockTransport( - lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + lambda request: httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ) ) ) as client: handler.client = client @@ -158,9 +171,28 @@ async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch model_info=ModelInfo(id="replaced-deployment"), ) ) - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + } + ] + }, + ) assert request.url.host == "replacement.test" - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -225,7 +257,14 @@ async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: if request.url.host == "unavailable.test": return httpx.Response(503) limit: Final = 8192 if request.url.host == "first.test" else 2048 - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": limit}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": limit, "max_input_tokens": limit, "max_output_tokens": limit} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -279,7 +318,14 @@ async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypa if request.url.host == "second.test": second_started.set() return httpx.Response(503) - return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + return httpx.Response( + 200, + json={ + "data": [ + {"id": "local-model", "max_model_len": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048} + ] + }, + ) handler: Final = AsyncHTTPHandler() await handler.client.aclose() @@ -311,8 +357,32 @@ async def test_discovered_limits_expire_after_the_last_successful_refresh(monkey router._discovered_model_info_cache = InMemoryCache(clock=clock, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS) responses: Final = iter( ( - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}), + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ), + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + } + ] + }, + ), httpx.Response(503), ) ) @@ -350,7 +420,20 @@ async def test_discovered_limits_are_isolated_overridable_and_refreshable( def respond(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/models" assert request.headers["authorization"] == "Bearer local-key" - return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": next(upstream_limit)}]}) + limit: Final = next(upstream_limit) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "org/local-model", + "max_model_len": limit, + "max_input_tokens": limit, + "max_output_tokens": limit, + } + ] + }, + ) router: Final = Router( model_list=[ @@ -406,7 +489,19 @@ async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypa monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) responses: Final = iter( ( - httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), + httpx.Response( + 200, + json={ + "data": [ + { + "id": "local-model", + "max_model_len": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + } + ] + }, + ), httpx.Response(503), ) ) @@ -3934,6 +4029,61 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info( assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price +@pytest.mark.parametrize( + "params", + ( + LiteLLM_Params(model="openai/*", api_base="https://fixture.test/v1"), + LiteLLM_Params(model="openai/fixture", api_base="https://fixture.test/v1", use_clientside_credentials=True), + LiteLLM_Params(model="anthropic/claude-sonnet-4-5"), + ), +) +async def test_deployment_metadata_skips_unresolved_or_unsupported_supplier_connections(params: LiteLLM_Params) -> None: + router: Final = Router(model_list=[]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + + def unexpected(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"No catalog request expected for {request.url.path}") + + async with httpx.AsyncClient(transport=httpx.MockTransport(unexpected)) as http_client: + handler.client = http_client + assert await router._aget_deployment_model_metadata(params, client=handler) == {} + + +async def test_deployment_metadata_uses_resolved_catalog_credentials_and_independent_limits() -> None: + router: Final = Router(model_list=[]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + paths: Final[list[str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + paths.append(request.url.path) + assert request.headers["authorization"] == "Bearer operator-override" + assert request.headers["x-provider-route"] == "fixture" + return httpx.Response( + 200, + json={ + "data": [{"id": "fixture", "context_window": 1200, "max_input_tokens": 1100, "max_output_tokens": 200}] + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler.client = http_client + metadata: Final = await router._aget_deployment_model_metadata( + LiteLLM_Params( + model="openai/fixture", + api_base="https://fixture.test/v1", + api_key="default-key", + extra_headers={"Authorization": "Bearer operator-override", "x-provider-route": "fixture"}, + ), + client=handler, + ) + assert paths == ["/v1/models"] + assert metadata["context_window"] == 1200 + assert metadata["max_input_tokens"] == 1100 + assert metadata["max_output_tokens"] == 200 + + def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") catalog: Final = litellm.get_model_cost_map(url="") diff --git a/tests/unit/types/proxy/test_model_offerings.py b/tests/unit/types/proxy/test_model_offerings.py new file mode 100644 index 00000000000..9b791caaec4 --- /dev/null +++ b/tests/unit/types/proxy/test_model_offerings.py @@ -0,0 +1,93 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.types.llms.openrouter import OpenRouterProviderPolicy +from litellm.types.proxy.model_offerings import ModelOfferingsConfig + + +@pytest.mark.parametrize( + "change", + ( + {"version": True}, + {"config_poll_seconds": 0}, + {"inventory_poll_seconds": "300"}, + { + "offerings": [ + {"model_name": "duplicate", "source": "manual", "provider": "supplier", "upstream_model": "backend"} + ] + * 2 + }, + { + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "backend", + "model_info": {"context_window": -1}, + } + ] + }, + {"providers": {"supplier": {"provider": "chatgpt", "api_key": "ignored-key"}}}, + {"providers": {"supplier": {"provider": "openai"}}}, + {"providers": {" supplier ": {"provider": "openrouter"}}}, + {"offerings": [{"model_name": "*", "source": "manual", "provider": "supplier", "upstream_model": "backend"}]}, + { + "offerings": [ + {"model_name": "selected", "source": "manual", "provider": "missing", "upstream_model": "backend"} + ] + }, + ), +) +def test_invalid_operator_edits_are_rejected(change) -> None: + config: Final = { + "version": 1, + "providers": {"supplier": {"provider": "openai", "api_base": "https://supplier.test/v1"}}, + "offerings": [], + } + with pytest.raises(ValidationError): + ModelOfferingsConfig.model_validate({**config, **change}) + + +@pytest.mark.parametrize( + "change", + ( + {"provider": "openai"}, + {"only": []}, + {"only": ["endpoint", "endpoint"]}, + {"only": ["endpoint "]}, + {"only": ["https://supplier.test"]}, + {"only": ["supplier/*"]}, + {"allow_fallbacks": True}, + {"allow_fallbacks": 0}, + {"api_key": "not-allowed"}, + ), +) +def test_provider_policy_rejects_nonexact_or_unsafe_configuration(change: dict[str, object]) -> None: + with pytest.raises(ValidationError): + OpenRouterProviderPolicy.model_validate( + {"provider": "openrouter", "only": ["synthetic/fp8"], "allow_fallbacks": False, **change} + ) + + +@pytest.mark.parametrize("supplier,mode", (("openai", "chat"), ("openrouter", None), ("openrouter", "embedding"))) +def test_provider_policy_requires_supported_supplier_and_api(supplier: str, mode: str | None) -> None: + with pytest.raises(ValidationError, match="matching supplier and explicit chat mode"): + ModelOfferingsConfig.model_validate( + { + "version": 1, + "providers": {"supplier": {"provider": supplier, "api_base": "https://supplier.test/v1"}}, + "offerings": [ + { + "model_name": "selected", + "source": "manual", + "provider": "supplier", + "upstream_model": "backend", + "model_info": {"mode": mode}, + "provider_policy": {"provider": "openrouter", "only": ["synthetic/fp8"]}, + } + ], + } + ) diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 2892a829eab..54e41b729ba 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -30,6 +30,67 @@ from litellm.types.utils import ( ) +@pytest.mark.parametrize("value,expected", (("1200", 1200), (1.5, 1), (0, 0), (-1, -1), (True, None), ("bad", None))) +def test_native_model_info_preserves_legacy_limits_without_a_context_contract( + value: object, expected: int | None +) -> None: + info: Final = ModelInfo(max_input_tokens=value) + assert info.max_input_tokens == expected + + +@pytest.mark.parametrize("value", ("1200", 1.5, 0, -1, True, "bad")) +def test_native_context_contract_does_not_invent_invalid_input_capacity(value: object) -> None: + info: Final = ModelInfo(context_window=2000, max_input_tokens=value) + assert info.context_window == 2000 + assert info.max_input_tokens is None + + +def test_native_deployment_compiles_private_policy_and_keeps_it_separate_from_mutable_request_body() -> None: + from litellm.router import Router + from litellm.types.llms.openrouter import OpenRouterProviderPolicy + + policy: Final = OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)) + native: Final = Router( + model_list=[ + { + "model_name": "public-model", + "litellm_params": {"model": "openrouter/org/backend", "extra_body": {"temperature": 0.5}}, + "model_info": {"id": "private-deployment"}, + "provider_policy": policy, + } + ] + ) + deployment: Final = native.get_deployment("private-deployment") + assert deployment is not None + assert deployment.provider_policy == policy + assert deployment.litellm_params.extra_body == { + "temperature": 0.5, + "provider": {"only": ["synthetic/fp8"], "allow_fallbacks": False}, + } + assert "provider_policy" not in deployment.model_dump() + assert "provider_policy" not in deployment.model_info.model_dump() + deployment.litellm_params.extra_body["provider"]["only"].append("caller/override") + assert native.get_deployment("private-deployment").provider_policy.only == ("synthetic/fp8",) + + +@pytest.mark.parametrize( + "model,body", + ( + ("openai/org/backend", {}), + ("openrouter/org/backend", {"provider": {"only": ["other"], "allow_fallbacks": True}}), + ), +) +def test_native_deployment_rejects_policy_provider_or_body_conflicts(model: str, body: dict[str, object]) -> None: + from litellm.types.llms.openrouter import OpenRouterProviderPolicy + + with pytest.raises(ValidationError, match=r"provider|routing body"): + Deployment( + model_name="public-model", + litellm_params=LiteLLM_Params(model=model, extra_body=body), + provider_policy=OpenRouterProviderPolicy(provider="openrouter", only=("synthetic/fp8",)), + ) + + def test_sdk_timeout_is_normalized_for_provider_clients() -> None: timeout: Final = SDKTimeout(connect=2.0, read=None, write=5.0, pool=7.0) params: Final = GenericLiteLLMParams(timeout=timeout) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 262e8b6deae..451bb9600e7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30205,6 +30205,11 @@ export interface components { * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. */ model_list_healthy_only?: boolean | null; + /** + * Model Offerings Path + * @description Path to the authoritative external offering file, reloaded without restarting the proxy + */ + model_offerings_path?: string | null; /** * Otel * @description [BETA] OpenTelemetry support - this might change, use with caution. @@ -31799,6 +31804,8 @@ export interface components { model_info: components["schemas"]["litellm__types__router__ModelInfo"]; /** Model Name */ model_name: string; + /** @description Gateway-owned supplier selection, excluded from public model metadata */ + provider_policy?: components["schemas"]["OpenRouterProviderPolicy"] | null; } & { [key: string]: unknown; }; @@ -38600,6 +38607,21 @@ export interface components { /** Model Name */ model_name: string; }; + /** ModelRequestDefaults */ + ModelRequestDefaults: { + /** + * Output Token Budget + * @description Default requested output budget, not a physical model limit + */ + output_token_budget?: number | null; + /** + * Output Token Budget By Reasoning Effort + * @description Requested output budgets keyed by exact reasoning effort + */ + output_token_budget_by_reasoning_effort?: { + [key: string]: number; + } | null; + }; /** ModelResponse */ ModelResponse: { /** Choices */ @@ -39867,6 +39889,22 @@ export interface components { */ type: "openIdConnect"; }; + /** OpenRouterProviderPolicy */ + OpenRouterProviderPolicy: { + /** + * Allow Fallbacks + * @default false + * @constant + */ + allow_fallbacks: false; + /** Only */ + only: string[]; + /** + * Provider + * @constant + */ + provider: "openrouter"; + }; /** OpenSourceClassifierConfig */ OpenSourceClassifierConfig: { /** @@ -50210,6 +50248,16 @@ export interface components { litellm__proxy___types__ModelInfo: { /** Base Model */ base_model: ("gpt-4-1106-preview" | "gpt-4-32k" | "gpt-4" | "gpt-3.5-turbo-16k" | "gpt-3.5-turbo" | "text-embedding-ada-002") | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Id */ @@ -50219,6 +50267,16 @@ export interface components { * @default 0 */ input_cost_per_token: number | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Max Tokens * @default 2048 @@ -50231,6 +50289,43 @@ export interface components { * @default 0 */ output_cost_per_token: number | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** + * Supported Modalities + * @description Accepted input modalities + */ + supported_modalities?: string[] | null; + /** + * Supported Output Modalities + * @description Supported output modalities + */ + supported_output_modalities?: string[] | null; + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ + supports_function_calling?: boolean | null; + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ + supports_parallel_function_calling?: boolean | null; + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ + supports_reasoning?: boolean | null; } & { [key: string]: unknown; }; @@ -50281,6 +50376,11 @@ export interface components { cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; + /** + * Context Window + * @description Maximum combined input and generated output tokens + */ + context_window?: number | null; /** Cost Per Ptu Per Hour */ cost_per_ptu_per_hour?: number | null; /** Created At */ @@ -50292,6 +50392,11 @@ export interface components { * @default false */ db_model: boolean; + /** + * Default Reasoning Effort + * @description Effort used when the request omits it + */ + default_reasoning_effort?: string | null; /** Discoverable */ discoverable?: boolean | null; /** Enable Tag Filtering */ @@ -50308,6 +50413,16 @@ export interface components { internal_router_model?: boolean | null; /** Max Embedding Batch Size */ max_embedding_batch_size?: number | null; + /** + * Max Input Tokens + * @description Independent maximum input tokens + */ + max_input_tokens?: number | null; + /** + * Max Output Tokens + * @description Independent maximum generated output tokens + */ + max_output_tokens?: number | null; /** * Member Auto Router * @default false @@ -50329,6 +50444,43 @@ export interface components { ptu_shares?: { [key: string]: number; } | null; + /** + * Reasoning Effort Levels + * @description Exact effort values this route accepts + */ + reasoning_effort_levels?: string[] | null; + /** @description Configured request defaults, independent of supplier capability limits */ + request_defaults?: components["schemas"]["ModelRequestDefaults"] | null; + /** + * Supported Endpoints + * @description Inference endpoints exposed by this route + */ + supported_endpoints?: string[] | null; + /** + * Supported Modalities + * @description Accepted input modalities + */ + supported_modalities?: string[] | null; + /** + * Supported Output Modalities + * @description Supported output modalities + */ + supported_output_modalities?: string[] | null; + /** + * Supports Function Calling + * @description Whether this model accepts tool calls + */ + supports_function_calling?: boolean | null; + /** + * Supports Parallel Function Calling + * @description Whether this model accepts parallel tool calls + */ + supports_parallel_function_calling?: boolean | null; + /** + * Supports Reasoning + * @description Whether this model supports reasoning controls + */ + supports_reasoning?: boolean | null; /** Team Id */ team_id?: string | null; /** Team Public Model Name */ diff --git a/uv.lock b/uv.lock index 531082730bb..f2b621d6f8d 100644 --- a/uv.lock +++ b/uv.lock @@ -2126,11 +2126,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.32.7" +version = "3.20.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, + { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] [[package]] @@ -4928,7 +4928,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, - { name = "filelock", specifier = ">=3.16.1,<4.0" }, + { name = "filelock", specifier = ">=3.16.1,<3.20.4" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 81e9425ef4e..16df05b41a2 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -189,19 +189,29 @@ } }, "node_modules/@azure/msal-node": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-6.0.1.tgz", - "integrity": "sha512-ixSO1Y/kCVRthRs+hSx/5qkwaunX1/RAePhlMN0wIpIQ4WEZ6AREGGnGd1AP0qspHVsAwzWQKGubpjh50JyJIQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-7.1.1.tgz", + "integrity": "sha512-zHXTJB38XVrsNaXrPBCMqEUjeyQysFfgmKtUHRDsEKTRjNIPckJ/w6fE5wBXrmJXBim1D3C7O7LOag0msPTrgw==", "dev": true, "license": "MIT", "dependencies": { - "@azure/msal-common": "16.14.1", + "@azure/msal-common": "16.14.3", "jsonwebtoken": "^9.0.0" }, "engines": { "node": ">=20" } }, + "node_modules/@azure/msal-node/node_modules/@azure/msal-common": { + "version": "16.14.3", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.14.3.tgz", + "integrity": "sha512-tz3GvMrR9V/plb2lDdW8qaO8CPpfZevdExBKjltzzbAtyhK0eB2UIvji1aU+gkyHj3fxQ2ie+LNHyMXdmo8wOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..0fbe7ad0da2 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -83,5 +83,10 @@ "esbuild": "^0.28.2", "typescript": "^5.9.3", "vitest": "^4.1.12" + }, + "overrides": { + "@azure/identity": { + "@azure/msal-node": "^7.1.0" + } } }