mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
chore: merge staging into Rust OCR cutover
Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>
This commit is contained in:
Devin AI
commit
5f42f3f50e
848 changed files with 18557 additions and 6950 deletions
|
|
@ -24,9 +24,12 @@ jobs:
|
|||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
|
||||
- name: Regenerate JSON Schema
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json
|
||||
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
|
|
|
|||
9
.github/workflows/test-model-map.yaml
vendored
9
.github/workflows/test-model-map.yaml
vendored
|
|
@ -22,3 +22,12 @@ jobs:
|
|||
- name: Validate model_prices_and_context_window.json
|
||||
run: |
|
||||
jq empty model_prices_and_context_window.json
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Check model_prices_and_context_window.schema.json is in sync
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py --check
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -141,3 +141,4 @@ crash.*.log
|
|||
.coverage
|
||||
|
||||
ui/litellm-dashboard/out/
|
||||
litellm.log
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"reportAny": {
|
||||
"limit": 34906
|
||||
"limit": 33216
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2701
|
||||
"limit": 2648
|
||||
},
|
||||
"reportAssignmentType": {
|
||||
"limit": 330
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 42
|
||||
},
|
||||
"reportExplicitAny": {
|
||||
"limit": 10230
|
||||
"limit": 10228
|
||||
},
|
||||
"reportFunctionMemberAccess": {
|
||||
"limit": 11
|
||||
|
|
@ -99,7 +99,7 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 45870
|
||||
"limit": 45567
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 113
|
||||
|
|
|
|||
325
ci_cd/generate_model_prices_schema.py
Normal file
325
ci_cd/generate_model_prices_schema.py
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import jsonschema
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
||||
JsonSchema = dict
|
||||
|
||||
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
|
||||
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
|
||||
BOOLEAN: JsonSchema = {"type": "boolean"}
|
||||
STRING: JsonSchema = {"type": "string"}
|
||||
|
||||
EXTRA_BOOLEAN_KEYS = frozenset(
|
||||
{
|
||||
"gemini_native_audio",
|
||||
"gemini_audio_only_live",
|
||||
"uses_embed_content",
|
||||
"use_openai_responses_path",
|
||||
"bedrock_converse_supports_strict_tools",
|
||||
}
|
||||
)
|
||||
|
||||
OBJECT_KEYS: dict[str, JsonSchema] = {
|
||||
"search_context_cost_per_query": {
|
||||
"type": "object",
|
||||
"description": "USD cost per web search query, keyed by search context size.",
|
||||
"properties": {
|
||||
"search_context_size_low": NONNEG_NUMBER,
|
||||
"search_context_size_medium": NONNEG_NUMBER,
|
||||
"search_context_size_high": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Free-form notes about the entry (e.g. pricing derivation).",
|
||||
},
|
||||
"provider_specific_entry": {
|
||||
"type": "object",
|
||||
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
|
||||
},
|
||||
}
|
||||
|
||||
ARRAY_KEYS: dict[str, JsonSchema] = {
|
||||
"supported_endpoints": {
|
||||
"type": "array",
|
||||
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
|
||||
"items": STRING,
|
||||
},
|
||||
"supported_modalities": {
|
||||
"type": "array",
|
||||
"description": "Input modalities the model accepts.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
|
||||
},
|
||||
"supported_output_modalities": {
|
||||
"type": "array",
|
||||
"description": "Output modalities the model can produce.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
|
||||
},
|
||||
"supported_regions": {
|
||||
"type": "array",
|
||||
"description": "Cloud regions the model is available in ('global' or region ids).",
|
||||
"items": STRING,
|
||||
},
|
||||
"tiered_pricing": {
|
||||
"type": "array",
|
||||
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] prompt-token span this tier applies to.",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"max_results_range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] result-count span this tier applies to (search models).",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"input_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_reasoning_token": NONNEG_NUMBER,
|
||||
"cache_read_input_token_cost": NONNEG_NUMBER,
|
||||
"input_cost_per_query": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
INTEGER_KEYS: dict[str, JsonSchema] = {
|
||||
"max_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
|
||||
},
|
||||
"max_input_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum prompt/context tokens the model accepts.",
|
||||
},
|
||||
"max_output_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum tokens the model can generate in one response.",
|
||||
},
|
||||
"output_vector_size": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Embedding dimension for embedding models.",
|
||||
},
|
||||
"prompt_cache_min_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
|
||||
},
|
||||
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
|
||||
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
|
||||
}
|
||||
|
||||
NUMBER_KEYS: dict[str, JsonSchema] = {
|
||||
"regional_processing_uplift_multiplier_eu": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
"regional_processing_uplift_multiplier_us": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
}
|
||||
|
||||
COST_DESCRIPTIONS: dict[str, str] = {
|
||||
"input_cost_per_token": "USD per prompt token.",
|
||||
"output_cost_per_token": "USD per generated token.",
|
||||
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
|
||||
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
|
||||
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
|
||||
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
|
||||
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
|
||||
}
|
||||
|
||||
|
||||
def cost_description(key: str) -> Optional[str]:
|
||||
if key in COST_DESCRIPTIONS:
|
||||
return COST_DESCRIPTIONS[key]
|
||||
if key.endswith("_flex"):
|
||||
return "Flex service-tier rate for the same-named base field."
|
||||
if key.endswith("_priority"):
|
||||
return "Priority service-tier rate for the same-named base field."
|
||||
if "_above_" in key:
|
||||
return "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
return None
|
||||
|
||||
|
||||
def cost_schema(key: str) -> JsonSchema:
|
||||
description = cost_description(key)
|
||||
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
|
||||
|
||||
|
||||
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
|
||||
return {
|
||||
"litellm_provider": {
|
||||
"type": "string",
|
||||
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"description": "Primary API surface / task type of the model.",
|
||||
"enum": list(modes),
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "URL of the provider pricing/model page this entry was taken from.",
|
||||
},
|
||||
"deprecation_date": {
|
||||
"type": "string",
|
||||
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
|
||||
"format": "date",
|
||||
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
|
||||
},
|
||||
"web_search_billing_unit": {
|
||||
"type": "string",
|
||||
"description": "Whether web search is billed per query or per prompt.",
|
||||
"enum": ["per_query", "per_prompt"],
|
||||
},
|
||||
"bedrock_output_config_effort_ceiling": {
|
||||
"type": "string",
|
||||
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
|
||||
"enum": ["low", "medium", "high", "max", "xhigh"],
|
||||
},
|
||||
"comment": STRING,
|
||||
"audio_transcription_config": STRING,
|
||||
}
|
||||
|
||||
|
||||
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
|
||||
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
|
||||
if key in curated:
|
||||
return curated[key]
|
||||
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
|
||||
return BOOLEAN
|
||||
if "cost" in key:
|
||||
return cost_schema(key)
|
||||
return None
|
||||
|
||||
|
||||
def build_schema(prices: dict) -> JsonSchema:
|
||||
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
|
||||
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
|
||||
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
|
||||
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
|
||||
if unclassified:
|
||||
raise SystemExit(
|
||||
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
|
||||
f"Add them to the key tables in {Path(__file__).name} and rerun it."
|
||||
)
|
||||
entry_properties = {key: classify(key, modes) for key in all_keys}
|
||||
return {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "LiteLLM model_prices_and_context_window.json",
|
||||
"description": (
|
||||
"Schema for LiteLLM's model price and context window registry "
|
||||
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
|
||||
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
|
||||
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
|
||||
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
|
||||
"ignore unknown fields rather than reject them."
|
||||
),
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Documentation placeholder illustrating the entry shape; not a real model and not "
|
||||
"schema-conformant (several values are prose)."
|
||||
),
|
||||
},
|
||||
"fallback_generalizations": {
|
||||
"type": "object",
|
||||
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
|
||||
"properties": {
|
||||
"rules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": STRING,
|
||||
"pattern": STRING,
|
||||
"description": STRING,
|
||||
},
|
||||
"required": ["name", "pattern"],
|
||||
"additionalProperties": True,
|
||||
},
|
||||
}
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
|
||||
"$defs": {
|
||||
"modelEntry": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
|
||||
"are optional; boolean capability flags are simply omitted when unknown or false."
|
||||
),
|
||||
"required": ["litellm_provider"],
|
||||
"properties": entry_properties,
|
||||
"additionalProperties": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(schema: JsonSchema) -> str:
|
||||
return json.dumps(schema, indent=2) + "\n"
|
||||
|
||||
|
||||
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
|
||||
validator = jsonschema.Draft202012Validator(
|
||||
schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER
|
||||
)
|
||||
return tuple(
|
||||
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
||||
for error in validator.iter_errors(prices)
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
check = "--check" in sys.argv[1:]
|
||||
prices = json.loads(PRICES_PATH.read_text())
|
||||
rendered = render(build_schema(prices))
|
||||
errors = validation_errors(prices, json.loads(rendered))
|
||||
if errors:
|
||||
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
|
||||
print("\n".join(errors[:20]))
|
||||
return 1
|
||||
if not check:
|
||||
SCHEMA_PATH.write_text(rendered)
|
||||
print(f"wrote {SCHEMA_PATH}")
|
||||
return 0
|
||||
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
|
||||
print(
|
||||
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
|
||||
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
|
||||
)
|
||||
return 1
|
||||
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
46
db_scripts/backfill_daily_tool_spend.sql
Normal file
46
db_scripts/backfill_daily_tool_spend.sql
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
-- One-shot backfill of the LiteLLM_DailyToolSpend rollup from the per-request
|
||||
-- LiteLLM_SpendLogToolIndex x LiteLLM_SpendLogs tables.
|
||||
--
|
||||
-- This is an opt-in, manual operation. New deployments do not need it: the
|
||||
-- rollup is written at request time from the moment the release is deployed.
|
||||
-- Run it only if you want the Cost Optimization "Spend by tool" card to show
|
||||
-- history from before the deploy, and only once.
|
||||
--
|
||||
-- IMPORTANT caveats before running:
|
||||
--
|
||||
-- 1. Pre-deploy index rows may include tools that were merely DECLARED in a
|
||||
-- request body but never invoked (the release this ships with stops
|
||||
-- recording those). For agentic clients that declare many tools per
|
||||
-- request, backfilled history attributes each request's full spend to
|
||||
-- every declared tool, overstating per-tool spend. Post-deploy rows do not
|
||||
-- have this problem. If your traffic is mostly such clients, consider not
|
||||
-- backfilling.
|
||||
--
|
||||
-- 2. Coverage is bounded by spend-log retention: rows older than
|
||||
-- maximum_spend_logs_retention_period are already gone.
|
||||
--
|
||||
-- 3. Replace the cutover timestamp below with the time you deployed the
|
||||
-- release, so backfilled per-request rows cannot double-count on top of
|
||||
-- rollup rows the new writer already created. ON CONFLICT DO NOTHING is a
|
||||
-- second guard for (date, tool_name) buckets the writer already touched:
|
||||
-- such buckets keep the writer's numbers and skip the backfill's.
|
||||
--
|
||||
-- Usage:
|
||||
-- psql "$DATABASE_URL" -v cutover="'2026-07-25T00:00:00Z'" -f db_scripts/backfill_daily_tool_spend.sql
|
||||
|
||||
SET TIME ZONE 'UTC';
|
||||
|
||||
INSERT INTO "LiteLLM_DailyToolSpend" (date, tool_name, spend, total_tokens, request_count, created_at, updated_at)
|
||||
SELECT
|
||||
to_char(ti.start_time, 'YYYY-MM-DD') AS date,
|
||||
ti.tool_name,
|
||||
COALESCE(SUM(sl.spend), 0) AS spend,
|
||||
COALESCE(SUM(sl.total_tokens), 0) AS total_tokens,
|
||||
COUNT(*) AS request_count,
|
||||
now() AS created_at,
|
||||
now() AS updated_at
|
||||
FROM "LiteLLM_SpendLogToolIndex" ti
|
||||
JOIN "LiteLLM_SpendLogs" sl ON sl.request_id = ti.request_id
|
||||
WHERE ti.start_time < :cutover::timestamptz
|
||||
GROUP BY 1, 2
|
||||
ON CONFLICT (date, tool_name) DO NOTHING;
|
||||
|
|
@ -54,6 +54,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/messages",
|
||||
"/v1/skills",
|
||||
"/v1/a2a/",
|
||||
"/a2a/",
|
||||
# LiteLLM-native LLM surface
|
||||
"/v1/rerank",
|
||||
"/v2/rerank",
|
||||
|
|
|
|||
|
|
@ -5,5 +5,5 @@ dependencies:
|
|||
- name: redis
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
version: 18.19.1
|
||||
digest: sha256:8660fe6287f9941d08c0902f3f13731079b8cecd2a5da2fbc54e5b7aae4a6f62
|
||||
generated: "2024-03-10T02:28:52.275022+05:30"
|
||||
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
|
||||
generated: "2026-07-28T10:21:22.511401-07:00"
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ type: application
|
|||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 1.1.0
|
||||
version: 1.1.1
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
|
|
@ -32,10 +32,10 @@ annotations:
|
|||
|
||||
dependencies:
|
||||
- name: "postgresql"
|
||||
version: ">=13.3.0"
|
||||
version: "14.3.1"
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
condition: db.deployStandalone
|
||||
- name: redis
|
||||
version: ">=18.0.0"
|
||||
version: "18.19.1"
|
||||
repository: oci://registry-1.docker.io/bitnamicharts
|
||||
condition: redis.enabled
|
||||
|
|
|
|||
|
|
@ -130,6 +130,16 @@ Set `billingMetrics.caSecretName` only when the collector is a private or test o
|
|||
| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` |
|
||||
| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) |
|
||||
| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` |
|
||||
| `postgresql.image.*` | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`. | `bitnamilegacy/postgresql:16.2.0-debian-12-r6` |
|
||||
| `redis.image.*` | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason. | `bitnamilegacy/redis:7.2.4-debian-12-r9` |
|
||||
|
||||
#### Bundled Postgres image
|
||||
|
||||
Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs.
|
||||
|
||||
Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`.
|
||||
|
||||
Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`.
|
||||
|
||||
#### Example Postgres `db.useExisting` Secret
|
||||
|
||||
|
|
|
|||
|
|
@ -146,3 +146,18 @@ Get redis service port
|
|||
{{ .Values.redis.master.service.ports.redis }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Reject an unpinned image tag for the bundled PostgreSQL.
|
||||
A floating tag lets a chart upgrade start a newer PostgreSQL major against the
|
||||
existing PersistentVolumeClaim. The server then refuses to start on a data
|
||||
directory written by another major version, and the only way back is a dump
|
||||
taken before the change, which by that point no longer exists.
|
||||
*/}}
|
||||
{{- define "litellm.validateBundledPostgresImageTag" -}}
|
||||
{{- $tag := .Values.postgresql.image.tag | default "" | toString -}}
|
||||
{{- $digest := .Values.postgresql.image.digest | default "" | toString -}}
|
||||
{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}}
|
||||
{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{{- if .Values.db.deployStandalone -}}
|
||||
{{- include "litellm.validateBundledPostgresImageTag" . -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ metadata:
|
|||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: bitnami/kubectl:latest
|
||||
image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
|
||||
command: ['sh', '-c']
|
||||
args:
|
||||
- |
|
||||
|
|
|
|||
94
helm/litellm-helm/tests/bundled_db_images_tests.yaml
Normal file
94
helm/litellm-helm/tests/bundled_db_images_tests.yaml
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
suite: test bundled database images
|
||||
templates:
|
||||
- charts/postgresql/templates/primary/statefulset.yaml
|
||||
- charts/redis/templates/master/application.yaml
|
||||
- charts/redis/templates/configmap.yaml
|
||||
- charts/redis/templates/health-configmap.yaml
|
||||
- charts/redis/templates/scripts-configmap.yaml
|
||||
- charts/redis/templates/secret.yaml
|
||||
- secret-dbcredentials.yaml
|
||||
- templates/tests/test-servicemonitor.yaml
|
||||
tests:
|
||||
- it: should pull the bundled postgres from a repository that still publishes the pinned tag
|
||||
template: charts/postgresql/templates/primary/statefulset.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6
|
||||
|
||||
- it: should pull the bundled postgres metrics exporter from the same repository
|
||||
template: charts/postgresql/templates/primary/statefulset.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
postgresql.metrics.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[1].image
|
||||
value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14
|
||||
|
||||
- it: should run the bundled postgres init container from the same repository
|
||||
template: charts/postgresql/templates/primary/statefulset.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
postgresql.volumePermissions.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.initContainers[0].image
|
||||
value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16
|
||||
|
||||
- it: should pull the bundled redis from a repository that still publishes the pinned tag
|
||||
template: charts/redis/templates/master/application.yaml
|
||||
set:
|
||||
redis.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9
|
||||
|
||||
- it: should reject a floating postgres tag that could cross a major version on an existing volume
|
||||
template: secret-dbcredentials.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
postgresql.image.tag: latest
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
|
||||
|
||||
- it: should reject an empty postgres tag
|
||||
template: secret-dbcredentials.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
postgresql.image.tag: ""
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
|
||||
|
||||
- it: should accept an empty postgres tag when the image is pinned by digest
|
||||
template: secret-dbcredentials.yaml
|
||||
set:
|
||||
db.deployStandalone: true
|
||||
postgresql.image.tag: ""
|
||||
postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
|
||||
- it: should run the servicemonitor test pod from a pinned image
|
||||
template: templates/tests/test-servicemonitor.yaml
|
||||
set:
|
||||
serviceMonitor.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.containers[0].image
|
||||
value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
|
||||
|
||||
- it: should not constrain the postgres tag when the bundled database is not deployed
|
||||
template: secret-dbcredentials.yaml
|
||||
set:
|
||||
db.deployStandalone: false
|
||||
postgresql.image.tag: latest
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
|
@ -328,8 +328,32 @@ lifecycle: {}
|
|||
|
||||
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
|
||||
# otherwise)
|
||||
#
|
||||
# Bitnami retired the versioned tags under docker.io/bitnami and republished the
|
||||
# archived builds under docker.io/bitnamilegacy, so the subchart's own image
|
||||
# defaults no longer resolve. The repository below points at the same build the
|
||||
# subchart was released with, which keeps the on-disk data directory layout
|
||||
# identical for existing installs.
|
||||
#
|
||||
# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`,
|
||||
# and starting a newer PostgreSQL major against an existing data directory
|
||||
# leaves the server refusing to boot ("database files are incompatible with
|
||||
# server") with no way back other than a dump taken beforehand. Crossing a major
|
||||
# version is a dump-and-restore, not an image bump. The chart refuses to render
|
||||
# an unpinned tag for this reason
|
||||
postgresql:
|
||||
architecture: standalone
|
||||
image:
|
||||
repository: bitnamilegacy/postgresql
|
||||
tag: 16.2.0-debian-12-r6
|
||||
volumePermissions:
|
||||
image:
|
||||
repository: bitnamilegacy/os-shell
|
||||
tag: 12-debian-12-r16
|
||||
metrics:
|
||||
image:
|
||||
repository: bitnamilegacy/postgres-exporter
|
||||
tag: 0.15.0-debian-12-r14
|
||||
auth:
|
||||
username: litellm
|
||||
database: litellm
|
||||
|
|
@ -359,9 +383,36 @@ postgresql:
|
|||
# When `redis.sentinel.enabled` is set, the coordination block is rendered with
|
||||
# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead
|
||||
# of host/port, because a plain Redis client cannot talk to the sentinel port
|
||||
#
|
||||
# The image repositories carry the same bitnamilegacy repoint as postgresql
|
||||
# above; the versioned tags the subchart ships with are gone from
|
||||
# docker.io/bitnami
|
||||
redis:
|
||||
enabled: false
|
||||
architecture: standalone
|
||||
image:
|
||||
repository: bitnamilegacy/redis
|
||||
tag: 7.2.4-debian-12-r9
|
||||
sentinel:
|
||||
image:
|
||||
repository: bitnamilegacy/redis-sentinel
|
||||
tag: 7.2.4-debian-12-r7
|
||||
metrics:
|
||||
image:
|
||||
repository: bitnamilegacy/redis-exporter
|
||||
tag: 1.58.0-debian-12-r4
|
||||
volumePermissions:
|
||||
image:
|
||||
repository: bitnamilegacy/os-shell
|
||||
tag: 12-debian-12-r16
|
||||
sysctl:
|
||||
image:
|
||||
repository: bitnamilegacy/os-shell
|
||||
tag: 12-debian-12-r16
|
||||
kubectl:
|
||||
image:
|
||||
repository: bitnamilegacy/kubectl
|
||||
tag: 1.29.2-debian-12-r3
|
||||
coordination:
|
||||
# Set to false to keep the bundled Redis for response caching only and leave
|
||||
# `general_settings.coordination_redis` out of the rendered config. A
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
|
||||
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
|
||||
"/v1/models" "/models" "/openai" "/engines"
|
||||
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
|
||||
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a"
|
||||
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
|
||||
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
|
||||
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyToolSpend" (
|
||||
"date" TEXT NOT NULL,
|
||||
"tool_name" TEXT NOT NULL,
|
||||
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
||||
"total_tokens" BIGINT NOT NULL DEFAULT 0,
|
||||
"request_count" BIGINT NOT NULL DEFAULT 0,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "LiteLLM_DailyToolSpend_pkey" PRIMARY KEY ("date","tool_name")
|
||||
);
|
||||
|
|
@ -1097,6 +1097,19 @@ model LiteLLM_SpendLogToolIndex {
|
|||
@@index([start_time])
|
||||
}
|
||||
|
||||
// Daily tool spend rollup (one row per tool per day) – the Cost Optimization card reads this, never SpendLogs
|
||||
model LiteLLM_DailyToolSpend {
|
||||
date String
|
||||
tool_name String
|
||||
spend Float @default(0.0)
|
||||
total_tokens BigInt @default(0)
|
||||
request_count BigInt @default(0)
|
||||
created_at DateTime @default(now())
|
||||
updated_at DateTime @updatedAt
|
||||
|
||||
@@id([date, tool_name])
|
||||
}
|
||||
|
||||
// Prompt table for storing prompt configurations
|
||||
model LiteLLM_PromptTable {
|
||||
id String @id @default(uuid())
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
# Adding a provider / route to litellm-rust
|
||||
|
||||
Three layers, same for every route (see `ocr` and `realtime` as references):
|
||||
Everything for a route lives in `crates/core/src/<route>/`; `crates/core/src/messages` is the reference. A host (the axum gateway, the Python bridge) only calls the route's entrypoint.
|
||||
|
||||
1. **Transform contract (pure)** — `crates/core/src/<route>/transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth.
|
||||
2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
|
||||
3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
|
||||
1. **Entrypoint** — `mod.rs`: `pub async fn <route>(request) -> CoreResult<Response>`, the Rust equivalent of `litellm.<route>()`, plus a `<route>_stream` variant when the route streams. It is the only thing a host touches.
|
||||
2. **Transform contract** — `transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) with types in `types.rs`.
|
||||
3. **Provider config** — `crates/core/src/providers/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
|
||||
4. **Prepare + handler** — `prepare.rs` resolves provider/model, credentials, auth headers, and URL, then transforms the request; `handler.rs` performs the provider call through the shared client in `client.rs` and transforms the response.
|
||||
|
||||
## Coding standards
|
||||
|
||||
|
|
@ -25,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider
|
|||
is a few declarative lines, not a new file of duplicated flow. Only diverge from
|
||||
the base when behavior is genuinely different, and say so explicitly in the PR.
|
||||
|
||||
**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
|
||||
**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
|
||||
|
|
|
|||
|
|
@ -4,14 +4,30 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (
|
|||
|
||||
## Crates
|
||||
|
||||
| Crate | Role | Pure / I/O |
|
||||
|-------|------|------------|
|
||||
| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure |
|
||||
| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
|
||||
|
||||
Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
|
||||
|
||||
## Where a route lives
|
||||
|
||||
A top-level LiteLLM call is a module under `crates/core/src/<route>/`, shaped like `messages`:
|
||||
|
||||
```
|
||||
core/src/messages/
|
||||
mod.rs # pub async fn messages(..) -> CoreResult<..> (+ messages_stream for SSE)
|
||||
types.rs # request/response types, MessagesRequest
|
||||
transformation.rs # the provider template trait
|
||||
prepare.rs # provider resolution, auth headers, URL
|
||||
handler.rs # the provider call
|
||||
client.rs # the shared reqwest client
|
||||
```
|
||||
|
||||
Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched.
|
||||
|
||||
Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
|
||||
|
||||
Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
|
||||
|
|
|
|||
|
|
@ -23,21 +23,34 @@ the base when behavior is genuinely different, and say so explicitly in the PR.
|
|||
|
||||
## Crates (exactly three — see AGENTS.md)
|
||||
|
||||
`litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge`
|
||||
exposes it to the Python SDK. A crate is a **layer**, not a route — add modules, not crates.
|
||||
`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
|
||||
`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
|
||||
`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not
|
||||
a route — add modules, not crates.
|
||||
|
||||
## Core Boundary
|
||||
|
||||
`litellm-core` is the pure translation layer; the `litellm-ai-gateway` host executes work.
|
||||
`litellm-core` owns the whole call. The Rust equivalent of `litellm.messages()`
|
||||
is `litellm_core::messages::messages(request).await`: you call it, it does the
|
||||
provider call, and you get a typed non-streaming response back.
|
||||
|
||||
Route-level Rust structure mirrors LiteLLM's Python responsibilities:
|
||||
- `core/src/<route>/` owns the route contract, shared types, and provider
|
||||
template traits. For OCR, this means `core/src/ocr`.
|
||||
- `core/src/<route>/` owns the route end to end: the public entrypoint fn named
|
||||
after the route in `mod.rs`, the request/response types (`types.rs`), the
|
||||
provider template trait (`transformation.rs`), the provider/auth/URL
|
||||
resolution (`prepare.rs`), the HTTP client (`client.rs`), and the handler that
|
||||
performs the call (`handler.rs`). `core/src/messages` is the reference.
|
||||
- `core/src/providers/<provider>/<route>/transformation.rs` owns the
|
||||
provider-specific transform. For Mistral OCR, this means
|
||||
`core/src/providers/mistral/ocr/transformation.rs`.
|
||||
- Network execution lives in the host crate `ai-gateway` (`ai-gateway/src/io/`),
|
||||
never inside `core`.
|
||||
provider-specific transform. For Anthropic Messages, this means
|
||||
`core/src/providers/anthropic/messages/transformation.rs`.
|
||||
- Handlers live in `core`, never in a host. `ai-gateway` must not contain a
|
||||
route handler that talks to a provider; its axum route reads the HTTP request,
|
||||
picks a deployment, and calls the `core` entrypoint. `python-bridge` marshals
|
||||
Python objects and calls the same entrypoint.
|
||||
|
||||
Streaming keeps the same shape: the route entrypoint has a `<route>_stream`
|
||||
variant in `core` that returns the upstream response so a host can splice it to
|
||||
its own caller; the host still owns no provider logic.
|
||||
|
||||
Call-hook and lifecycle instrumentation, including phase timing, usage
|
||||
accumulation, and callback payload construction, always lives in `core`.
|
||||
|
|
@ -45,21 +58,31 @@ Hosts feed observed events into core and dispatch the completed payloads through
|
|||
their I/O logger; hosts must not own callback orchestration.
|
||||
|
||||
Allowed in `core`:
|
||||
- Pure request transforms
|
||||
- Pure response transforms
|
||||
- Pure stream chunk normalization
|
||||
- The public entrypoint for a top-level LiteLLM call
|
||||
- Request/response transforms and stream chunk normalization
|
||||
- Provider resolution, auth header construction, and URL building
|
||||
- The provider HTTP call itself, through a shared reused client with connect and
|
||||
request timeouts
|
||||
- Shared data types and validation errors
|
||||
- Deterministic token/cost helper logic
|
||||
|
||||
Not allowed in `core`:
|
||||
- Network calls
|
||||
- Environment variable or secret reads
|
||||
- Serving HTTP: axum routes, extractors, and transport concerns stay in the host
|
||||
- Filesystem access
|
||||
- Database or cache access
|
||||
- Provider SDK signing or auth flows
|
||||
- Database access
|
||||
- Config file reading and rollout state
|
||||
- Logging callbacks, spend writes, or custom callbacks
|
||||
- Global mutable runtime state
|
||||
|
||||
Env reads in `core` are limited to credential fallback inside a route's
|
||||
`prepare.rs` (the `env_lookup` closure), mirroring what the Python SDK does when
|
||||
no key is passed. Everything else config-shaped is resolved by the host and
|
||||
passed in.
|
||||
|
||||
Routes still hosted in `ai-gateway` (`ocr`, `audio_transcription`, `realtime`)
|
||||
predate this rule and are being moved into `core` route modules; do not add new
|
||||
ones there, and prefer moving one when you touch it.
|
||||
|
||||
Python owns rollout state and fallback while Rust is being introduced. Rust
|
||||
paths must be off by default until parity tests prove equivalence with Python.
|
||||
A new provider/route may instead be implemented rust-only with no Python
|
||||
|
|
@ -93,10 +116,10 @@ the first PR:
|
|||
- Preserve Python output shape intentionally. If a field is always serialized as
|
||||
`null` for Python parity, leave a short comment explaining that parity choice.
|
||||
|
||||
## Host I/O Rules
|
||||
## Network I/O Rules
|
||||
|
||||
These rules apply when adding future crates or modules that execute network I/O,
|
||||
such as `ai-gateway`, router hosts, or standalone servers:
|
||||
These rules apply to every module that executes network I/O, whether it is a
|
||||
`core` route handler or a host such as `ai-gateway`:
|
||||
|
||||
- Set connect and full-request timeouts. No unbounded waits.
|
||||
- Reuse HTTP clients; do not construct clients per request.
|
||||
|
|
|
|||
|
|
@ -2,18 +2,31 @@
|
|||
|
||||
This workspace contains the staged Rust implementation for LiteLLM.
|
||||
|
||||
Rust starts as a pure transform core used by the existing Python host. Python
|
||||
continues to own auth, configuration, network I/O, retries, routing, logging,
|
||||
`litellm-core` is the LiteLLM SDK in Rust: one entrypoint per top-level call
|
||||
that makes the LLM call and hands back a typed response, the same shape as
|
||||
`litellm.messages()` in Python.
|
||||
|
||||
```rust
|
||||
let response = litellm_core::messages::messages(MessagesRequest {
|
||||
model: "claude-sonnet-4-5",
|
||||
body,
|
||||
api_key: Some(key),
|
||||
..
|
||||
})
|
||||
.await?;
|
||||
```
|
||||
|
||||
Python continues to own configuration, retries, routing policy, logging,
|
||||
callbacks, spend tracking, and customer plugins until each Rust path has parity
|
||||
coverage and production evidence.
|
||||
|
||||
## Crates
|
||||
|
||||
| Crate | Role | Pure / I/O |
|
||||
|-------|------|------------|
|
||||
| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure |
|
||||
| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
|
||||
|
||||
Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
|
||||
|
||||
|
|
@ -21,16 +34,16 @@ Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-
|
|||
|
||||
```text
|
||||
crates/
|
||||
core/ Route contracts, shared pure types, errors, and templates.
|
||||
src/ocr/
|
||||
providers/ Provider-specific pure transforms.
|
||||
src/mistral/ocr/transformation.rs
|
||||
core/ The SDK: route modules + provider transforms.
|
||||
src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client
|
||||
src/providers/anthropic/messages/transformation.rs
|
||||
ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints.
|
||||
python-bridge/ PyO3 bridge for Python LiteLLM.
|
||||
```
|
||||
|
||||
The folder shape should follow the Python provider tree:
|
||||
`providers/src/<provider>/<route>/transformation.rs`. The bridge should expose
|
||||
one function per top-level route, starting with `ocr(payload)`.
|
||||
The folder shape follows the Python provider tree:
|
||||
`core/src/providers/<provider>/<route>/transformation.rs`. The bridge exposes one
|
||||
function per top-level route, mirroring the core entrypoints.
|
||||
|
||||
## Checks
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Provider coding standards (litellm-rust)
|
||||
|
||||
Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MISTRAL_OCR_CONFIG`) is the reference; `messages` (`ANTHROPIC_MESSAGES_CONFIG`) is the next port.
|
||||
Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` (`core/src/messages`, `ANTHROPIC_MESSAGES_CONFIG`) is the reference: a route is a `core` module with a public entrypoint that makes the call and returns a typed response.
|
||||
|
||||
## Provider resolution
|
||||
|
||||
|
|
@ -16,10 +16,10 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MIST
|
|||
|
||||
## Boundaries
|
||||
|
||||
7. Layers never cross: `core` = pure transforms/types (no network, env, secrets, auth, logging, global mutable state); `ai-gateway` = all I/O, auth headers, HTTP/SSE, lifecycle hooks; `python-bridge` = thin PyO3 adapter.
|
||||
7. Layers never cross: `core` = the call itself (entrypoint, types, transforms, provider resolution, auth headers, provider HTTP, lifecycle hooks); `ai-gateway` = serving HTTP/WS (routing, extractors, auth of *our* callers, streaming to the client); `python-bridge` = thin PyO3 adapter. Hosts call the core entrypoint; they never build a provider request.
|
||||
8. Generic/route files contain zero provider-specific branches. A provider is one module under `core/src/providers/<provider>/<route>/`; a route is a module, never a new crate.
|
||||
9. Route entry point stays thin: `<route>()` -> `prepare_*` -> `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing. Handlers validate and delegate; no business logic in them.
|
||||
10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Env reads happen only at the host/config layer, with the `DEFAULT_*` fallback defined in `constants.rs`.
|
||||
9. Route entry point stays thin: `core::<route>::<route>()` -> `prepare_*` -> handler (or `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing). Axum handlers validate and delegate to a service that calls the entrypoint; no business logic in them.
|
||||
10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Config-shaped env reads happen at the host/config layer with the `DEFAULT_*` fallback defined in `constants.rs`; the only env read in `core` is the credential fallback in a route's `prepare.rs`.
|
||||
|
||||
## Types and errors
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MIST
|
|||
|
||||
16. Never log request/response bodies, base64 payloads, document contents, or secrets. Truncate and bound any upstream body before it crosses a host boundary.
|
||||
17. Treat empty/whitespace credentials, URLs, and config values as absent at the host resolution layer.
|
||||
18. Host I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS.
|
||||
18. Network I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS.
|
||||
|
||||
## Tests and rollout
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
# ai-gateway — folder architecture
|
||||
|
||||
The Axum server that fronts the Rust gateway. It owns transport + config + auth
|
||||
only; deployment selection lives in `core::router`, transforms in `core`/`providers`.
|
||||
only; deployment selection lives in `core::router`, and the LLM call itself
|
||||
(transforms, auth headers, provider HTTP) lives behind a `core` route entrypoint
|
||||
such as `litellm_core::messages::messages`. No provider handler lives here.
|
||||
|
||||
```
|
||||
src/
|
||||
|
|
@ -32,6 +34,11 @@ src/
|
|||
args; it runs during extraction. Never re-implement the check per route.
|
||||
- **Handlers are thin.** A handler validates and delegates to its `service`. No
|
||||
business logic, no provider calls, no transforms in handlers.
|
||||
- **Services call `core`, they don't reimplement it.** A `service` picks the
|
||||
deployment and calls the `core` route entrypoint. Provider resolution, auth
|
||||
headers, URL building, and the HTTP call are `core`'s job; a service that
|
||||
builds a provider request itself is a bug (`routes/messages/service.rs` is
|
||||
the reference).
|
||||
- **State is shared and cheap to clone.** Long-lived handles live behind `Arc` in
|
||||
`state.rs`; read env/config only in `main.rs` when building state.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
|
|||
|
||||
`litellm-rust` is exactly three crates (a crate is a **layer**, not a route):
|
||||
|
||||
| Crate | Role | Pure / I/O |
|
||||
|-------|------|------------|
|
||||
| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. Builds requests/responses; no network. | Pure |
|
||||
| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under `io/`) plus the Axum server binary (behind the `server` feature). | I/O |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding |
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
|
||||
| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
|
||||
|
||||
Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
|
||||
|
||||
|
|
|
|||
|
|
@ -47,18 +47,6 @@ pub(crate) const DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: f64 = 50.0;
|
|||
|
||||
pub(crate) const MAX_SAFE_FETCH_REDIRECTS: usize = 10;
|
||||
|
||||
/// Full-request timeout ceiling for Anthropic Messages provider calls, in
|
||||
/// seconds. Mirrors the Python Anthropic Messages default. The per-request
|
||||
/// timeout from `litellm_params` still overrides this on the request builder.
|
||||
pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Connect timeout for Anthropic Messages provider calls, in seconds.
|
||||
pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
/// Max characters of an upstream error body echoed across the host boundary
|
||||
/// before truncation, so provider bodies are bounded and data-minimized.
|
||||
pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;
|
||||
|
||||
pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300;
|
||||
|
||||
|
|
@ -66,10 +54,6 @@ pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300;
|
|||
#[cfg(feature = "server")]
|
||||
pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages";
|
||||
|
||||
/// Provider name used by the Anthropic Messages route when a deployment's
|
||||
/// provider model does not carry an explicit provider prefix.
|
||||
pub(crate) const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic";
|
||||
|
||||
/// Request headers owned by the gateway and never forwarded upstream.
|
||||
#[cfg(feature = "server")]
|
||||
pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] =
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
pub use crate::messages::{MessagesRequest, messages};
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
pub mod audio_transcription;
|
||||
pub mod messages;
|
||||
pub mod ocr;
|
||||
pub mod realtime;
|
||||
pub mod realtime_pool;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@
|
|||
//! without pulling in the HTTP server:
|
||||
//!
|
||||
//! - Call-type modules such as [`ocr`]: provider transforms, lifecycle hooks,
|
||||
//! and provider I/O. Always available — no feature required.
|
||||
//! and provider I/O. Always available — no feature required. These predate the
|
||||
//! rule that a route's entrypoint and handler live in `litellm-core` (see
|
||||
//! `litellm_core::messages`) and move there as they are touched.
|
||||
//! - [`io`]: compatibility exports and realtime WebSocket splice helpers.
|
||||
//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
|
||||
//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
|
||||
|
|
@ -15,7 +17,6 @@ pub mod audio_transcription;
|
|||
mod client;
|
||||
pub(crate) mod config;
|
||||
pub mod io;
|
||||
pub mod messages;
|
||||
pub mod ocr;
|
||||
|
||||
/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
use litellm_core::CoreResult;
|
||||
use serde_json::Value;
|
||||
|
||||
mod client;
|
||||
mod common_utils;
|
||||
mod handler;
|
||||
mod prepare;
|
||||
mod types;
|
||||
|
||||
pub use types::MessagesRequest;
|
||||
|
||||
use handler::{execute_messages_provider_call, execute_messages_provider_stream};
|
||||
use prepare::prepare_messages_call;
|
||||
|
||||
pub async fn messages(request: MessagesRequest<'_>) -> CoreResult<Value> {
|
||||
match execute_messages(request, false).await? {
|
||||
MessagesResponse::Json(body) => Ok(body),
|
||||
MessagesResponse::Stream(response) => {
|
||||
drop(response);
|
||||
Err(litellm_core::CoreError::InvalidResponse(
|
||||
"non-streaming messages execution returned a stream".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum MessagesResponse {
|
||||
Json(Value),
|
||||
Stream(reqwest::Response),
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_messages(
|
||||
request: MessagesRequest<'_>,
|
||||
stream: bool,
|
||||
) -> CoreResult<MessagesResponse> {
|
||||
let prepared = prepare_messages_call(request)?;
|
||||
if stream {
|
||||
execute_messages_provider_stream(prepared)
|
||||
.await
|
||||
.map(MessagesResponse::Stream)
|
||||
} else {
|
||||
execute_messages_provider_call(prepared)
|
||||
.await
|
||||
.map(MessagesResponse::Json)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub struct MessagesRequest<'a> {
|
||||
pub model: &'a str,
|
||||
pub body: Value,
|
||||
pub api_key: Option<&'a str>,
|
||||
pub api_base: Option<&'a str>,
|
||||
pub custom_llm_provider: Option<&'a str>,
|
||||
pub extra_headers: Option<Map<String, Value>>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub(crate) struct ProviderMessagesRequest {
|
||||
pub(crate) provider: String,
|
||||
pub(crate) model: String,
|
||||
pub(crate) config: &'static dyn AnthropicMessagesProviderConfig,
|
||||
pub(crate) url: String,
|
||||
pub(crate) body: Value,
|
||||
pub(crate) upstream_headers: Vec<(String, String)>,
|
||||
pub(crate) timeout: Option<Duration>,
|
||||
}
|
||||
|
|
@ -19,7 +19,10 @@ async fn handle(...) -> impl IntoResponse { ... }
|
|||
When a route has business logic worth testing without axum, put it in a sibling
|
||||
`service` (a file, or a folder if the route grows). The route file stays the
|
||||
**axum surface** (router + handler + any socket/SSE adapter); `service` is plain
|
||||
Rust with **no axum types**. `realtime/` is the example:
|
||||
Rust with **no axum types**, and its job is to pick the deployment and call the
|
||||
`core` route entrypoint (see `messages/service.rs` calling
|
||||
`litellm_core::messages::messages`). Never build a provider request, resolve a
|
||||
key, or perform the provider call here. `realtime/` is the older example:
|
||||
```
|
||||
realtime/
|
||||
mod.rs # axum surface: router() + handler + the WS<->events adapter
|
||||
|
|
@ -33,6 +36,8 @@ genuinely gets hard to read.
|
|||
`crate::auth::RequireMasterKey` to its arguments; it runs during extraction.
|
||||
Never re-implement the check per route.
|
||||
- **Handlers contain no business logic; `service` contains no axum types.**
|
||||
- **No provider handlers in this crate.** Transforms, auth headers, and the
|
||||
provider HTTP call live in `core/src/<route>/`.
|
||||
- A route owns its paths in its own `router()`; `mod.rs` only merges.
|
||||
- Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`,
|
||||
not duplicated in handlers.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER;
|
||||
use litellm_core::messages::types::MessagesRequest;
|
||||
use litellm_core::messages::{messages, messages_stream};
|
||||
use litellm_core::router::Router;
|
||||
use litellm_core::{CoreError, CoreResult};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
|
||||
use crate::messages::{MessagesRequest, execute_messages};
|
||||
|
||||
pub(crate) enum MessagesResponse {
|
||||
Json(Value),
|
||||
Stream(reqwest::Response),
|
||||
|
|
@ -52,13 +52,14 @@ pub async fn run(
|
|||
extra_headers,
|
||||
timeout: None,
|
||||
};
|
||||
let stream = request.body.get("stream").and_then(Value::as_bool) == Some(true);
|
||||
execute_messages(request, stream)
|
||||
.await
|
||||
.map(|response| match response {
|
||||
crate::messages::MessagesResponse::Json(body) => MessagesResponse::Json(body),
|
||||
crate::messages::MessagesResponse::Stream(upstream) => {
|
||||
MessagesResponse::Stream(upstream)
|
||||
}
|
||||
if request.body.get("stream").and_then(Value::as_bool) == Some(true) {
|
||||
return messages_stream(request).await.map(MessagesResponse::Stream);
|
||||
}
|
||||
|
||||
let response = messages(request).await?;
|
||||
serde_json::to_value(response)
|
||||
.map(MessagesResponse::Json)
|
||||
.map_err(|err| {
|
||||
CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
litellm-core is the PURE translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. No network, no I/O, no env reads.
|
||||
litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-level call is a module under `src/<route>/` exposing a public entrypoint named after the route (`messages::messages()`, the Rust equivalent of `litellm.messages()`): you call it and get a typed non-streaming response back.
|
||||
|
||||
Routes (ocr, realtime) and providers (mistral, openai) are modules, not crates.
|
||||
A route module owns everything the call needs: types, the provider template trait, provider transforms (under `providers/`), provider/auth/URL resolution, and the handler that performs the HTTP call. Handlers belong here, never in a host crate.
|
||||
|
||||
Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback dispatch. Env reads are limited to credential fallback in a route's `prepare.rs`.
|
||||
|
||||
Routes (messages, ocr, realtime) and providers (anthropic, mistral, openai) are modules, not crates.
|
||||
|
|
|
|||
|
|
@ -4,20 +4,28 @@ Rules for `litellm-rust/crates/core`.
|
|||
|
||||
## Responsibility
|
||||
|
||||
`core` owns shared data types, typed errors, and deterministic helper contracts.
|
||||
It must stay pure and host-independent.
|
||||
`core` is the LiteLLM SDK in Rust: it makes the LLM call. Every top-level
|
||||
LiteLLM call has a public entrypoint here, named after the route
|
||||
(`messages::messages()` is the Rust equivalent of `litellm.messages()`), and
|
||||
calling it returns a typed non-streaming response.
|
||||
|
||||
Allowed:
|
||||
- The public entrypoint for a route, plus its `<route>_stream` variant when the
|
||||
route supports streaming.
|
||||
- Provider resolution, auth header construction, URL building, and the provider
|
||||
HTTP call (shared reused client, connect + request timeouts).
|
||||
- Shared request/response structs.
|
||||
- Typed errors with stable, non-sensitive messages.
|
||||
- Deterministic validation helpers.
|
||||
- Serialization helpers that intentionally mirror Python output shape.
|
||||
- Route templates that match Python base config responsibilities, such as
|
||||
`ocr::transformation::OcrProviderConfig`.
|
||||
`messages::transformation::AnthropicMessagesProviderConfig`.
|
||||
|
||||
Not allowed:
|
||||
- Network, filesystem, database, cache, or environment access.
|
||||
- Secret reads or auth/header construction.
|
||||
- Serving HTTP: axum routers, extractors, and other transport concerns.
|
||||
- Filesystem, database, or cache access.
|
||||
- Config file reading or rollout state; the host resolves those and passes them
|
||||
in. Env reads are limited to credential fallback in a route's `prepare.rs`.
|
||||
- Logging callbacks, tracing spans, spend writes, or customer callbacks.
|
||||
- Provider-specific branching that belongs in `providers`.
|
||||
- Panics for user/provider-controlled input.
|
||||
|
|
@ -33,10 +41,21 @@ typed field on a struct, not a raw string threaded through the API.
|
|||
|
||||
## Structure
|
||||
|
||||
Use route names directly under `src/`: `ocr`, future `messages`,
|
||||
Use route names directly under `src/`: `messages`, `ocr`, future
|
||||
`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
|
||||
invent broad names like `engine` for route contracts.
|
||||
|
||||
`src/messages` is the reference shape for a route module:
|
||||
|
||||
```
|
||||
mod.rs pub async fn messages(..) (+ messages_stream)
|
||||
types.rs request/response types
|
||||
transformation.rs the provider template trait
|
||||
prepare.rs provider resolution, auth headers, URL
|
||||
handler.rs the provider call
|
||||
client.rs the shared reqwest client
|
||||
```
|
||||
|
||||
## Parity Rules
|
||||
|
||||
- Every shared type used by a provider transform needs unit tests for
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ rust-version.workspace = true
|
|||
|
||||
[dependencies]
|
||||
rand.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
|
@ -31,5 +32,4 @@ bedrock-auth = [
|
|||
]
|
||||
|
||||
[dev-dependencies]
|
||||
reqwest.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
|
|
|
|||
|
|
@ -5,3 +5,19 @@ pub(crate) const BEARER_SCHEME: &str = "Bearer";
|
|||
pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com";
|
||||
pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1";
|
||||
pub const OPENAI_RESPONSES_PATH: &str = "/responses";
|
||||
|
||||
/// Full-request timeout ceiling for Anthropic Messages provider calls, in
|
||||
/// seconds. Mirrors the Python Anthropic Messages default. The per-request
|
||||
/// timeout from the caller still overrides this on the request builder.
|
||||
pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Connect timeout for Anthropic Messages provider calls, in seconds.
|
||||
pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
/// Max characters of an upstream error body echoed across the call boundary
|
||||
/// before truncation, so provider bodies are bounded and data-minimized.
|
||||
pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;
|
||||
|
||||
/// Provider name used for Anthropic Messages when a deployment's provider model
|
||||
/// does not carry an explicit provider prefix.
|
||||
pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic";
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
use litellm_core::CoreResult;
|
||||
use litellm_core::error::{CoreError, json_type_name};
|
||||
use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
|
||||
use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
|
||||
use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS;
|
||||
use crate::error::{CoreError, CoreResult, json_type_name};
|
||||
use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
|
||||
use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
|
||||
|
||||
use super::transformation::AnthropicMessagesProviderConfig;
|
||||
|
||||
pub(super) fn truncate_error_body(body: &str) -> String {
|
||||
if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS {
|
||||
|
|
@ -1,15 +1,13 @@
|
|||
use litellm_core::CoreResult;
|
||||
use litellm_core::error::CoreError;
|
||||
use serde_json::Value;
|
||||
use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
|
||||
use crate::error::{CoreError, CoreResult};
|
||||
|
||||
use super::client::http_client;
|
||||
use super::common_utils::truncate_error_body;
|
||||
use super::types::ProviderMessagesRequest;
|
||||
use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
|
||||
use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest};
|
||||
|
||||
pub(super) async fn execute_messages_provider_call(
|
||||
request: ProviderMessagesRequest,
|
||||
) -> CoreResult<Value> {
|
||||
) -> CoreResult<AnthropicMessagesResponse> {
|
||||
let mut request_builder = http_client().post(&request.url).json(&request.body);
|
||||
for (key, value) in &request.upstream_headers {
|
||||
request_builder = request_builder.header(key, value);
|
||||
|
|
@ -39,12 +37,7 @@ pub(super) async fn execute_messages_provider_call(
|
|||
let response = serde_json::from_str(&text).map_err(|err| {
|
||||
CoreError::InvalidResponse(format!("invalid messages response JSON: {err}"))
|
||||
})?;
|
||||
let transformed = request
|
||||
.config
|
||||
.transform_response(&request.model, response)?;
|
||||
serde_json::to_value(transformed).map_err(|err| {
|
||||
CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
|
||||
})
|
||||
request.config.transform_response(&request.model, response)
|
||||
}
|
||||
|
||||
pub(super) async fn execute_messages_provider_stream(
|
||||
|
|
@ -1,2 +1,32 @@
|
|||
//! The Anthropic Messages call, the Rust equivalent of Python's
|
||||
//! `litellm.messages()`.
|
||||
//!
|
||||
//! [`messages`] is the top-level entrypoint: give it a model, a body, and
|
||||
//! credentials, and it resolves the provider, transforms the request, calls the
|
||||
//! provider, and returns a typed non-streaming response. [`messages_stream`]
|
||||
//! is the streaming variant; it hands the raw upstream response back so a host
|
||||
//! can splice the event stream to its own caller.
|
||||
|
||||
mod client;
|
||||
mod common_utils;
|
||||
mod handler;
|
||||
mod prepare;
|
||||
pub mod transformation;
|
||||
pub mod types;
|
||||
|
||||
use crate::error::CoreResult;
|
||||
|
||||
use handler::{execute_messages_provider_call, execute_messages_provider_stream};
|
||||
use prepare::prepare_messages_call;
|
||||
use types::{AnthropicMessagesResponse, MessagesRequest};
|
||||
|
||||
pub async fn messages(request: MessagesRequest<'_>) -> CoreResult<AnthropicMessagesResponse> {
|
||||
execute_messages_provider_call(prepare_messages_call(request)?).await
|
||||
}
|
||||
|
||||
pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult<reqwest::Response> {
|
||||
execute_messages_provider_stream(prepare_messages_call(request)?).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
use litellm_core::CoreError;
|
||||
use litellm_core::CoreResult;
|
||||
use litellm_core::messages::transformation::MessagesAuthStrategy;
|
||||
use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
|
||||
use crate::error::{CoreError, CoreResult};
|
||||
use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
|
||||
|
||||
use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers};
|
||||
use super::transformation::MessagesAuthStrategy;
|
||||
use super::types::{MessagesRequest, ProviderMessagesRequest};
|
||||
|
||||
pub(super) fn prepare_messages_call(
|
||||
|
|
@ -1,14 +1,16 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_core::error::CoreError;
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
use crate::error::CoreError;
|
||||
|
||||
use super::common_utils::{
|
||||
has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body,
|
||||
};
|
||||
use super::{MessagesRequest, messages};
|
||||
use super::messages;
|
||||
use super::types::MessagesRequest;
|
||||
|
||||
async fn read_http_request(socket: &mut TcpStream) -> String {
|
||||
let mut request = Vec::new();
|
||||
|
|
@ -152,8 +154,8 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through()
|
|||
.await
|
||||
.expect("messages request succeeds");
|
||||
|
||||
assert_eq!(response["content"][0]["text"], "hi");
|
||||
assert_eq!(response["stop_reason"], "end_turn");
|
||||
assert_eq!(response.content[0]["text"], "hi");
|
||||
assert_eq!(response.stop_reason.as_deref(), Some("end_turn"));
|
||||
|
||||
let request = server.await.expect("server task completes");
|
||||
let (head, body) = request.split_once("\r\n\r\n").expect("has body");
|
||||
|
|
@ -208,8 +210,8 @@ async fn messages_round_trip_builds_native_anthropic_request() {
|
|||
.await
|
||||
.expect("messages request succeeds");
|
||||
|
||||
assert_eq!(response["content"][0]["text"], "hi");
|
||||
assert_eq!(response["stop_reason"], "end_turn");
|
||||
assert_eq!(response.content[0]["text"], "hi");
|
||||
assert_eq!(response.stop_reason.as_deref(), Some("end_turn"));
|
||||
|
||||
let request = server.await.expect("server task completes");
|
||||
let (head, _) = request.split_once("\r\n\r\n").expect("has body");
|
||||
|
|
@ -1,6 +1,30 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::transformation::AnthropicMessagesProviderConfig;
|
||||
|
||||
pub struct MessagesRequest<'a> {
|
||||
pub model: &'a str,
|
||||
pub body: Value,
|
||||
pub api_key: Option<&'a str>,
|
||||
pub api_base: Option<&'a str>,
|
||||
pub custom_llm_provider: Option<&'a str>,
|
||||
pub extra_headers: Option<Map<String, Value>>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub(super) struct ProviderMessagesRequest {
|
||||
pub(super) provider: String,
|
||||
pub(super) model: String,
|
||||
pub(super) config: &'static dyn AnthropicMessagesProviderConfig,
|
||||
pub(super) url: String,
|
||||
pub(super) body: Value,
|
||||
pub(super) upstream_headers: Vec<(String, String)>,
|
||||
pub(super) timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum SystemPrompt {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over litellm-ai-gateway.
|
||||
litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`).
|
||||
|
||||
Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call into litellm-ai-gateway.
|
||||
Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint.
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ Python-compatible dictionaries.
|
|||
## Bridge Shape
|
||||
|
||||
- Prefer one stable method per top-level LiteLLM route, for example
|
||||
`ocr(payload)`.
|
||||
`messages(...)`, calling the matching `litellm-core` entrypoint.
|
||||
- Do not add one exported PyO3 function per provider helper unless there is a
|
||||
measured reason.
|
||||
- Provider dispatch belongs in Rust route modules such as
|
||||
`litellm_providers::ocr`, not in this PyO3 crate.
|
||||
- Provider dispatch belongs in the `litellm-core` route module (e.g.
|
||||
`litellm_core::messages`), not in this PyO3 crate.
|
||||
- Python owns rollout state and fallback. Rust should return errors; Python
|
||||
decides whether to raise or fall back. For a rust-only provider/route (no
|
||||
Python reference), the Python side is a thin dispatch that calls Rust and
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ use std::time::Duration;
|
|||
use litellm_ai_gateway::io::audio_transcription::{
|
||||
AudioTranscriptionRequest, audio_transcription as run_audio_transcription,
|
||||
};
|
||||
use litellm_ai_gateway::io::messages::{MessagesRequest, messages as run_messages};
|
||||
use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr};
|
||||
use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
|
||||
use litellm_core::error::CoreError;
|
||||
use litellm_core::messages::messages as run_messages;
|
||||
use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest};
|
||||
use pyo3::exceptions::{PyRuntimeError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyAny, PyDict};
|
||||
|
|
@ -41,6 +42,15 @@ fn ocr_error_to_pyerr(py: Python<'_>, err: CoreError) -> PyErr {
|
|||
build_rust_ocr_error(py, &message, status_code).unwrap_or_else(|import_err| import_err)
|
||||
}
|
||||
|
||||
fn messages_response_to_py(
|
||||
py: Python<'_>,
|
||||
response: AnthropicMessagesResponse,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let value =
|
||||
serde_json::to_value(response).map_err(|err| PyValueError::new_err(err.to_string()))?;
|
||||
json_to_py(py, value)
|
||||
}
|
||||
|
||||
fn core_error_to_pyerr(err: CoreError) -> PyErr {
|
||||
match err {
|
||||
CoreError::Auth(message) => PyValueError::new_err(message),
|
||||
|
|
@ -400,7 +410,7 @@ fn messages(
|
|||
});
|
||||
|
||||
match result {
|
||||
Ok(value) => json_to_py(py, value),
|
||||
Ok(response) => messages_response_to_py(py, response),
|
||||
Err(err) => Err(core_error_to_pyerr(err)),
|
||||
}
|
||||
}
|
||||
|
|
@ -422,7 +432,7 @@ fn amessages(
|
|||
marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?;
|
||||
|
||||
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||
let value = run_messages(MessagesRequest {
|
||||
let response = run_messages(MessagesRequest {
|
||||
model: &model,
|
||||
body,
|
||||
api_key: api_key.as_deref(),
|
||||
|
|
@ -434,7 +444,7 @@ fn amessages(
|
|||
.await
|
||||
.map_err(core_error_to_pyerr)?;
|
||||
|
||||
Python::attach(|py| json_to_py(py, value))
|
||||
Python::attach(|py| messages_response_to_py(py, response))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -209,7 +209,15 @@ class ResponsesToCompletionBridgeHandler:
|
|||
json_mode=kwargs.get("json_mode"),
|
||||
)
|
||||
elif isinstance(result, ModelResponse):
|
||||
return result
|
||||
if not stream:
|
||||
return result
|
||||
return self._completed_response_as_stream(
|
||||
response=result,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
)
|
||||
elif not stream:
|
||||
responses_api_response = self._collect_response_from_stream(result)
|
||||
return self.transformation_handler.transform_response(
|
||||
|
|
@ -299,7 +307,15 @@ class ResponsesToCompletionBridgeHandler:
|
|||
json_mode=kwargs.get("json_mode"),
|
||||
)
|
||||
elif isinstance(result, ModelResponse):
|
||||
return result
|
||||
if not stream:
|
||||
return result
|
||||
return self._completed_response_as_stream(
|
||||
response=result,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
)
|
||||
elif not stream:
|
||||
responses_api_response = await self._collect_response_from_stream_async(result)
|
||||
return self.transformation_handler.transform_response(
|
||||
|
|
@ -331,6 +347,25 @@ class ResponsesToCompletionBridgeHandler:
|
|||
)
|
||||
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
|
||||
|
||||
def _completed_response_as_stream(
|
||||
self,
|
||||
response: "ModelResponse",
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
json_mode: bool | None,
|
||||
) -> "CustomStreamWrapper":
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
|
||||
streamwrapper = CustomStreamWrapper(
|
||||
completion_stream=MockResponseIterator(model_response=response, json_mode=json_mode),
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
|
||||
|
||||
@staticmethod
|
||||
def _apply_post_stream_processing(
|
||||
stream: "CustomStreamWrapper",
|
||||
|
|
|
|||
|
|
@ -1077,6 +1077,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
||||
def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False):
|
||||
super().__init__(streaming_response, sync_stream, json_mode)
|
||||
self._chat_completion_id: str | None = None
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: Union[str, "BaseModel"]
|
||||
|
|
@ -1384,4 +1385,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
ModelResponseStream: OpenAI-formatted streaming chunk
|
||||
"""
|
||||
verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}")
|
||||
return OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk)
|
||||
return self._with_stream_scoped_id(
|
||||
OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk)
|
||||
)
|
||||
|
||||
def _with_stream_scoped_id(self, chunk: "ModelResponseStream") -> "ModelResponseStream":
|
||||
if self._chat_completion_id is None:
|
||||
self._chat_completion_id = chunk.id
|
||||
else:
|
||||
chunk.id = self._chat_completion_id
|
||||
return chunk
|
||||
|
|
|
|||
|
|
@ -1457,7 +1457,7 @@ SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEA
|
|||
SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
|
||||
os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
|
||||
)
|
||||
TOOL_SPEND_MAX_WINDOW_DAYS = 30
|
||||
TOOL_SPEND_TOP_TOOLS = 100
|
||||
SPEND_LOG_PARTITION_INTERVAL = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
|
||||
SPEND_LOG_PARTITION_PRECREATE_AHEAD = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
|
||||
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ from litellm.exceptions import (
|
|||
# proxy's metadata sanitizer.
|
||||
_PRE_CALL_EXECUTED_TOKEN = secrets.token_hex(16)
|
||||
|
||||
_GUARDRAIL_BLOCK_STATUS_CODES = frozenset({400, 403, 422})
|
||||
|
||||
_guardrail_self_recorded: contextvars.ContextVar[bool] = contextvars.ContextVar(
|
||||
"litellm_guardrail_self_recorded", default=False
|
||||
)
|
||||
|
|
@ -1055,8 +1057,15 @@ class CustomGuardrail(CustomLogger):
|
|||
- GuardrailRaisedException (generic guardrail API, tool permission)
|
||||
- BlockedPiiEntityError (Presidio PII detection)
|
||||
- SensitiveDataRouteException (sensitive-data reroute to on-premise model)
|
||||
- HTTPException with status 400 (content policy violation)
|
||||
- HTTPException with a block-signalling status (400, 403, 422)
|
||||
- ModifyResponseException (passthrough mode violation)
|
||||
|
||||
Only the statuses guardrails use in-tree to signal a deliberate rejection
|
||||
count as an intervention: 400 (content policy), 403 (e.g. akto) and 422
|
||||
(e.g. llm_as_a_judge). Other 4xx codes are commonly propagated from an
|
||||
upstream guardrail provider response (401 bad key, 408 timeout, 429 rate
|
||||
limit, or a raw upstream status), which are technical failures, not
|
||||
blocks, so they stay guardrail_failed_to_respond.
|
||||
"""
|
||||
if isinstance(e, ModifyResponseException):
|
||||
return True
|
||||
|
|
@ -1069,7 +1078,11 @@ class CustomGuardrail(CustomLogger):
|
|||
),
|
||||
):
|
||||
return True
|
||||
if HTTPException is not None and isinstance(e, HTTPException) and e.status_code == 400:
|
||||
if (
|
||||
HTTPException is not None
|
||||
and isinstance(e, HTTPException)
|
||||
and e.status_code in _GUARDRAIL_BLOCK_STATUS_CODES
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -133,6 +133,15 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
"dotted_order": metadata.get("dotted_order", None),
|
||||
}
|
||||
|
||||
def _redact_metadata(self, metadata: dict) -> dict:
|
||||
# helper is shallow; also scrub nested requester_metadata since
|
||||
# LangSmith forwards the whole dict into the run
|
||||
redacted = redact_user_api_key_info(metadata=dict(metadata))
|
||||
nested = redacted.get("requester_metadata")
|
||||
if isinstance(nested, dict):
|
||||
redacted["requester_metadata"] = redact_user_api_key_info(metadata=nested)
|
||||
return redacted
|
||||
|
||||
def _build_extra_metadata(self, metadata: Dict):
|
||||
extra_metadata = dict(metadata)
|
||||
requester_metadata = extra_metadata.get("requester_metadata")
|
||||
|
|
@ -141,13 +150,7 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
if key in requester_metadata and key not in extra_metadata:
|
||||
extra_metadata[key] = requester_metadata[key]
|
||||
|
||||
# helper is shallow; also scrub nested requester_metadata since
|
||||
# LangSmith forwards the whole dict into `extra`
|
||||
extra_metadata = redact_user_api_key_info(metadata=extra_metadata)
|
||||
nested = extra_metadata.get("requester_metadata")
|
||||
if isinstance(nested, dict):
|
||||
extra_metadata["requester_metadata"] = redact_user_api_key_info(metadata=nested)
|
||||
return extra_metadata
|
||||
return self._redact_metadata(extra_metadata)
|
||||
|
||||
def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> Dict[str, Any]:
|
||||
response = payload["response"]
|
||||
|
|
@ -200,12 +203,13 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
|
||||
metadata = payload["metadata"]
|
||||
extra_metadata = self._build_extra_metadata(dict(metadata))
|
||||
inputs = {**payload, "metadata": self._redact_metadata(dict(metadata))}
|
||||
outputs = self._build_outputs_with_usage(payload)
|
||||
|
||||
data = {
|
||||
"name": fields["run_name"],
|
||||
"run_type": "llm",
|
||||
"inputs": payload,
|
||||
"inputs": inputs,
|
||||
"outputs": outputs,
|
||||
"session_name": fields["project_name"],
|
||||
"start_time": payload["startTime"],
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
|
|||
OTELSemconvCategory,
|
||||
parse_semconv_opt_in,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import Metric
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.secret_managers.main import get_secret_bool, str_to_bool
|
||||
|
|
@ -597,32 +598,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
meter = meter_provider.get_meter(__name__)
|
||||
|
||||
self._operation_duration_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38
|
||||
name=Metric.OPERATION_DURATION,
|
||||
description="GenAI operation duration",
|
||||
unit="s",
|
||||
)
|
||||
self._token_usage_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38
|
||||
name=Metric.TOKEN_USAGE,
|
||||
description="GenAI token usage",
|
||||
unit="{token}",
|
||||
)
|
||||
self._cost_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.token.cost",
|
||||
name=Metric.TOKEN_COST,
|
||||
description="GenAI request cost",
|
||||
unit="USD",
|
||||
)
|
||||
self._time_to_first_token_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.response.time_to_first_token",
|
||||
name=Metric.TIME_TO_FIRST_TOKEN,
|
||||
description="Time to first token for streaming requests",
|
||||
unit="s",
|
||||
)
|
||||
self._time_per_output_token_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.response.time_per_output_token",
|
||||
name=Metric.TIME_PER_OUTPUT_TOKEN,
|
||||
description="Average time per output token (generation time / completion tokens)",
|
||||
unit="s",
|
||||
)
|
||||
self._response_duration_histogram = meter.create_histogram(
|
||||
name="gen_ai.client.response.duration",
|
||||
name=Metric.RESPONSE_DURATION,
|
||||
description="Total LLM API generation time (excludes LiteLLM overhead)",
|
||||
unit="s",
|
||||
)
|
||||
|
|
@ -2980,10 +2981,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
def _get_metric_reader(self):
|
||||
"""
|
||||
Get the appropriate metric reader based on the configuration.
|
||||
|
||||
Histograms keep the SDK's default cumulative temporality: Prometheus-backed
|
||||
OTLP receivers reject delta histograms and drop the whole batch, while
|
||||
backends that prefer delta still accept cumulative.
|
||||
"""
|
||||
from opentelemetry.sdk.metrics import Histogram
|
||||
from opentelemetry.sdk.metrics.export import (
|
||||
AggregationTemporality,
|
||||
ConsoleMetricExporter,
|
||||
PeriodicExportingMetricReader,
|
||||
)
|
||||
|
|
@ -3014,7 +3017,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
exporter = OTLPMetricExporter(
|
||||
endpoint=normalized_endpoint,
|
||||
headers=_split_otel_headers,
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
|
||||
|
|
@ -3032,7 +3034,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
exporter = OTLPMetricExporter(
|
||||
endpoint=normalized_endpoint,
|
||||
headers=_split_otel_headers,
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
|
||||
|
||||
|
|
|
|||
|
|
@ -257,13 +257,27 @@ class LiteLLM:
|
|||
|
||||
|
||||
class Metric:
|
||||
"""GenAI metric instrument names."""
|
||||
"""GenAI metric instrument names.
|
||||
|
||||
Every name here that a convention or a backend defines uses that name, so a
|
||||
consumer charting GenAI telemetry finds litellm's series where it looks for
|
||||
them. ``TOKEN_USAGE``, ``OPERATION_DURATION``, ``TIME_TO_FIRST_TOKEN`` and
|
||||
``TIME_PER_OUTPUT_TOKEN`` are semconv instruments, defined in the GenAI
|
||||
conventions; the ``gen_ai.client.response.*`` spellings litellm used for the
|
||||
latter two are not conventions at all, so nothing downstream could chart
|
||||
them. Cost has no semconv instrument, so it takes ``gen_ai.usage.cost``, the
|
||||
name backends already query for spend.
|
||||
|
||||
``RESPONSE_DURATION`` keeps its vendor spelling deliberately: the closest
|
||||
convention, ``gen_ai.server.request.duration``, would collide in meaning with
|
||||
``OPERATION_DURATION``, which litellm already emits for the whole operation.
|
||||
"""
|
||||
|
||||
TOKEN_USAGE: Final = "gen_ai.client.token.usage"
|
||||
OPERATION_DURATION: Final = "gen_ai.client.operation.duration"
|
||||
TOKEN_COST: Final = "gen_ai.client.token.cost"
|
||||
TIME_TO_FIRST_TOKEN: Final = "gen_ai.client.response.time_to_first_token"
|
||||
TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.client.response.time_per_output_token"
|
||||
TOKEN_COST: Final = "gen_ai.usage.cost"
|
||||
TIME_TO_FIRST_TOKEN: Final = "gen_ai.server.time_to_first_token"
|
||||
TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.server.time_per_output_token"
|
||||
RESPONSE_DURATION: Final = "gen_ai.client.response.duration"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
"""Shared, OpenTelemetry-free helpers for the otel integration.
|
||||
|
||||
Generic value coercion (for reading heterogeneous logging-payload dicts), time
|
||||
conversion, and header parsing — pulled out of the individual modules so they
|
||||
live in one place. Deliberately free of any ``opentelemetry`` import so the
|
||||
OTel-free sources of truth (payloads, semconv, spans, config) can use it too.
|
||||
Generic value coercion (for reading heterogeneous logging-payload dicts) and
|
||||
time conversion — pulled out of the individual modules so they live in one
|
||||
place. Deliberately free of any ``opentelemetry`` import so the OTel-free
|
||||
sources of truth (payloads, semconv, spans, config) can use it too. OTLP header
|
||||
parsing lives in :mod:`litellm.integrations.otel.plumbing.providers` instead,
|
||||
because it delegates to the OTel SDK's own W3C Baggage parser.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
|
@ -89,15 +91,3 @@ def to_seconds(value: datetime | float | int | str | None) -> float | None:
|
|||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_headers(raw: str | None) -> dict[str, str]:
|
||||
"""Parse an OTLP ``"k=v,k=v"`` header string into a dict."""
|
||||
headers: dict[str, str] = {}
|
||||
if not raw:
|
||||
return headers
|
||||
for pair in raw.split(","):
|
||||
if "=" in pair:
|
||||
key, _, value = pair.partition("=")
|
||||
headers[key.strip()] = value.strip()
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -29,15 +29,13 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
|||
InMemorySpanExporter,
|
||||
)
|
||||
from opentelemetry.trace import Span, SpanKind, Tracer
|
||||
from opentelemetry.util.re import parse_env_headers
|
||||
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.semconv import LiteLLM
|
||||
from litellm.integrations.otel.model.spans import LiteLLMSpanKind
|
||||
|
||||
# Re-exported so ``providers.parse_headers`` remains a stable entry point.
|
||||
from litellm.integrations.otel.model.utils import parse_headers as parse_headers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.metrics import Meter
|
||||
from opentelemetry.sdk.metrics.export import MetricReader
|
||||
|
|
@ -119,6 +117,23 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None:
|
|||
return endpoint + "/v1/traces"
|
||||
|
||||
|
||||
def parse_headers(raw: str | None) -> dict[str, str]:
|
||||
"""Parse an OTLP ``"k=v,k=v"`` header string into a dict.
|
||||
|
||||
``OTEL_EXPORTER_OTLP_HEADERS`` is W3C Baggage encoded per the OTLP spec, so
|
||||
values are percent-decoded: a vendor that documents
|
||||
``Authorization=Basic%20<token>`` (Grafana Cloud does, because a bare space
|
||||
is not representable there) has to reach the exporter as ``Basic <token>``,
|
||||
not with a literal ``%20`` that the backend rejects as malformed. The SDK's
|
||||
own parser is used so litellm decodes exactly what the OTLP exporters do
|
||||
when they read the env var themselves; ``liberal`` keeps values that are not
|
||||
percent-encoded (``Authorization=Bearer <token>``) working unchanged.
|
||||
"""
|
||||
if not raw:
|
||||
return {}
|
||||
return dict(parse_env_headers(raw, liberal=True))
|
||||
|
||||
|
||||
def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
|
||||
kind = (spec.kind or "console").lower()
|
||||
factory = _EXPORTER_FACTORIES.get(kind)
|
||||
|
|
@ -191,6 +206,13 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
|
|||
``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
|
||||
and ``otlp_grpc`` export over OTLP with the configured endpoint/headers. The
|
||||
reader exports on a 5s period, matching v1.
|
||||
|
||||
Histograms keep the SDK's default cumulative temporality. Prometheus-backed
|
||||
OTLP receivers (Grafana Cloud / Mimir, and the Prometheus OTLP endpoint)
|
||||
reject delta histograms outright with ``invalid temporality and type
|
||||
combination``, which drops the whole metric batch, while backends that
|
||||
prefer delta still accept cumulative. The enterprise billing exporter
|
||||
already relies on the same default.
|
||||
"""
|
||||
from opentelemetry.sdk.metrics.export import (
|
||||
ConsoleMetricExporter,
|
||||
|
|
@ -202,18 +224,12 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
|
|||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
||||
OTLPMetricExporter as HTTPMetricExporter,
|
||||
)
|
||||
from opentelemetry.sdk.metrics import Histogram
|
||||
from opentelemetry.sdk.metrics.export import AggregationTemporality
|
||||
|
||||
exporter: Any = HTTPMetricExporter(
|
||||
endpoint=_otlp_metrics_endpoint(config.endpoint),
|
||||
headers=parse_headers(config.headers),
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
elif kind in ("otlp_grpc", "grpc"):
|
||||
from opentelemetry.sdk.metrics import Histogram
|
||||
from opentelemetry.sdk.metrics.export import AggregationTemporality
|
||||
|
||||
try:
|
||||
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
|
||||
OTLPMetricExporter as GRPCMetricExporter,
|
||||
|
|
@ -227,7 +243,6 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
|
|||
exporter = GRPCMetricExporter(
|
||||
endpoint=config.endpoint,
|
||||
headers=parse_headers(config.headers),
|
||||
preferred_temporality={Histogram: AggregationTemporality.DELTA},
|
||||
)
|
||||
else:
|
||||
exporter = ConsoleMetricExporter()
|
||||
|
|
|
|||
|
|
@ -69,6 +69,15 @@ else:
|
|||
|
||||
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
|
||||
|
||||
# Tiers a caller may name in a request, across the providers that accept the
|
||||
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
|
||||
# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
|
||||
# maps "default" to "standard". Used to bound the caller-controlled fallback in
|
||||
# ``get_service_tier_from_standard_logging_payload``.
|
||||
KNOWN_REQUEST_SERVICE_TIERS = frozenset(
|
||||
{"auto", "batch", "default", "flex", "priority", "scale", "standard", "standard_only"}
|
||||
)
|
||||
|
||||
|
||||
def _get_budget_metrics_per_request_timeout() -> float:
|
||||
raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
|
||||
|
|
@ -1245,6 +1254,7 @@ class PrometheusLogger(CustomLogger):
|
|||
client_ip=standard_logging_payload["metadata"].get("requester_ip_address"),
|
||||
user_agent=standard_logging_payload["metadata"].get("user_agent"),
|
||||
stream=(str(standard_logging_payload.get("stream")) if litellm.prometheus_emit_stream_label else None),
|
||||
service_tier=get_service_tier_from_standard_logging_payload(standard_logging_payload),
|
||||
)
|
||||
|
||||
if user_api_key is not None and isinstance(user_api_key, str) and user_api_key.startswith("sk-"):
|
||||
|
|
@ -4098,6 +4108,44 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
|
|||
return result
|
||||
|
||||
|
||||
def get_service_tier_from_standard_logging_payload(
|
||||
standard_logging_payload: StandardLoggingPayload,
|
||||
) -> str | None:
|
||||
"""
|
||||
Resolve the service tier a request ran on, for the ``service_tier`` label.
|
||||
|
||||
The tier the provider actually served wins over the tier the caller asked for,
|
||||
so latency and spend stay segmentable when the request said ``auto`` and the
|
||||
provider picked the concrete tier. Providers report the served tier either at
|
||||
the top level of the response (OpenAI, Bedrock, Groq) or on the usage object
|
||||
(Anthropic).
|
||||
|
||||
Streaming responses carry no served tier, so the requested tier is the
|
||||
fallback. That value is caller-controlled and survives param mapping even
|
||||
where the provider then ignores it (Bedrock and Groq accept the request and
|
||||
drop an unrecognized tier), so it is only labelled when it names a known
|
||||
tier; otherwise one caller could mint a Prometheus series per string. Values
|
||||
the provider itself reports are not caller-controlled and stay unrestricted,
|
||||
so a tier a provider adds later is still labelled correctly.
|
||||
"""
|
||||
response = standard_logging_payload.get("response")
|
||||
usage_object = standard_logging_payload.get("metadata", {}).get("usage_object")
|
||||
|
||||
served_candidates: tuple[object, ...] = (
|
||||
response.get("service_tier") if isinstance(response, dict) else None,
|
||||
usage_object.get("service_tier") if isinstance(usage_object, dict) else None,
|
||||
)
|
||||
served_tier = next((tier for tier in served_candidates if isinstance(tier, str) and tier), None)
|
||||
if served_tier is not None:
|
||||
return served_tier
|
||||
|
||||
model_parameters = standard_logging_payload.get("model_parameters")
|
||||
requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None
|
||||
if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS:
|
||||
return requested_tier
|
||||
return None
|
||||
|
||||
|
||||
def _get_combined_custom_metadata_from_standard_logging_payload(
|
||||
standard_logging_payload: Optional[dict],
|
||||
) -> Dict[str, Any]:
|
||||
|
|
|
|||
|
|
@ -5554,18 +5554,6 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
|
|||
litellm_params["_langfuse_masking_function"] = masking_fn
|
||||
litellm_params["metadata"] = metadata
|
||||
|
||||
## check user_api_key_metadata for sensitive logging keys
|
||||
cleaned_user_api_key_metadata = {}
|
||||
if "user_api_key_metadata" in metadata and isinstance(metadata["user_api_key_metadata"], dict):
|
||||
for k, v in metadata["user_api_key_metadata"].items():
|
||||
if k == "logging": # prevent logging user logging keys
|
||||
cleaned_user_api_key_metadata[k] = "scrubbed_by_litellm_for_sensitive_keys"
|
||||
else:
|
||||
cleaned_user_api_key_metadata[k] = v
|
||||
|
||||
metadata["user_api_key_metadata"] = cleaned_user_api_key_metadata
|
||||
litellm_params["metadata"] = metadata
|
||||
|
||||
return litellm_params
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import mimetypes
|
|||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from enum import Enum
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union, cast, overload
|
||||
|
||||
from jinja2.sandbox import ImmutableSandboxedEnvironment
|
||||
|
|
@ -5350,7 +5351,9 @@ def prompt_factory(
|
|||
def get_attribute_or_key(tool_or_function, attribute, default=None):
|
||||
if hasattr(tool_or_function, attribute):
|
||||
return getattr(tool_or_function, attribute)
|
||||
return tool_or_function.get(attribute, default)
|
||||
if isinstance(tool_or_function, Mapping):
|
||||
return tool_or_function.get(attribute, default)
|
||||
return default
|
||||
|
||||
|
||||
class NormalizedToolCall(TypedDict):
|
||||
|
|
@ -5379,14 +5382,18 @@ def _parse_tool_call_arguments(raw: Any, tool_name: Optional[str], context: str)
|
|||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _tool_calls_from_chat_completion_response(response: Any) -> list[NormalizedToolCall]:
|
||||
def _tool_calls_from_chat_completion_response(
|
||||
response: Any, include_all_choices: bool = False
|
||||
) -> list[NormalizedToolCall]:
|
||||
choices = get_attribute_or_key(response, "choices", None)
|
||||
if not (isinstance(choices, list) and choices):
|
||||
return []
|
||||
message = get_attribute_or_key(choices[0], "message", None)
|
||||
tool_calls = get_attribute_or_key(message, "tool_calls", None) if message else None
|
||||
if not isinstance(tool_calls, list):
|
||||
return []
|
||||
tool_calls: list[Any] = []
|
||||
for choice in choices if include_all_choices else choices[:1]:
|
||||
message = get_attribute_or_key(choice, "message", None)
|
||||
choice_tool_calls = get_attribute_or_key(message, "tool_calls", None) if message else None
|
||||
if isinstance(choice_tool_calls, list):
|
||||
tool_calls.extend(choice_tool_calls)
|
||||
result: list[NormalizedToolCall] = []
|
||||
for tc in tool_calls:
|
||||
fn = get_attribute_or_key(tc, "function", None)
|
||||
|
|
@ -5449,7 +5456,7 @@ def _tool_calls_from_anthropic_messages_response(response: Any) -> list[Normaliz
|
|||
return result
|
||||
|
||||
|
||||
def get_tool_calls_from_response(response: Any) -> list[NormalizedToolCall]:
|
||||
def get_tool_calls_from_response(response: Any, include_all_choices: bool = False) -> list[NormalizedToolCall]:
|
||||
"""
|
||||
Extract tool/function calls from a response object into a normalized
|
||||
``{"id", "name", "arguments"}`` shape, regardless of which API surface
|
||||
|
|
@ -5457,11 +5464,20 @@ def get_tool_calls_from_response(response: Any) -> list[NormalizedToolCall]:
|
|||
the Responses API (``output`` items of type ``function_call``), or the
|
||||
Anthropic Messages API (``content`` blocks of type ``tool_use``).
|
||||
|
||||
``include_all_choices`` decides the chat-completions scope: the default
|
||||
reads only ``choices[0]``, which is what consumers that act on THE reply
|
||||
(e.g. guardrails rebuilding the primary assistant message) want; usage
|
||||
accounting passes True because every choice of an ``n>1`` request costs
|
||||
money and its tool calls really ran. The other surfaces have a single
|
||||
output, so the flag has no effect on them.
|
||||
|
||||
Callers that only care about a specific tool should filter the result by
|
||||
``name`` themselves -- this returns every tool call found.
|
||||
"""
|
||||
chat_tool_calls = _tool_calls_from_chat_completion_response(response, include_all_choices=include_all_choices)
|
||||
if chat_tool_calls:
|
||||
return chat_tool_calls
|
||||
for extractor in (
|
||||
_tool_calls_from_chat_completion_response,
|
||||
_tool_calls_from_responses_api_response,
|
||||
_tool_calls_from_anthropic_messages_response,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -393,24 +393,25 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
if compaction_event is not None:
|
||||
return compaction_event
|
||||
|
||||
if self.sent_content_block_start is False:
|
||||
self.sent_content_block_start = True
|
||||
self.sent_content_block_finish = False
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
}
|
||||
)
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
|
||||
should_start_new_block = self._should_start_new_content_block(chunk)
|
||||
if should_start_new_block:
|
||||
is_opening_first_block = self.sent_content_block_start is False
|
||||
if is_opening_first_block and self._is_blank_delta(chunk):
|
||||
continue
|
||||
if is_opening_first_block:
|
||||
self.sent_content_block_start = True
|
||||
self.sent_content_block_finish = False
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": self.current_content_block_start,
|
||||
}
|
||||
)
|
||||
elif should_start_new_block:
|
||||
self._increment_content_block_index()
|
||||
|
||||
# applied_edits only needs to flow to the final message_delta
|
||||
|
|
@ -447,7 +448,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
# ``not self.queued_usage_chunk``.
|
||||
continue
|
||||
|
||||
if should_start_new_block and not self.sent_content_block_finish:
|
||||
if should_start_new_block and not is_opening_first_block and not self.sent_content_block_finish:
|
||||
# Queue the sequence: content_block_stop -> content_block_start
|
||||
# -> (optionally) the trigger chunk's delta.
|
||||
#
|
||||
|
|
@ -615,25 +616,25 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
if compaction_event is not None:
|
||||
return compaction_event
|
||||
|
||||
if self.sent_content_block_start is False:
|
||||
self.sent_content_block_start = True
|
||||
self.sent_content_block_finish = False
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
}
|
||||
)
|
||||
return self.chunk_queue.popleft()
|
||||
|
||||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
|
||||
# Check if we need to start a new content block
|
||||
should_start_new_block = self._should_start_new_content_block(chunk)
|
||||
if should_start_new_block:
|
||||
is_opening_first_block = self.sent_content_block_start is False
|
||||
if is_opening_first_block and self._is_blank_delta(chunk):
|
||||
continue
|
||||
if is_opening_first_block:
|
||||
self.sent_content_block_start = True
|
||||
self.sent_content_block_finish = False
|
||||
self.chunk_queue.append(
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": self.current_content_block_index,
|
||||
"content_block": self.current_content_block_start,
|
||||
}
|
||||
)
|
||||
elif should_start_new_block:
|
||||
self._increment_content_block_index()
|
||||
|
||||
# applied_edits only needs to flow to the final message_delta
|
||||
|
|
@ -664,7 +665,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
# Check if this processed chunk has a stop_reason - hold it for next chunk
|
||||
|
||||
if not self.queued_usage_chunk:
|
||||
if should_start_new_block and not self.sent_content_block_finish:
|
||||
if should_start_new_block and not is_opening_first_block and not self.sent_content_block_finish:
|
||||
# Queue the sequence: content_block_stop -> content_block_start
|
||||
# -> (optionally) the trigger chunk's delta.
|
||||
#
|
||||
|
|
@ -875,6 +876,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
return False
|
||||
return bool(delta.get(_delta_payload_field(delta_type)))
|
||||
|
||||
@staticmethod
|
||||
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
|
||||
choice = chunk.choices[0]
|
||||
if choice.finish_reason is not None:
|
||||
return False
|
||||
delta = choice.delta
|
||||
if getattr(delta, "tool_calls", None):
|
||||
return False
|
||||
if getattr(delta, "content", None):
|
||||
return False
|
||||
if getattr(delta, "reasoning_content", None):
|
||||
return False
|
||||
if getattr(delta, "thinking_blocks", None):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
|
||||
"""
|
||||
Determine if we should start a new content block based on the processed chunk.
|
||||
|
|
|
|||
|
|
@ -331,6 +331,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"thinking",
|
||||
"output_format",
|
||||
"output_config",
|
||||
"stop_sequences",
|
||||
]
|
||||
|
||||
def _is_web_search_tool(self, tool: Dict[str, Any]) -> bool:
|
||||
|
|
@ -615,7 +616,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
thinking_type = thinking.get("type", "disabled")
|
||||
|
||||
if thinking_type == "disabled":
|
||||
return None
|
||||
return "none"
|
||||
elif thinking_type == "enabled":
|
||||
return reasoning_effort_from_thinking_budget(thinking.get("budget_tokens", 0))
|
||||
elif thinking_type == "adaptive":
|
||||
|
|
@ -683,25 +684,37 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
thinking
|
||||
)
|
||||
if reasoning_effort:
|
||||
summary = thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
}
|
||||
}
|
||||
elif auto_summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
}
|
||||
return {"reasoning_effort": reasoning_effort}
|
||||
return {
|
||||
"reasoning_effort": LiteLLMAnthropicMessagesAdapter._apply_reasoning_summary_wrapping(
|
||||
reasoning_effort, thinking
|
||||
)
|
||||
}
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _apply_reasoning_summary_wrapping(
|
||||
reasoning_effort: str,
|
||||
thinking: Dict[str, Any],
|
||||
) -> Any:
|
||||
"""
|
||||
Apply the reasoning_effort/summary wrapping rules shared by every
|
||||
thinking->reasoning_effort translation path.
|
||||
|
||||
Disabled thinking always stays a plain string - there's no reasoning
|
||||
trace to summarize, and non-Claude providers (e.g. Fireworks) expect
|
||||
reasoning_effort as a plain string, not a summary dict.
|
||||
"""
|
||||
thinking_type = thinking.get("type") if isinstance(thinking, dict) else None
|
||||
if thinking_type == "disabled":
|
||||
return reasoning_effort
|
||||
|
||||
summary = thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
if summary:
|
||||
return {"effort": reasoning_effort, "summary": summary}
|
||||
if is_reasoning_auto_summary_enabled():
|
||||
return {"effort": reasoning_effort, "summary": "detailed"}
|
||||
return reasoning_effort
|
||||
|
||||
def translate_anthropic_tool_choice_to_openai(
|
||||
self, tool_choice: AnthropicMessagesToolChoice
|
||||
) -> ChatCompletionToolChoiceValues:
|
||||
|
|
@ -919,6 +932,18 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_choice=cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
|
||||
def _translate_stop_sequences_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
if "stop_sequences" not in anthropic_message_request:
|
||||
return
|
||||
stop_sequences = anthropic_message_request["stop_sequences"]
|
||||
if not stop_sequences:
|
||||
return
|
||||
new_kwargs["stop"] = stop_sequences
|
||||
|
||||
def _translate_tools_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
|
|
@ -976,32 +1001,17 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if not reasoning_effort:
|
||||
return
|
||||
|
||||
thinking_type = thinking.get("type") if isinstance(thinking, dict) else None
|
||||
|
||||
# For adaptive thinking, override with output_config.effort if available
|
||||
if isinstance(thinking, dict) and thinking.get("type") == "adaptive":
|
||||
if thinking_type == "adaptive":
|
||||
output_config = anthropic_message_request.get("output_config")
|
||||
if isinstance(output_config, dict) and output_config.get("effort"):
|
||||
reasoning_effort = output_config["effort"]
|
||||
|
||||
summary = thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
},
|
||||
)
|
||||
elif auto_summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
},
|
||||
)
|
||||
else:
|
||||
new_kwargs["reasoning_effort"] = reasoning_effort
|
||||
new_kwargs["reasoning_effort"] = self._apply_reasoning_summary_wrapping(
|
||||
reasoning_effort, cast(Dict[str, Any], thinking)
|
||||
)
|
||||
|
||||
def _translate_output_format_to_openai(
|
||||
self,
|
||||
|
|
@ -1098,6 +1108,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT STOP_SEQUENCES
|
||||
self._translate_stop_sequences_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT
|
||||
self._translate_output_format_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
|
|
|
|||
|
|
@ -143,13 +143,26 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
client: Union[ClientSession, Callable[[], ClientSession]],
|
||||
ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None,
|
||||
owns_session: bool = True,
|
||||
session_factory: Callable[[], ClientSession] | None = None,
|
||||
):
|
||||
self.client = client
|
||||
self._ssl_verify = ssl_verify # Store for per-request SSL override
|
||||
super().__init__(client=client, owns_session=owns_session)
|
||||
# Store the client factory for recreating sessions when needed
|
||||
if callable(client):
|
||||
self._client_factory = client
|
||||
default_factory: Callable[[], ClientSession] = client if callable(client) else ClientSession
|
||||
self._client_factory: Callable[[], ClientSession] = session_factory or default_factory
|
||||
|
||||
def _rebuild_session(self) -> ClientSession:
|
||||
"""
|
||||
Build a replacement session from the configured factory.
|
||||
|
||||
The replacement is reachable only from this transport, so the transport
|
||||
owns it from here on even when it was originally handed a session it did
|
||||
not own (the proxy's shared session).
|
||||
"""
|
||||
session = self._client_factory()
|
||||
self._owns_session = True
|
||||
return session
|
||||
|
||||
def _get_valid_client_session(self) -> ClientSession:
|
||||
"""
|
||||
|
|
@ -158,24 +171,16 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
This handles the case where the session was created in a different
|
||||
event loop that may have been closed (common in CI/CD environments).
|
||||
"""
|
||||
from aiohttp.client import ClientSession
|
||||
|
||||
# If we don't have a client or it's not a ClientSession, create one
|
||||
if not isinstance(self.client, ClientSession):
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
self.client = self._rebuild_session()
|
||||
# Don't return yet - check if the newly created session is valid
|
||||
|
||||
# Check if the session itself is closed
|
||||
if self.client.closed:
|
||||
verbose_logger.debug("Session is closed, creating new session")
|
||||
# Create a new session
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
self.client = self._rebuild_session()
|
||||
return self.client
|
||||
|
||||
# Check if the existing session is still valid for the current event loop
|
||||
|
|
@ -188,7 +193,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
# Close old session to prevent leaks
|
||||
old_session = self.client
|
||||
try:
|
||||
if not old_session.closed:
|
||||
if self._owns_session and not old_session.closed:
|
||||
try:
|
||||
asyncio.create_task(old_session.close())
|
||||
except RuntimeError:
|
||||
|
|
@ -198,17 +203,11 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
verbose_logger.debug(f"Error closing old session: {e}")
|
||||
|
||||
# Create a new session in the current event loop
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
self.client = self._rebuild_session()
|
||||
|
||||
except (RuntimeError, AttributeError):
|
||||
# If we can't check the loop or session is invalid, recreate it
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
self.client = self._rebuild_session()
|
||||
|
||||
return self.client
|
||||
|
||||
|
|
@ -303,10 +302,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
|
|||
if "Session is closed" in str(e):
|
||||
verbose_logger.debug(f"Session closed during request, retrying with new session: {e}")
|
||||
# Force creation of a new session
|
||||
if hasattr(self, "_client_factory") and callable(self._client_factory):
|
||||
self.client = self._client_factory()
|
||||
else:
|
||||
self.client = ClientSession()
|
||||
self.client = self._rebuild_session()
|
||||
client_session = self.client
|
||||
|
||||
# Retry the request with the new session
|
||||
|
|
|
|||
|
|
@ -1013,17 +1013,6 @@ class AsyncHTTPHandler:
|
|||
|
||||
verbose_logger.debug("Creating AiohttpTransport...")
|
||||
|
||||
# Use shared session if provided and valid
|
||||
if shared_session is not None and not shared_session.closed:
|
||||
verbose_logger.debug(f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})")
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=shared_session,
|
||||
ssl_verify=ssl_for_transport,
|
||||
owns_session=False,
|
||||
)
|
||||
|
||||
# Create new session only if none provided or existing one is invalid
|
||||
verbose_logger.debug("NEW SESSION: Creating new ClientSession (no shared session provided)")
|
||||
transport_connector_kwargs = {
|
||||
"keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
|
||||
"ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
|
||||
|
|
@ -1041,11 +1030,26 @@ class AsyncHTTPHandler:
|
|||
if socket_factory is not None:
|
||||
transport_connector_kwargs["socket_factory"] = socket_factory
|
||||
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=lambda: ClientSession(
|
||||
def session_factory() -> ClientSession:
|
||||
return ClientSession(
|
||||
connector=TCPConnector(**transport_connector_kwargs),
|
||||
trust_env=trust_env,
|
||||
),
|
||||
)
|
||||
|
||||
# Use shared session if provided and valid
|
||||
if shared_session is not None and not shared_session.closed:
|
||||
verbose_logger.debug(f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})")
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=shared_session,
|
||||
ssl_verify=ssl_for_transport,
|
||||
owns_session=False,
|
||||
session_factory=session_factory,
|
||||
)
|
||||
|
||||
# Create new session only if none provided or existing one is invalid
|
||||
verbose_logger.debug("NEW SESSION: Creating new ClientSession (no shared session provided)")
|
||||
return LiteLLMAiohttpTransport(
|
||||
client=session_factory,
|
||||
ssl_verify=ssl_for_transport,
|
||||
)
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,9 +1,9 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientPageRoot"]
|
||||
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/1l2mgm5v3tjci.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/2n26sdz53rm0a.js","/litellm-asset-prefix/_next/static/chunks/11khk745tfruy.js","/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","/litellm-asset-prefix/_next/static/chunks/0-k_4_s7m108w.js","/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","/litellm-asset-prefix/_next/static/chunks/395_vbpmrlvpu.js","/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","/litellm-asset-prefix/_next/static/chunks/2s_ce-opzrkzr.js","/litellm-asset-prefix/_next/static/chunks/2ptdxz8qnchh_.js","/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","/litellm-asset-prefix/_next/static/chunks/3y674jhwchpcq.js","/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","/litellm-asset-prefix/_next/static/chunks/3f9uewf5w-e-p.js","/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js"],"default"]
|
||||
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"OutletBoundary"]
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientPageRoot"]
|
||||
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js","/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","/litellm-asset-prefix/_next/static/chunks/1fmx49l6q8v39.js","/litellm-asset-prefix/_next/static/chunks/0ww76lz_0cphv.js","/litellm-asset-prefix/_next/static/chunks/2uc2pi4ob086w.js","/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","/litellm-asset-prefix/_next/static/chunks/0dsiq_ok1yngk.js","/litellm-asset-prefix/_next/static/chunks/3srzg1la93pwv.js","/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","/litellm-asset-prefix/_next/static/chunks/0am68mi9t9cb6.js","/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js"],"default"]
|
||||
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"OutletBoundary"]
|
||||
7:"$Sreact.suspense"
|
||||
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0-k_4_s7m108w.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/395_vbpmrlvpu.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2s_ce-opzrkzr.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2ptdxz8qnchh_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/3y674jhwchpcq.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/3f9uewf5w-e-p.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0ljiPmkOdq7_yE4sZoXlJ"}
|
||||
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/00g6xfr4yow7h.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2kcxwg1mpncp6.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1fmx49l6q8v39.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0ww76lz_0cphv.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2uc2pi4ob086w.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2hu1vyy-5pv13.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2l25bmiiw9ixp.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1uz3jt-tj9lkf.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3drq2_k-jeio2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dsiq_ok1yngk.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/3srzg1la93pwv.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/17nqbxvhztf3k.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0kap_rdm2-lem.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/09l_m9l1emin2.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/323l6h8s7ahat.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/112n0hv3cc2rg.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/199uwr871eene.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/105643dvf00hu.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/22iools_e0k44.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/0am68mi9t9cb6.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/23-g73xaw3kap.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
8:null
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientSegmentRoot"]
|
||||
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/1l2mgm5v3tjci.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/2n26sdz53rm0a.js","/litellm-asset-prefix/_next/static/chunks/11khk745tfruy.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1l2mgm5v3tjci.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/2n26sdz53rm0a.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/11khk745tfruy.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0ljiPmkOdq7_yE4sZoXlJ"}
|
||||
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ClientSegmentRoot"]
|
||||
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1ioy8obpggx93.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3_3dj4vdy-3xy.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1zr7rrk4wkmju.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2cngn5bal3278.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2up3bks93iqds.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0fk0i3e2aixp7.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0zduf1gntl_f8.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dbvgsc7ha049.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/1vquuz09jxl5_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/17-6zku8f68gf.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/1iakmimqrlpn0.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1a0bgy7kzrj91.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/0f5fel02jwglw.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/0g8wwba6umbim.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/1di-caw05k3tq.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
|
||||
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ViewportBoundary"]
|
||||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"MetadataBoundary"]
|
||||
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"ViewportBoundary"]
|
||||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"IconMark"]
|
||||
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0ljiPmkOdq7_yE4sZoXlJ"}
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"IconMark"]
|
||||
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"AuthProvider"]
|
||||
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
4:I[557951,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"AuthProvider"]
|
||||
5:I[339756,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
6:I[837457,["/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js"],"default"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/23vtcpdpp2h9h.css","style"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/23vtcpdpp2h9h.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0map77ee0fk0e.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0ljiPmkOdq7_yE4sZoXlJ"}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","style"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0vggytdohwe7o.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3c02m_kr-u94p.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1jfookxfajkeo.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/23vtcpdpp2h9h.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3254j4ut19q6_.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0ljiPmkOdq7_yE4sZoXlJ"}
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"qXutWsQW5C1Pf62WxTkEI"}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue